From 2da19b4ac2ff554f0a87f07e823931da34e3e29a Mon Sep 17 00:00:00 2001 From: Jack Decker <24392469+jackowfish@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:03:14 -0400 Subject: [PATCH] feat: volume file reads --- README.md | 38 ++- porter_sandbox/__init__.py | 100 +++++++- porter_sandbox/_async_base_client.py | 76 ++++-- porter_sandbox/_base_client.py | 61 ++++- porter_sandbox/_binary.py | 58 +++++ porter_sandbox/_config.py | 3 + porter_sandbox/_models.py | 24 +- porter_sandbox/_retries.py | 3 + porter_sandbox/enums.py | 7 +- porter_sandbox/resources/volumes.py | 75 +++++- porter_sandbox/sandbox.py | 3 + porter_sandbox/sandboxes.py | 6 +- porter_sandbox/volume.py | 368 +++++++++++++++++++++++++++ porter_sandbox/volumes.py | 31 ++- scripts/sync-generated.sh | 67 ----- tests/test_models_round_trip.py | 16 ++ 16 files changed, 812 insertions(+), 124 deletions(-) create mode 100644 porter_sandbox/_binary.py create mode 100644 porter_sandbox/volume.py delete mode 100755 scripts/sync-generated.sh diff --git a/README.md b/README.md index 2460e33..fdc7991 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,27 @@ with Porter() as porter: sb.terminate() ``` +Volumes can be created up front and mounted into sandboxes at launch, and their +contents browsed and read straight from the volume handle: + +```python +volume = porter.volumes.create(name="my-data") + +sb = porter.sandboxes.create( + image="python:3.11-alpine", + volume_mounts={"/mnt/my-data": volume.id}, +) + +for file in volume.listdir("/checkpoints"): + print(file.path, file.size_bytes) + +config = volume.read_text("/checkpoints/config.json") + +with open("model.safetensors", "wb") as out: + for chunk in volume.stream("/checkpoints/model.safetensors"): + out.write(chunk) +``` + Inside a sandbox-enabled Porter cluster, the SDK connects to the in-cluster sandbox API at `http://sandbox-api.porter-sandbox-system.svc.cluster.local:8080` automatically, with no configuration needed. @@ -70,10 +91,13 @@ async with AsyncPorter() as porter: ## Layout -- `porter_sandbox/porter.py`, resource namespace modules like `porter_sandbox/sandboxes.py`, `porter_sandbox/_client.py`, `_models.py`, `enums.py`, `_errors.py`, `resources/` — generated from the [sandbox-api OpenAPI spec](https://github.com/porter-dev/workstation/tree/main/code/sandbox/schemas) via the [sdk-gen workspace](https://github.com/porter-dev/workstation/tree/main/code/sandbox/sdk-gen). Do not edit by hand. -- `porter_sandbox/sandbox.py` — hand-written rich sandbox handle used by the generated `sandboxes` namespace -- `porter_sandbox/_base_client.py` / `_async_base_client.py` — hand-written sync and async HTTP transports -- `porter_sandbox/_config.py`, `_retries.py` — hand-written runtime (env-var resolution, retry/backoff) +Everything under `porter_sandbox/` is generated from the Porter Sandbox OpenAPI +spec. Do not edit it by hand - changes there are overwritten on the next release. + +- `porter_sandbox/sandbox.py`, `porter_sandbox/volume.py` - sync and async `Sandbox` and `Volume` handles +- `porter_sandbox/porter.py` and resource namespace modules like `porter_sandbox/sandboxes.py` - public client and namespaces +- `porter_sandbox/_client.py`, `_models.py`, `enums.py`, `_errors.py`, `resources/` - low-level client, models, and errors +- `porter_sandbox/_base_client.py`, `_async_base_client.py`, `_config.py`, `_retries.py` - sync and async HTTP transports, env-var resolution, retry/backoff ## Development @@ -83,9 +107,3 @@ pytest ruff check . mypy ``` - -To pull in a fresh generation from the sdk-gen workspace: - -```bash -./scripts/sync-generated.sh /path/to/workstation/code/sandbox/sdk-gen/out/python -``` diff --git a/porter_sandbox/__init__.py b/porter_sandbox/__init__.py index 95ede97..1a7ace6 100644 --- a/porter_sandbox/__init__.py +++ b/porter_sandbox/__init__.py @@ -1,32 +1,118 @@ +# Generated by oagen. DO NOT EDIT. + + from __future__ import annotations +from ._binary import BinaryContent, ContentRange from ._client import AsyncPorterSandboxApiClient, PorterSandboxApiClient from ._errors import ( AuthenticationError, NotFoundError, RateLimitError, SandboxError, + SandboxTimeoutError, ServerError, ) +from ._models import ( + CountPoint, + CountResponse, + CreateResponse, + Error, + ExecRequest, + ExecResponse, + ExecTarget, + FilterValuesResponse, + HealthResponse, + ListResponse, + LogLine, + LogsResponse, + LookupResult, + Pagination, + ReadinessResponse, + SandboxDomainSpec, + SandboxEgressSpec, + SandboxNetworkingSpec, + SandboxSpec, + StatusResponse, + VolumeFileEntry, + VolumeFileListResponse, + VolumeListResponse, + VolumeSpec, +) +from ._models import Volume as VolumeRecord +from .enums import ( + FilterValuesResponsePhases, + LogLineLevel, + SandboxDomainSpecVisibility, + SandboxesPhase, + StatusResponsePhase, + VolumeFileEntryType, + VolumePhase, +) +from .healthz import AsyncHealthz, Healthz from .porter import AsyncPorter, Porter +from .readyz import AsyncReadyz, Readyz from .sandbox import AsyncSandbox, Sandbox from .sandboxes import AsyncSandboxes, Sandboxes +from .volume import AsyncVolume, Volume, VolumeFile from .volumes import AsyncVolumes, Volumes __all__ = [ - "Porter", + "AsyncHealthz", "AsyncPorter", - "Sandbox", + "AsyncPorterSandboxApiClient", + "AsyncReadyz", "AsyncSandbox", - "Sandboxes", "AsyncSandboxes", - "Volumes", + "AsyncVolume", "AsyncVolumes", - "PorterSandboxApiClient", - "AsyncPorterSandboxApiClient", - "SandboxError", "AuthenticationError", + "BinaryContent", + "ContentRange", + "CountPoint", + "CountResponse", + "CreateResponse", + "Error", + "ExecRequest", + "ExecResponse", + "ExecTarget", + "FilterValuesResponse", + "FilterValuesResponsePhases", + "HealthResponse", + "Healthz", + "ListResponse", + "LogLine", + "LogLineLevel", + "LogsResponse", + "LookupResult", "NotFoundError", + "Pagination", + "Porter", + "PorterSandboxApiClient", "RateLimitError", + "ReadinessResponse", + "Readyz", + "Sandbox", + "SandboxDomainSpec", + "SandboxDomainSpecVisibility", + "SandboxEgressSpec", + "SandboxError", + "SandboxNetworkingSpec", + "SandboxSpec", + "SandboxTimeoutError", + "Sandboxes", + "SandboxesPhase", "ServerError", + "StatusResponse", + "StatusResponsePhase", + "Volume", + "VolumeFile", + "VolumeFileEntry", + "VolumeFileEntryType", + "VolumeFileListResponse", + "VolumeListResponse", + "VolumePhase", + "VolumeRecord", + "VolumeSpec", + "Volumes", ] diff --git a/porter_sandbox/_async_base_client.py b/porter_sandbox/_async_base_client.py index a7e8946..4e11666 100644 --- a/porter_sandbox/_async_base_client.py +++ b/porter_sandbox/_async_base_client.py @@ -1,12 +1,16 @@ +# Generated by oagen. DO NOT EDIT. + + from __future__ import annotations -import json as json_lib from collections.abc import Mapping from typing import Any import httpx from httpx._client import UseClientDefault +from ._base_client import _decode_body, _error_message, _request_headers +from ._binary import BinaryContent, _binary_content from ._config import Config from ._errors import SandboxError, SandboxTimeoutError, error_for_status from ._retries import DEFAULT_MAX_RETRIES, should_retry, sleep_for_attempt @@ -14,31 +18,13 @@ USER_AGENT = "porter-sandbox-python/0.0.1" -def _decode_body(response: httpx.Response) -> Any: - if response.status_code == 204 or not response.content: - return None - content_type = response.headers.get("content-type", "") - if "application/json" in content_type: - try: - return response.json() - except json_lib.JSONDecodeError: - return response.text - return response.text - - -def _error_message(body: Any, status_code: int) -> str: - if isinstance(body, dict): - error = body.get("error") - if isinstance(error, str): - return error - return f"HTTP {status_code}" - - class _AsyncBaseClient: """Async HTTP transport shared by all generated async resource classes. Owns the httpx.AsyncClient, header injection, retry loop, and error mapping. - Resource methods call `await self._client._request(...)`. + Resource methods call `await self._client._request(...)`, or + `_request_binary(...)` for the endpoints that answer with bytes rather than + JSON. """ def __init__( @@ -80,9 +66,52 @@ async def _request( path: str, params: Mapping[str, Any] | None = None, json: Any = None, + headers: Mapping[str, str | None] | None = None, timeout: float | None | UseClientDefault = httpx.USE_CLIENT_DEFAULT, retry: bool = True, ) -> Any: + response = await self._send( + method=method, + path=path, + params=params, + json=json, + headers=headers, + accept="application/json", + timeout=timeout, + retry=retry, + ) + return _decode_body(response) + + async def _request_binary( + self, + *, + method: str, + path: str, + params: Mapping[str, Any] | None = None, + headers: Mapping[str, str | None] | None = None, + ) -> BinaryContent: + response = await self._send( + method=method, + path=path, + params=params, + json=None, + headers=headers, + accept="application/octet-stream", + ) + return _binary_content(response) + + async def _send( + self, + *, + method: str, + path: str, + params: Mapping[str, Any] | None, + json: Any, + headers: Mapping[str, str | None] | None, + accept: str, + timeout: float | None | UseClientDefault = httpx.USE_CLIENT_DEFAULT, + retry: bool = True, + ) -> httpx.Response: # `timeout=None` disables the timeout entirely, used for long-running # calls like exec, where the API works for the full duration of the # request. `retry=False` is for calls that must not be re-sent (exec): @@ -98,6 +127,7 @@ async def _request( url=path, params=params, json=json, + headers=_request_headers(headers, accept), timeout=timeout, ) except httpx.TimeoutException as exc: @@ -113,7 +143,7 @@ async def _request( raise SandboxError(f"Network error: {exc}") from exc if 200 <= response.status_code < 300: - return _decode_body(response) + return response if should_retry(response.status_code) and attempt < max_retries: await sleep_for_attempt(attempt) diff --git a/porter_sandbox/_base_client.py b/porter_sandbox/_base_client.py index 28bb139..1bca67b 100644 --- a/porter_sandbox/_base_client.py +++ b/porter_sandbox/_base_client.py @@ -1,3 +1,6 @@ +# Generated by oagen. DO NOT EDIT. + + from __future__ import annotations import json as json_lib @@ -7,6 +10,7 @@ import httpx from httpx._client import UseClientDefault +from ._binary import BinaryContent, _binary_content from ._config import Config from ._errors import SandboxError, SandboxTimeoutError, error_for_status from ._retries import DEFAULT_MAX_RETRIES, should_retry, sleep_for_attempt_sync @@ -34,11 +38,20 @@ def _error_message(body: Any, status_code: int) -> str: return f"HTTP {status_code}" +def _request_headers(headers: Mapping[str, str | None] | None, accept: str) -> dict[str, str]: + sent = {"Accept": accept} + for name, value in (headers or {}).items(): + if value is not None: + sent[name] = value + return sent + + class _BaseClient: """Sync HTTP transport shared by all generated sync resource classes. Owns the httpx.Client, header injection, retry loop, and error mapping. - Resource methods call `self._client._request(...)`. + Resource methods call `self._client._request(...)`, or `_request_binary(...)` + for the endpoints that answer with bytes rather than JSON. """ def __init__( @@ -80,9 +93,52 @@ def _request( path: str, params: Mapping[str, Any] | None = None, json: Any = None, + headers: Mapping[str, str | None] | None = None, timeout: float | None | UseClientDefault = httpx.USE_CLIENT_DEFAULT, retry: bool = True, ) -> Any: + response = self._send( + method=method, + path=path, + params=params, + json=json, + headers=headers, + accept="application/json", + timeout=timeout, + retry=retry, + ) + return _decode_body(response) + + def _request_binary( + self, + *, + method: str, + path: str, + params: Mapping[str, Any] | None = None, + headers: Mapping[str, str | None] | None = None, + ) -> BinaryContent: + response = self._send( + method=method, + path=path, + params=params, + json=None, + headers=headers, + accept="application/octet-stream", + ) + return _binary_content(response) + + def _send( + self, + *, + method: str, + path: str, + params: Mapping[str, Any] | None, + json: Any, + headers: Mapping[str, str | None] | None, + accept: str, + timeout: float | None | UseClientDefault = httpx.USE_CLIENT_DEFAULT, + retry: bool = True, + ) -> httpx.Response: # `timeout=None` disables the timeout entirely, used for long-running # calls like exec, where the API works for the full duration of the # request. `retry=False` is for calls that must not be re-sent (exec): @@ -98,6 +154,7 @@ def _request( url=path, params=params, json=json, + headers=_request_headers(headers, accept), timeout=timeout, ) except httpx.TimeoutException as exc: @@ -113,7 +170,7 @@ def _request( raise SandboxError(f"Network error: {exc}") from exc if 200 <= response.status_code < 300: - return _decode_body(response) + return response if should_retry(response.status_code) and attempt < max_retries: sleep_for_attempt_sync(attempt) diff --git a/porter_sandbox/_binary.py b/porter_sandbox/_binary.py new file mode 100644 index 0000000..363b5ff --- /dev/null +++ b/porter_sandbox/_binary.py @@ -0,0 +1,58 @@ +# Generated by oagen. DO NOT EDIT. + + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime +from email.utils import parsedate_to_datetime + +import httpx + +# `bytes 0-1023/8192`. Absent on a whole-file response. +_CONTENT_RANGE_RE = re.compile(r"^bytes (\d+)-(\d+)/(\d+)$") + + +@dataclass(frozen=True) +class ContentRange: + """A byte range within a file, read back from a response's Content-Range header.""" + + start: int + end: int + size: int + + +@dataclass(frozen=True) +class BinaryContent: + """Raw bytes from an octet-stream endpoint, with the metadata needed to read a file in pieces.""" + + data: bytes + # Which bytes of the file arrived and how large the file is, set when the + # request asked for a range. + content_range: ContentRange | None + last_modified: datetime | None + + +def _parse_content_range(header: str | None) -> ContentRange | None: + match = _CONTENT_RANGE_RE.match(header) if header else None + if match is None: + return None + return ContentRange(start=int(match[1]), end=int(match[2]), size=int(match[3])) + + +def _parse_http_date(header: str | None) -> datetime | None: + if not header: + return None + try: + return parsedate_to_datetime(header) + except (TypeError, ValueError): + return None + + +def _binary_content(response: httpx.Response) -> BinaryContent: + return BinaryContent( + data=response.content, + content_range=_parse_content_range(response.headers.get("content-range")), + last_modified=_parse_http_date(response.headers.get("last-modified")), + ) diff --git a/porter_sandbox/_config.py b/porter_sandbox/_config.py index 8b89889..76cbffa 100644 --- a/porter_sandbox/_config.py +++ b/porter_sandbox/_config.py @@ -1,3 +1,6 @@ +# Generated by oagen. DO NOT EDIT. + + from __future__ import annotations import base64 diff --git a/porter_sandbox/_models.py b/porter_sandbox/_models.py index 0a38263..1b906a7 100644 --- a/porter_sandbox/_models.py +++ b/porter_sandbox/_models.py @@ -10,6 +10,7 @@ LogLineLevel, SandboxDomainSpecVisibility, StatusResponsePhase, + VolumeFileEntryType, VolumePhase, ) @@ -98,6 +99,10 @@ class SandboxDomainSpec(BaseModel): visibility: SandboxDomainSpecVisibility | None = Field(default=None, description="Which sandbox ingress serves the domain when the cluster has both a\npublic and a private one. Omit to default to whichever is\nconfigured, public winning. Rejected when the sandbox exposes a\nport but the requested ingress is not configured on the cluster.\n") +class SandboxEgressSpec(BaseModel): + allowed_destinations: list[str] = Field(description="Destinations the sandbox may reach; all other outbound traffic is\ndenied. An entry is a hostname (api.example.com), a wildcard\n(*.example.com) matching any host under that domain but not the\ndomain itself, an IP literal, a CIDR range (203.0.113.0/24), or the\ncluster-internal hostname of a Service on the sandbox's cluster\n(name.namespace.svc.cluster.local), which allows the Service's\nbacking pods as they change. Enforcement is transparent at the\nnetwork layer, so any protocol and client works without proxy\nconfiguration. An empty list denies all egress; omit egress entirely\nto leave the sandbox's internet access unrestricted.\n") + + class SandboxNetworkingSpec(BaseModel): port: int = Field(description="Port the workload listens on; the per-sandbox Service targets it on\nthe pod. Privileged ports (1-1023) are not allowed.\n") domains: list[SandboxDomainSpec] | None = Field(default=None, description="Domains the port is served on through a sandbox ingress. Omit to\nserve the port at the default hostname through the default ingress.\nCurrently only one entry is supported.\n") @@ -113,6 +118,7 @@ class SandboxSpec(BaseModel): env_groups: list[str] | None = Field(default=None, description="Names of environment groups on the cluster whose variables are\ninjected into the sandbox, resolved to their latest version at create\ntime. On a key conflict a later group wins over an earlier one, and\nan explicit env entry wins over any group value.\n") volume_mounts: dict[str, str] | None = Field(default=None, description="Volumes to mount, keyed by the absolute mount path inside the\nsandbox; values are volume IDs.\n") networking: list[SandboxNetworkingSpec] | None = Field(default=None, description="Network exposure for the sandbox. Omit to expose nothing. Currently\nonly one entry is supported.\n") + egress: SandboxEgressSpec | None = Field(default=None) ttl_seconds: int | None = Field(default=None, description="Maximum lifetime in seconds, counted from creation. The sandbox is\nterminated once it elapses. Omit for no limit.\n") @@ -133,11 +139,27 @@ class StatusResponse(BaseModel): class Volume(BaseModel): id: str = Field(description="Volume ID, assigned when the volume is created. All volume\noperations address volumes by ID; the name is informational.\n") name: str = Field(description="Volume name") + path: str = Field(description="Subdirectory, relative to the shared sandbox volumes mount, where this\nvolume's data lives. An app that mounts the cluster's sandbox volumes\nreads this volume at /.\n") phase: VolumePhase = Field(description="Current lifecycle phase of the volume") attached_to: list[str] = Field(description="IDs of sandboxes the volume is attached to") created_at: str = Field(description="When the volume was created") +class VolumeFileEntry(BaseModel): + name: str = Field(description="Entry name within its parent directory, without any path separator\n(e.g. model.bin). Entries nest, so a directory name is carried once\non the directory itself rather than repeated in the path of every\nentry beneath it. Reconstruct an entry's path relative to the volume\nroot by joining the names from the listing's path down to the entry.\n") + type: VolumeFileEntryType = Field(description="Whether the entry is a regular file or a directory") + size_bytes: int = Field(description="Size of the entry in bytes. Zero for directories.") + modified_at: str | None = Field(default=None, description="When the entry was last modified. Always set on files; may be absent\non directories when the volume's storage keeps no directory\nmodification time.\n") + truncated: bool | None = Field(default=None, description="Set on a directory entry whose contents the walk did not fully read:\nthe entry budget ran out before entering it, or the directory alone\nholds more entries than the budget. List the directory directly to\nread more of them. Never set on files.\n") + entries: list[VolumeFileEntry] | None = Field(default=None, description="Entries directly inside this directory, directories before files and\neach group by name. Omitted on files, and on a directory whose\nentries the walk did not read (which carries truncated instead).\n") + + +class VolumeFileListResponse(BaseModel): + path: str = Field(description="The walked directory, relative to the volume root") + entries: list[VolumeFileEntry] = Field(description="Entries directly inside the walked directory, directories before\nfiles and each group by name. Entries nest, so everything the walk\nread below this level hangs off these entries rather than appearing\nhere as a flat path.\n") + truncated: bool = Field(description="Whether the walk stopped at its entry budget before reading every\nentry. Directories left unread or partially read carry their own\ntruncated marker.\n") + + class VolumeListResponse(BaseModel): volumes: list[Volume] = Field(description="All volumes in the cluster") @@ -146,4 +168,4 @@ class VolumeSpec(BaseModel): name: str | None = Field(default=None, description="Volume name, unique within the cluster. Must be a valid DNS label\n(lowercase alphanumeric and dashes). Defaults to the volume's id\nwhen omitted.\n") -__all__ = ["CountPoint", "CountResponse", "CreateResponse", "Error", "ExecRequest", "ExecResponse", "ExecTarget", "FilterValuesResponse", "HealthResponse", "ListResponse", "LogLine", "LogsResponse", "LookupResult", "Pagination", "ReadinessResponse", "SandboxDomainSpec", "SandboxNetworkingSpec", "SandboxSpec", "StatusResponse", "Volume", "VolumeListResponse", "VolumeSpec"] +__all__ = ["CountPoint", "CountResponse", "CreateResponse", "Error", "ExecRequest", "ExecResponse", "ExecTarget", "FilterValuesResponse", "HealthResponse", "ListResponse", "LogLine", "LogsResponse", "LookupResult", "Pagination", "ReadinessResponse", "SandboxDomainSpec", "SandboxEgressSpec", "SandboxNetworkingSpec", "SandboxSpec", "StatusResponse", "Volume", "VolumeFileEntry", "VolumeFileListResponse", "VolumeListResponse", "VolumeSpec"] diff --git a/porter_sandbox/_retries.py b/porter_sandbox/_retries.py index 5df2cfc..a1e1d85 100644 --- a/porter_sandbox/_retries.py +++ b/porter_sandbox/_retries.py @@ -1,3 +1,6 @@ +# Generated by oagen. DO NOT EDIT. + + from __future__ import annotations import asyncio diff --git a/porter_sandbox/enums.py b/porter_sandbox/enums.py index b4f90ba..20ef16a 100644 --- a/porter_sandbox/enums.py +++ b/porter_sandbox/enums.py @@ -44,10 +44,15 @@ class StatusResponsePhase(str, Enum): TERMINATED = "terminated" +class VolumeFileEntryType(str, Enum): + FILE = "file" + DIRECTORY = "directory" + + class VolumePhase(str, Enum): PENDING = "pending" READY = "ready" FAILED = "failed" -__all__ = ["FilterValuesResponsePhases", "LogLineLevel", "SandboxDomainSpecVisibility", "SandboxesPhase", "StatusResponsePhase", "VolumePhase"] +__all__ = ["FilterValuesResponsePhases", "LogLineLevel", "SandboxDomainSpecVisibility", "SandboxesPhase", "StatusResponsePhase", "VolumeFileEntryType", "VolumePhase"] diff --git a/porter_sandbox/resources/volumes.py b/porter_sandbox/resources/volumes.py index 24ff367..b7bbadb 100644 --- a/porter_sandbox/resources/volumes.py +++ b/porter_sandbox/resources/volumes.py @@ -9,7 +9,8 @@ from .._async_base_client import _AsyncBaseClient from .._base_client import _BaseClient -from .._models import LookupResult, Volume, VolumeListResponse, VolumeSpec +from .._binary import BinaryContent +from .._models import LookupResult, Volume, VolumeFileListResponse, VolumeListResponse, VolumeSpec _M = TypeVar("_M", bound=BaseModel) @@ -78,6 +79,42 @@ def delete_volume(self, id: str) -> None: self._client._request(method="DELETE", path=path) return None + def list_volume_files(self, id: str, path: str | None = None, search: str | None = None) -> VolumeFileListResponse: + """ + List volume files + + List the files and directories under a path inside a volume, read from + the shared sandbox volumes mount. The listing walks the tree + breadth-first up to a server-side entry budget: small trees come back + complete in one response, and directories the walk did not fully read + are marked truncated for clients to list directly. Requires the control + plane to have the mount configured; without it every listing reports + the files API as unavailable. + """ + path_ = f"/v1/volume/{id}/files" + params: dict[str, Any] = {} + if path is not None: + params["path"] = path + if search is not None: + params["search"] = search + response = self._client._request(method="GET", path=path_, params=params) + return _coerce(VolumeFileListResponse, response) + + def read_volume_file(self, id: str, path: str, range: str | None = None) -> BinaryContent: + """ + Read volume file + + Stream a file's raw bytes from a volume. The response is the file + content itself, so clients bound how much they read: send a single + byte range (e.g. bytes=0-1048575) to read part of the file, or omit + the Range header for the whole file. + """ + path_ = f"/v1/volume/{id}/files/content" + params: dict[str, Any] = {} + params["path"] = path + response: BinaryContent = self._client._request_binary(method="GET", path=path_, params=params, headers={"Range": range}) + return response + class AsyncVolumes: """Volumes resource.""" @@ -137,3 +174,39 @@ async def delete_volume(self, id: str) -> None: path = f"/v1/volume/{id}" await self._client._request(method="DELETE", path=path) return None + + async def list_volume_files(self, id: str, path: str | None = None, search: str | None = None) -> VolumeFileListResponse: + """ + List volume files + + List the files and directories under a path inside a volume, read from + the shared sandbox volumes mount. The listing walks the tree + breadth-first up to a server-side entry budget: small trees come back + complete in one response, and directories the walk did not fully read + are marked truncated for clients to list directly. Requires the control + plane to have the mount configured; without it every listing reports + the files API as unavailable. + """ + path_ = f"/v1/volume/{id}/files" + params: dict[str, Any] = {} + if path is not None: + params["path"] = path + if search is not None: + params["search"] = search + response = await self._client._request(method="GET", path=path_, params=params) + return _coerce(VolumeFileListResponse, response) + + async def read_volume_file(self, id: str, path: str, range: str | None = None) -> BinaryContent: + """ + Read volume file + + Stream a file's raw bytes from a volume. The response is the file + content itself, so clients bound how much they read: send a single + byte range (e.g. bytes=0-1048575) to read part of the file, or omit + the Range header for the whole file. + """ + path_ = f"/v1/volume/{id}/files/content" + params: dict[str, Any] = {} + params["path"] = path + response: BinaryContent = await self._client._request_binary(method="GET", path=path_, params=params, headers={"Range": range}) + return response diff --git a/porter_sandbox/sandbox.py b/porter_sandbox/sandbox.py index 4fc9591..bbeb5a3 100644 --- a/porter_sandbox/sandbox.py +++ b/porter_sandbox/sandbox.py @@ -1,3 +1,6 @@ +# Generated by oagen. DO NOT EDIT. + + from __future__ import annotations import builtins diff --git a/porter_sandbox/sandboxes.py b/porter_sandbox/sandboxes.py index 8b1e561..52019e0 100644 --- a/porter_sandbox/sandboxes.py +++ b/porter_sandbox/sandboxes.py @@ -5,7 +5,7 @@ import builtins -from porter_sandbox._models import SandboxNetworkingSpec, SandboxSpec +from porter_sandbox._models import SandboxEgressSpec, SandboxNetworkingSpec, SandboxSpec from porter_sandbox.enums import SandboxesPhase from porter_sandbox.resources.sandboxes import AsyncSandboxes as AsyncSandboxesResource from porter_sandbox.resources.sandboxes import Sandboxes as SandboxesResource @@ -35,6 +35,7 @@ def create( env_groups: list[str] | None = None, volume_mounts: dict[str, str] | None = None, networking: list[SandboxNetworkingSpec] | None = None, + egress: SandboxEgressSpec | None = None, ttl_seconds: int | None = None, ) -> Sandbox: spec = SandboxSpec( @@ -47,6 +48,7 @@ def create( env_groups=env_groups, volume_mounts=volume_mounts, networking=networking, + egress=egress, ttl_seconds=ttl_seconds, ) created = self._resource.create_sandbox(body=spec) @@ -97,6 +99,7 @@ async def create( env_groups: list[str] | None = None, volume_mounts: dict[str, str] | None = None, networking: list[SandboxNetworkingSpec] | None = None, + egress: SandboxEgressSpec | None = None, ttl_seconds: int | None = None, ) -> AsyncSandbox: spec = SandboxSpec( @@ -109,6 +112,7 @@ async def create( env_groups=env_groups, volume_mounts=volume_mounts, networking=networking, + egress=egress, ttl_seconds=ttl_seconds, ) created = await self._resource.create_sandbox(body=spec) diff --git a/porter_sandbox/volume.py b/porter_sandbox/volume.py new file mode 100644 index 0000000..bb52383 --- /dev/null +++ b/porter_sandbox/volume.py @@ -0,0 +1,368 @@ +# Generated by oagen. DO NOT EDIT. + + +from __future__ import annotations + +import builtins +from collections.abc import AsyncIterator, Iterator +from datetime import datetime + +from porter_sandbox._models import Volume as VolumeRecord +from porter_sandbox._models import VolumeFileEntry +from porter_sandbox.enums import VolumeFileEntryType, VolumePhase +from porter_sandbox.resources.volumes import AsyncVolumes as AsyncVolumesResource +from porter_sandbox.resources.volumes import Volumes as VolumesResource + +# How much of a file one request of `stream` reads. Files are read through +# ranged requests, so this bounds how many bytes a caller holds at once. +DEFAULT_CHUNK_BYTES = 8 * 1024 * 1024 + + +def _normalize_path(path: str) -> str: + """Take a volume path with or without a leading slash, return the '/'-rooted form the API echoes back.""" + trimmed = path.strip("/") + return f"/{trimmed}" if trimmed else "/" + + +def _join_path(parent: str, name: str) -> str: + return f"/{name}" if parent == "/" else f"{parent}/{name}" + + +def _byte_range_header(offset: int, length: int | None) -> str: + return f"bytes={offset}-" if length is None else f"bytes={offset}-{offset + length - 1}" + + +def _parse_timestamp(value: str | None) -> datetime | None: + if not value: + return None + # The API sends RFC 3339, where fromisoformat only learned to read the 'Z' + # suffix in Python 3.11. + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +class VolumeFile: + """A file or directory inside a volume.""" + + def __init__(self, entry: VolumeFileEntry, parent_path: str) -> None: + #: Entry name within its parent directory, without any path separator. + self.name = entry.name + #: Path relative to the volume root, e.g. "/models/config.json". + self.path = _join_path(parent_path, entry.name) + self.type: VolumeFileEntryType = entry.type + self.size_bytes = entry.size_bytes + self.modified_at = _parse_timestamp(entry.modified_at) + #: Set on a directory the server's listing walk did not fully read — it + #: holds more entries than one listing covers. `iterdir` reads these + #: directly; `listdir` leaves them for the caller to list. + self.truncated = entry.truncated or False + + @property + def is_directory(self) -> bool: + return self.type == VolumeFileEntryType.DIRECTORY + + @property + def is_file(self) -> bool: + return self.type == VolumeFileEntryType.FILE + + def __repr__(self) -> str: + return f"VolumeFile(path={self.path!r}, type={self.type!r}, size_bytes={self.size_bytes})" + + +class Volume: + """Ergonomic handle for a single volume (sync). + + Constructed by `Porter().volumes`. Holds a back-reference to the generated + volume resource so file reads and lifecycle calls do not need a client to be + re-passed. + + For async usage, see `AsyncVolume` (same surface, async methods). + """ + + def __init__(self, *, record: VolumeRecord, resource: VolumesResource) -> None: + self._record = record + self._volumes = resource + + @property + def id(self) -> str: + return self._record.id + + @property + def name(self) -> str: + return self._record.name + + @property + def phase(self) -> VolumePhase: + return self._record.phase + + @property + def attached_to(self) -> builtins.list[str]: + """IDs of the sandboxes the volume is attached to.""" + return self._record.attached_to + + @property + def created_at(self) -> datetime | None: + return _parse_timestamp(self._record.created_at) + + def refresh(self) -> VolumeRecord: + """Refetch and cache the volume record.""" + self._record = self._volumes.get_volume(id=self.id) + return self._record + + def delete(self) -> None: + """Delete the volume. Fails while it is attached to a sandbox.""" + self._volumes.delete_volume(id=self.id) + + def listdir(self, path: str = "/") -> builtins.list[VolumeFile]: + """The entries directly inside `path`, directories first and each group by name.""" + parent = _normalize_path(path) + listing = self._volumes.list_volume_files(id=self.id, path=parent) + return [VolumeFile(entry, parent) for entry in listing.entries] + + def iterdir(self, path: str = "/") -> Iterator[VolumeFile]: + """Every entry under `path`, depth-first. + + One listing covers a bounded number of entries, so directories it left + unread are listed as the walk reaches them — a large tree costs more + than one request. + """ + parent = _normalize_path(path) + listing = self._volumes.list_volume_files(id=self.id, path=parent) + yield from self._walk_entries(parent, listing.entries, follow_truncated=True) + + def search(self, query: str, *, path: str = "/") -> builtins.list[VolumeFile]: + """Entries under `path` whose name contains `query`, case-insensitively. + + The search runs inside one listing walk, so a tree too large for a + single listing is searched as far as that walk reached. + """ + parent = _normalize_path(path) + listing = self._volumes.list_volume_files(id=self.id, path=parent, search=query) + needle = query.lower() + # The API prunes the tree to matches plus the directories holding them; + # keep only the entries that match so callers get the hits alone. + return [ + file + for file in self._walk_entries(parent, listing.entries, follow_truncated=False) + if needle in file.name.lower() + ] + + def read_file(self, path: str, *, offset: int | None = None, length: int | None = None) -> bytes: + """Read a file's bytes. `offset` and `length` read a slice of it instead of the whole file.""" + ranged = offset is not None or length is not None + content = self._volumes.read_volume_file( + id=self.id, + path=_normalize_path(path), + range=_byte_range_header(offset or 0, length) if ranged else None, + ) + return content.data + + def read_text( + self, + path: str, + *, + offset: int | None = None, + length: int | None = None, + encoding: str = "utf-8", + ) -> str: + """Read a file as text.""" + return self.read_file(path, offset=offset, length=length).decode(encoding) + + def stream(self, path: str, *, chunk_size: int = DEFAULT_CHUNK_BYTES) -> Iterator[bytes]: + """Read a file in chunks, so a large one never lands in memory whole. + + Each chunk is its own ranged request. + """ + file_path = _normalize_path(path) + offset = 0 + + while True: + content = self._volumes.read_volume_file( + id=self.id, + path=file_path, + range=_byte_range_header(offset, chunk_size), + ) + if content.data: + yield content.data + + # An empty file comes back whole, with no Content-Range to page from. + if content.content_range is None: + return + + offset = content.content_range.end + 1 + if offset >= content.content_range.size: + return + + # Depth-first over a listing's nested entries. `follow_truncated` lists the + # directories the server's walk left unread, which `iterdir` wants and + # `search` does not: listing one of those directly would drop the search + # term and pull in everything under it. + def _walk_entries( + self, + parent: str, + entries: builtins.list[VolumeFileEntry], + *, + follow_truncated: bool, + ) -> Iterator[VolumeFile]: + for entry in entries: + file = VolumeFile(entry, parent) + yield file + if not file.is_directory: + continue + if entry.entries is not None: + yield from self._walk_entries( + file.path, entry.entries, follow_truncated=follow_truncated + ) + elif file.truncated and follow_truncated: + yield from self.iterdir(file.path) + + +class AsyncVolume: + """Ergonomic handle for a single volume (async). + + Same surface as `Volume`, but every method is awaitable and the walks are + async iterators. + """ + + def __init__(self, *, record: VolumeRecord, resource: AsyncVolumesResource) -> None: + self._record = record + self._volumes = resource + + @property + def id(self) -> str: + return self._record.id + + @property + def name(self) -> str: + return self._record.name + + @property + def phase(self) -> VolumePhase: + return self._record.phase + + @property + def attached_to(self) -> builtins.list[str]: + """IDs of the sandboxes the volume is attached to.""" + return self._record.attached_to + + @property + def created_at(self) -> datetime | None: + return _parse_timestamp(self._record.created_at) + + async def refresh(self) -> VolumeRecord: + """Refetch and cache the volume record.""" + self._record = await self._volumes.get_volume(id=self.id) + return self._record + + async def delete(self) -> None: + """Delete the volume. Fails while it is attached to a sandbox.""" + await self._volumes.delete_volume(id=self.id) + + async def listdir(self, path: str = "/") -> builtins.list[VolumeFile]: + """The entries directly inside `path`, directories first and each group by name.""" + parent = _normalize_path(path) + listing = await self._volumes.list_volume_files(id=self.id, path=parent) + return [VolumeFile(entry, parent) for entry in listing.entries] + + async def iterdir(self, path: str = "/") -> AsyncIterator[VolumeFile]: + """Every entry under `path`, depth-first. + + One listing covers a bounded number of entries, so directories it left + unread are listed as the walk reaches them — a large tree costs more + than one request. + """ + parent = _normalize_path(path) + listing = await self._volumes.list_volume_files(id=self.id, path=parent) + async for file in self._walk_entries(parent, listing.entries, follow_truncated=True): + yield file + + async def search(self, query: str, *, path: str = "/") -> builtins.list[VolumeFile]: + """Entries under `path` whose name contains `query`, case-insensitively. + + The search runs inside one listing walk, so a tree too large for a + single listing is searched as far as that walk reached. + """ + parent = _normalize_path(path) + listing = await self._volumes.list_volume_files(id=self.id, path=parent, search=query) + needle = query.lower() + # The API prunes the tree to matches plus the directories holding them; + # keep only the entries that match so callers get the hits alone. + return [ + file + async for file in self._walk_entries(parent, listing.entries, follow_truncated=False) + if needle in file.name.lower() + ] + + async def read_file( + self, path: str, *, offset: int | None = None, length: int | None = None + ) -> bytes: + """Read a file's bytes. `offset` and `length` read a slice of it instead of the whole file.""" + ranged = offset is not None or length is not None + content = await self._volumes.read_volume_file( + id=self.id, + path=_normalize_path(path), + range=_byte_range_header(offset or 0, length) if ranged else None, + ) + return content.data + + async def read_text( + self, + path: str, + *, + offset: int | None = None, + length: int | None = None, + encoding: str = "utf-8", + ) -> str: + """Read a file as text.""" + return (await self.read_file(path, offset=offset, length=length)).decode(encoding) + + async def stream( + self, path: str, *, chunk_size: int = DEFAULT_CHUNK_BYTES + ) -> AsyncIterator[bytes]: + """Read a file in chunks, so a large one never lands in memory whole. + + Each chunk is its own ranged request. + """ + file_path = _normalize_path(path) + offset = 0 + + while True: + content = await self._volumes.read_volume_file( + id=self.id, + path=file_path, + range=_byte_range_header(offset, chunk_size), + ) + if content.data: + yield content.data + + # An empty file comes back whole, with no Content-Range to page from. + if content.content_range is None: + return + + offset = content.content_range.end + 1 + if offset >= content.content_range.size: + return + + # Depth-first over a listing's nested entries. `follow_truncated` lists the + # directories the server's walk left unread, which `iterdir` wants and + # `search` does not: listing one of those directly would drop the search + # term and pull in everything under it. + async def _walk_entries( + self, + parent: str, + entries: builtins.list[VolumeFileEntry], + *, + follow_truncated: bool, + ) -> AsyncIterator[VolumeFile]: + for entry in entries: + file = VolumeFile(entry, parent) + yield file + if not file.is_directory: + continue + if entry.entries is not None: + async for nested in self._walk_entries( + file.path, entry.entries, follow_truncated=follow_truncated + ): + yield nested + elif file.truncated and follow_truncated: + async for nested in self.iterdir(file.path): + yield nested diff --git a/porter_sandbox/volumes.py b/porter_sandbox/volumes.py index 5cc4952..d08bf72 100644 --- a/porter_sandbox/volumes.py +++ b/porter_sandbox/volumes.py @@ -3,9 +3,12 @@ from __future__ import annotations -from porter_sandbox._models import Volume, VolumeListResponse, VolumeSpec +import builtins + +from porter_sandbox._models import VolumeSpec from porter_sandbox.resources.volumes import AsyncVolumes as AsyncVolumesResource from porter_sandbox.resources.volumes import Volumes as VolumesResource +from porter_sandbox.volume import AsyncVolume, Volume class Volumes: @@ -27,14 +30,17 @@ def create( spec = VolumeSpec( name=name, ) - return self._resource.create_volume(body=spec) + record = self._resource.create_volume(body=spec) + return Volume(record=record, resource=self._resource) - def list(self) -> VolumeListResponse: - return self._resource.list_volumes() + def list(self) -> builtins.list[Volume]: + response = self._resource.list_volumes() + return [Volume(record=r, resource=self._resource) for r in response.volumes] def get(self, name: str) -> Volume: ref = self._resource.lookup_volume(name=name) - return self._resource.get_volume(id=ref.id) + record = self._resource.get_volume(id=ref.id) + return Volume(record=record, resource=self._resource) def delete(self, name: str) -> None: ref = self._resource.lookup_volume(name=name) @@ -57,18 +63,21 @@ async def create( self, *, name: str | None = None, - ) -> Volume: + ) -> AsyncVolume: spec = VolumeSpec( name=name, ) - return await self._resource.create_volume(body=spec) + record = await self._resource.create_volume(body=spec) + return AsyncVolume(record=record, resource=self._resource) - async def list(self) -> VolumeListResponse: - return await self._resource.list_volumes() + async def list(self) -> builtins.list[AsyncVolume]: + response = await self._resource.list_volumes() + return [AsyncVolume(record=r, resource=self._resource) for r in response.volumes] - async def get(self, name: str) -> Volume: + async def get(self, name: str) -> AsyncVolume: ref = await self._resource.lookup_volume(name=name) - return await self._resource.get_volume(id=ref.id) + record = await self._resource.get_volume(id=ref.id) + return AsyncVolume(record=record, resource=self._resource) async def delete(self, name: str) -> None: ref = await self._resource.lookup_volume(name=name) diff --git a/scripts/sync-generated.sh b/scripts/sync-generated.sh deleted file mode 100755 index ce1ec20..0000000 --- a/scripts/sync-generated.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Sync generated SDK code from the sdk-gen workspace into this repo. -# -# Usage: ./scripts/sync-generated.sh -# -# Copies only files that the emitter owns. Hand-written runtime/domain files -# (_base_client.py, _async_base_client.py, _config.py, _retries.py, sandbox.py) -# are preserved. - -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -SRC="$1" -REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" - -if [[ ! -d "$SRC/porter_sandbox" ]]; then - echo "Error: $SRC/porter_sandbox not found" >&2 - exit 1 -fi - -# Files the emitter owns (overwrite freely). -# Note: porter_sandbox/__init__.py is hand-written (it re-exports the public -# client/domain classes alongside generated wrappers). Don't sync it. -GENERATED_FILES=( - porter_sandbox/_client.py - porter_sandbox/_errors.py - porter_sandbox/_models.py - porter_sandbox/enums.py -) - -for f in "${GENERATED_FILES[@]}"; do - if [[ -f "$SRC/$f" ]]; then - cp "$SRC/$f" "$REPO_ROOT/$f" - echo " copied $f" - fi -done - -# Public client and namespace wrappers are generated too. Copy top-level -# generated modules, but preserve hand-written runtime/domain modules. -for f in "$SRC"/porter_sandbox/*.py; do - name="$(basename "$f")" - case "$name" in - __init__.py|_*.py|enums.py|sandbox.py) - continue - ;; - esac - cp "$f" "$REPO_ROOT/porter_sandbox/$name" - echo " copied porter_sandbox/$name" -done - -# Resources are a whole directory. -rm -rf "$REPO_ROOT/porter_sandbox/resources" -cp -r "$SRC/porter_sandbox/resources" "$REPO_ROOT/porter_sandbox/resources" -echo " copied porter_sandbox/resources/" - -# Generated tests go alongside hand-written ones. -mkdir -p "$REPO_ROOT/tests" -if [[ -f "$SRC/tests/test_models_round_trip.py" ]]; then - cp "$SRC/tests/test_models_round_trip.py" "$REPO_ROOT/tests/test_models_round_trip.py" - echo " copied tests/test_models_round_trip.py" -fi - -echo "Done." diff --git a/tests/test_models_round_trip.py b/tests/test_models_round_trip.py index 09727ac..8d48089 100644 --- a/tests/test_models_round_trip.py +++ b/tests/test_models_round_trip.py @@ -18,8 +18,10 @@ Pagination, ReadinessResponse, SandboxDomainSpec, + SandboxEgressSpec, SandboxNetworkingSpec, SandboxSpec, + VolumeFileListResponse, VolumeListResponse, VolumeSpec, ) @@ -123,6 +125,13 @@ def test_sandbox_domain_spec_round_trip() -> None: assert round_tripped.model_dump(by_alias=True, exclude_none=True) == serialized +def test_sandbox_egress_spec_round_trip() -> None: + instance = SandboxEgressSpec(allowed_destinations=[]) + serialized = instance.model_dump(by_alias=True, exclude_none=True) + round_tripped = SandboxEgressSpec.model_validate(serialized) + assert round_tripped.model_dump(by_alias=True, exclude_none=True) == serialized + + def test_sandbox_networking_spec_round_trip() -> None: instance = SandboxNetworkingSpec(port=1) serialized = instance.model_dump(by_alias=True, exclude_none=True) @@ -137,6 +146,13 @@ def test_sandbox_spec_round_trip() -> None: assert round_tripped.model_dump(by_alias=True, exclude_none=True) == serialized +def test_volume_file_list_response_round_trip() -> None: + instance = VolumeFileListResponse(path="x", entries=[], truncated=True) + serialized = instance.model_dump(by_alias=True, exclude_none=True) + round_tripped = VolumeFileListResponse.model_validate(serialized) + assert round_tripped.model_dump(by_alias=True, exclude_none=True) == serialized + + def test_volume_list_response_round_trip() -> None: instance = VolumeListResponse(volumes=[]) serialized = instance.model_dump(by_alias=True, exclude_none=True)