feat(api): generate and commit the API deployment OpenAPI spec

backend/api_v2/api_deployment_views.py

@@ -5,6 +5,13 @@
 
 from django.db.models import F, OuterRef, QuerySet, Subquery
 from django.http import HttpResponse
+from drf_spectacular.utils import (
+    OpenApiParameter,
+    OpenApiResponse,
+    extend_schema,
+    extend_schema_serializer,
+    extend_schema_view,
+)
 from permissions.membership_views import OwnerManagementMixin
 from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
 from permissions.resource_share_views import ResourceShareManagementMixin
@@ -37,6 +44,7 @@
 from api_v2.serializers import (
     APIDeploymentListSerializer,
     APIDeploymentSerializer,
+    APIExecutionResponseSerializer,
     DeploymentResponseSerializer,
     ExecutionQuerySerializer,
     ExecutionRequestSerializer,
@@ -50,6 +58,149 @@
 logger = logging.getLogger(__name__)
 
 
+# Declares no field of its own, so every backend parameter arrives free and a
+# change to the real serializer moves the spec. It exists only to carry a
+# caller-facing description in place of the implementation docstring, and to
+# keep the published model name stable.
+@extend_schema_serializer(component_name="ExecuteRequest")
+class ExecuteRequest(ExecutionRequestSerializer):
+    """The documents to run, and the options that shape the result.
+
+    Supply `files`, `presigned_urls`, or both.
+    """
+
+
+class FileResult(serializers.Serializer):
+    file = serializers.CharField()
+    file_execution_id = serializers.CharField(required=False)
+    status = serializers.CharField(required=False)
+    result = serializers.JSONField(required=False)
+    metadata = serializers.JSONField(required=False)
+    metrics = serializers.JSONField(required=False)
+    error = serializers.CharField(required=False, allow_null=True)
+
+
+# Subclasses the serializer that builds the response, so a field added or
+# removed there moves the spec. Docstrings on these annotation serializers are
+# published as the client-facing model description, so they are written for
+# the caller rather than the maintainer.
+class ExecutionMessage(APIExecutionResponseSerializer):
+    """The execution's identity and, once it has finished, its per-file
+    results.
+    """
+
+    # The one field that has to be restated: the real declaration is an
+    # untyped JSONField, which gives generated clients nothing to work with.
+    # The backend also sends `result: null` while pending, and without
+    # allow_null the generated deserialiser iterates None and crashes.
+    result = FileResult(many=True, required=False, allow_null=True)
+
+
+class ExecuteResponse(serializers.Serializer):
+    message = ExecutionMessage()
+
+
+class StatusResponse(serializers.Serializer):
+    status = serializers.CharField()
+    message = FileResult(many=True, required=False, allow_null=True)
+
+
+class ErrorResponse(serializers.Serializer):
+    status = serializers.CharField(required=False)
+    message = serializers.JSONField(required=False, allow_null=True)
+
+
+# The pattern the route itself enforces, restated so a generated client can
+# reject a mistyped identifier without a round trip.
+PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"}
+
+DEPLOYMENT_PATH_PARAMETERS = [
+    OpenApiParameter(
+        "org_name",
+        PATH_SEGMENT,
+        OpenApiParameter.PATH,
+        description="Organization identifier.",
+    ),
+    OpenApiParameter(
+        "api_name",
+        PATH_SEGMENT,
+        OpenApiParameter.PATH,
+        description="API deployment name.",
+    ),
+]
+
+
+DEPLOYMENT_AUTH = [{"deploymentKey": []}]
+
+# Every failure a caller has to handle. Declared explicitly because a client
+# generated without them treats an authentication or rate-limit response as an
+# unknown status and has nothing to branch on.
+DEPLOYMENT_ERRORS = {
+    400: OpenApiResponse(ErrorResponse, description="The request failed validation."),
+    401: OpenApiResponse(ErrorResponse, description="The API key is not valid."),
+    403: OpenApiResponse(ErrorResponse, description="No API key was supplied."),
+    404: OpenApiResponse(ErrorResponse, description="No such active deployment."),
+    429: OpenApiResponse(
+        ErrorResponse, description="Too many concurrent executions; retry later."
+    ),
+    500: ErrorResponse,
+}
+
+EXECUTE_DESCRIPTION = (
+    "Execute an API deployment against one or more documents.\n\n"
+    "Supply the documents either as `files` (multipart upload) or as "
+    "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is "
+    f"rejected, and the two together may not exceed "
+    f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n"
+    "With the default `timeout` of -1 the call returns as soon as the "
+    "execution is queued; read the outcome from the status endpoint."
+)
+
+STATUS_DESCRIPTION = (
+    "Read the result of a previously started execution.\n\n"
+    "This read is one-shot: the first call that observes a completed execution "
+    "acknowledges it and the stored result is discarded, so every later call "
+    "for that execution answers 406. Poll while the execution is pending, and "
+    "keep the payload of the call that returns it — it cannot be fetched again."
+)
+
+
+# The generated clients take their command names, module paths and request
+# shapes from here, so this block is part of the public API surface.
+@extend_schema_view(
+    post=extend_schema(
+        operation_id="execute",
+        tags=["deployment"],
+        auth=DEPLOYMENT_AUTH,
+        parameters=DEPLOYMENT_PATH_PARAMETERS,
+        request={"multipart/form-data": ExecuteRequest},
+        responses={
+            200: ExecuteResponse,
+            409: OpenApiResponse(
+                ErrorResponse, description="The deployment has no active API key."
+            ),
+            422: ExecuteResponse,
+            **DEPLOYMENT_ERRORS,
+        },
+        description=EXECUTE_DESCRIPTION,
+    ),
+    get=extend_schema(
+        operation_id="status",
+        tags=["deployment"],
+        auth=DEPLOYMENT_AUTH,
+        parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer],
+        responses={
+            200: StatusResponse,
+            406: OpenApiResponse(
+                ErrorResponse,
+                description="The result was already consumed by an earlier call.",
+            ),
+            422: StatusResponse,
+            **DEPLOYMENT_ERRORS,
+        },
+        description=STATUS_DESCRIPTION,
+    ),
+)
 class DeploymentExecution(views.APIView):
     def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:
         """To remove csrf request for public API.

backend/api_v2/deployment_spec_urls.py

@@ -0,0 +1,27 @@
+"""URLconf used only to generate the API deployment OpenAPI spec.
+
+``api_v2.execution_urls`` is an included sub-urlconf, so generating against it
+directly yields paths without the prefix it is mounted at — a spec describing
+URLs the server does not serve. The mount is selected out of the served
+urlconf rather than restated here, so a change to where the deployment API is
+mounted moves the generated paths with it.
+"""
+
+from django.core.exceptions import ImproperlyConfigured
+
+from backend import base_urls
+
+DEPLOYMENT_URLCONF = "api_v2.execution_urls"
+
+urlpatterns = [
+    entry
+    for entry in base_urls.urlpatterns
+    if getattr(getattr(entry, "urlconf_name", None), "__name__", None)
+    == DEPLOYMENT_URLCONF
+]
+
+if not urlpatterns:
+    raise ImproperlyConfigured(
+        f"{DEPLOYMENT_URLCONF} is not mounted in backend.base_urls; the API "
+        "deployment spec would be generated for no routes at all."
+    )

backend/api_v2/management/commands/generate_docstudio_spec.py

@@ -0,0 +1,97 @@
+"""Regenerate the committed API deployment OpenAPI spec.
+
+The spec is the contract the published clients and their generated SDKs are
+built from, so it is committed and CI fails on drift: change a route, a
+serializer or the schema annotation, and regenerate in the same PR.
+
+    uv run python manage.py generate_docstudio_spec           # from backend/
+    uv run python manage.py generate_docstudio_spec --check   # no write, drift is an error
+
+The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so regenerate in an
+environment that does not override it — the committed artifact describes the
+deployment as it is served publicly, not as one installation mounts it.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError
+from drf_spectacular.drainage import GENERATOR_STATS
+from drf_spectacular.generators import SchemaGenerator
+
+DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json"
+URLCONF = "api_v2.deployment_spec_urls"
+REGENERATE = "uv run python manage.py generate_docstudio_spec"
+
+
+class SpecGenerationFailed(CommandError):
+    """Raised when the generator had to guess."""
+
+
+def render_spec() -> str:
+    """The committed artifact, byte for byte.
+
+    Shared with the drift test: two copies of this could disagree, and then
+    the gate rejects exactly the file the command it names produces.
+    """
+    GENERATOR_STATS.reset()
+    schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True)
+    if GENERATOR_STATS:
+        # spectacular downgrades "unable to guess serializer" to a warning and
+        # writes a plausible, wrong operation. Nothing downstream can tell that
+        # apart from an annotation that is simply thin.
+        diagnostics = "\n".join(
+            f"  {severity}: {message}"
+            for severity, cache in (
+                ("error", GENERATOR_STATS._error_cache),
+                ("warning", GENERATOR_STATS._warn_cache),
+            )
+            for message in cache
+        )
+        raise SpecGenerationFailed(
+            f"The generator reported problems, so the spec would describe an "
+            f"API nobody implements:\n{diagnostics}"
+        )
+    # Sorted keys are what make the committed artifact a usable drift signal.
+    return json.dumps(schema, indent=2, sort_keys=True) + "\n"
+
+
+class Command(BaseCommand):
+    help = "Generate the API deployment OpenAPI spec."
+
+    def add_arguments(self, parser: Any) -> None:
+        parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+        parser.add_argument(
+            "--check",
+            action="store_true",
+            help="Fail if the file on disk differs, instead of writing it.",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        rendered = render_spec()
+
+        out: Path = options["out"]
+        if options["check"]:
+            current = out.read_text() if out.exists() else ""
+            if current != rendered:
+                raise CommandError(
+                    f"{out} is out of date. Run `{REGENERATE}` from `backend/` "
+                    "and commit the result."
+                )
+            self.stdout.write(f"{out} is up to date")
+            return
+
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(rendered)
+        schema = json.loads(rendered)
+        operations = sum(
+            1
+            for methods in schema["paths"].values()
+            for method in methods
+            if method in {"get", "post", "put", "patch", "delete"}
+        )
+        self.stdout.write(
+            f"{out}: {len(schema['paths'])} paths, {operations} operations, "
+            f"{len(schema.get('components', {}).get('schemas', {}))} schemas"
+        )

backend/api_v2/serializers.py

@@ -6,6 +6,8 @@
 
 from django.apps import apps
 from django.core.validators import RegexValidator
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
 from pipeline_v2.models import Pipeline
 from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
 from rest_framework import serializers
@@ -218,6 +220,13 @@ def to_representation(self, instance: APIKey) -> OrderedDict[str, Any]:
         return representation
 
 
+@extend_schema_field(OpenApiTypes.BINARY)
+class UploadField(FileField):
+    """A bare ``FileField`` maps to ``format: uri`` -- correct on output, wrong
+    for a multipart upload, and generators emit ``str`` for it.
+    """
+
+
 class ExecutionRequestSerializer(TagParamsSerializer):
     """Execution request serializer.
 
@@ -320,7 +329,7 @@ def validate_custom_data(self, value):
         return value
 
     files = ListField(
-        child=FileField(),
+        child=UploadField(),
         required=False,
         allow_empty=True,
     )