feat(app-api): gzip JSON responses, with SSE excluded and un-buffered

backend/src/apis/app_api/main.py

@@ -160,8 +160,10 @@ async def lifespan(app: FastAPI):
 # Starlette `add_middleware` prepends, so the LAST-added middleware is
 # outermost. Request-side order is therefore the reverse of the call order
 # below:
-#   request:  SessionRefresh → CSRF → AgentCoreContext → CORS → router
-#   response: router → CORS → AgentCoreContext → CSRF → SessionRefresh
+#   request:  ProxiedRedirect → GZip → SessionRefresh → CSRF →
+#             AgentCoreContext → CORS → router
+#   response: router → CORS → AgentCoreContext → CSRF → SessionRefresh →
+#             GZip → ProxiedRedirect
 # This is the order we need: SessionRefresh has to populate
 # `state.bff_session` before CSRF reads it.
 from apis.shared.middleware.csrf import CSRFMiddleware
@@ -171,6 +173,29 @@ async def lifespan(app: FastAPI):
 app.add_middleware(SessionRefreshMiddleware)
 logger.info("Added BFF session-refresh + CSRF middlewares (dormant until cookie present)")
 
+# gzip the JSON surface. Nothing compressed app-api responses before this:
+# CloudFront's `/api/*` behaviour is deliberately `compress: false` so the
+# edge never buffers a `text/event-stream`, which leaves compression to the
+# origin — the only layer that knows a response's content type rather than
+# guessing from its path. Measured on this repo's own payload shapes at
+# `compresslevel=6`: ~3.0x on a conversation-history response, ~3.9x on a
+# 5,000-row spreadsheet-shaped one. Level 9 (Starlette's default) buys 3%
+# more for 4x the CPU on the large case, so 6 — zlib's own default — it is.
+#
+# `StreamSafeGZipMiddleware` passes `text/event-stream` and already-encoded
+# bodies through untouched *and un-buffered*; see its module docstring for
+# why the second half of that matters on the chat path.
+#
+# Sits one layer inside ProxiedRedirect so it compresses everything the app
+# emits — including the error bodies CSRF and SessionRefresh return — while
+# leaving ProxiedRedirect genuinely outermost. The two never collide:
+# ProxiedRedirect reads and rewrites only `Location`, on responses (3xx)
+# whose bodies are empty or below the compression threshold anyway.
+from apis.shared.middleware.compression import StreamSafeGZipMiddleware
+
+app.add_middleware(StreamSafeGZipMiddleware, minimum_size=500, compresslevel=6)
+logger.info("Added gzip compression middleware (SSE and pre-encoded bodies excluded)")
+
 # Outermost middleware: repair `Location` headers on redirects this app
 # generates for itself (Starlette's `redirect_slashes`, chiefly). Behind
 # CloudFront those come out as `http://api.<domain>/<path>` — the internal

backend/src/apis/inference_api/main.py

@@ -29,7 +29,6 @@
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.gzip import GZipMiddleware
 from contextlib import asynccontextmanager
 import logging
 
@@ -118,14 +117,31 @@ async def lifespan(app: FastAPI):
 register_aws_client_error_handler(app)
 logger.info("Registered AWS ClientError handler")
 
-# Add GZip compression middleware for SSE streams
-# Compresses responses over 1KB, reducing bandwidth by 50-70%
+# Compress responses over 1KB. Despite what this block used to claim, it is
+# emphatically *not* "for SSE streams": Starlette excludes `text/event-stream`
+# from compression by content type, so on the one route this service exists to
+# serve — the `/invocations` SSE turn — it compresses nothing at all.
+#
+# What stock `GZipMiddleware` did do on that route is withhold
+# `http.response.start` until the first body chunk, because until then it can't
+# know whether it will need to set `Content-Encoding`. On an SSE turn the first
+# chunk is the model's first token, so the response headers were arriving behind
+# the agent's entire thinking time — measured locally at 2.0s of pure delay on a
+# turn that stalls 2.0s before its first event, with or without `Accept-Encoding:
+# gzip`. app-api's chat proxy reads this response's `content-type` before it can
+# open its own stream to the SPA, so that delay was propagating all the way to
+# the browser.
+#
+# `StreamSafeGZipMiddleware` keeps the compression and forwards an excluded
+# response's headers immediately; see its module docstring.
+from apis.shared.middleware.compression import StreamSafeGZipMiddleware
+
 app.add_middleware(
-    GZipMiddleware,
+    StreamSafeGZipMiddleware,
     minimum_size=1000,  # Only compress responses > 1KB
     compresslevel=6  # Balance between speed and compression ratio (1-9)
 )
-logger.info("Added GZip middleware for response compression")
+logger.info("Added gzip compression middleware (SSE and pre-encoded bodies excluded)")
 
 # Bridge AgentCore Runtime headers (WorkloadAccessToken, OAuth2CallbackUrl,
 # session ID) into BedrockAgentCoreContext so downstream code can look up

backend/src/apis/shared/middleware/compression.py

@@ -0,0 +1,164 @@
+"""StreamSafeGZipMiddleware — gzip JSON responses without touching SSE.
+
+Nothing compressed app-api responses before this. CloudFront can't do it for
+us: the `/api/*` behaviour is deliberately `compress: false`
+(`spa-distribution-construct.ts`) precisely so CloudFront never buffers a
+`text/event-stream`, and that decision stands — compression belongs at the
+origin, where the response's own content type is known, not at an edge that
+has to guess from a path pattern.
+
+The payoff is on the JSON surface, which is most of what app-api serves; the
+risk is entirely on the SSE surface, which is the product's main path (the
+chat proxy in `app_api/chat/proxy_routes.py`, plus assistants and
+`chat/converse_routes.py`). So this middleware is defined by what it refuses
+to touch.
+
+**Two things are passed straight through, untouched:**
+
+1. Responses whose `Content-Type` is in :data:`EXCLUDED_CONTENT_TYPES` —
+   `text/event-stream` first and foremost, plus payload types that are
+   already compressed on the wire (zip archives, raster images, media,
+   web fonts), where gzip costs CPU and returns nothing.
+2. Responses that already carry a `Content-Encoding`, whatever it is. That
+   header is the origin saying "this body is already encoded"; re-encoding
+   it would produce a body no client can read.
+
+Starlette's own `GZipMiddleware` skips both of those as of its
+`DEFAULT_EXCLUDED_CONTENT_TYPES`/`content_encoding_set` checks, and this
+class reuses that machinery rather than reimplementing it. What it adds is
+the part Starlette does not do: **it sends `http.response.start` eagerly.**
+
+Starlette's responders withhold the response-start message until the first
+`http.response.body` arrives, because until then they can't know whether
+they'll need to set `Content-Encoding` and drop `Content-Length`. That is
+correct for a body they might compress, and wrong for one they've already
+decided not to. On an SSE turn the first body chunk is the model's first
+token — seconds away, or up to `_SSE_KEEPALIVE_SECONDS` if the turn opens
+with a silent tool call — and today those response headers go out
+immediately. Buffering them until the first event would delay every client's
+"stream is open" transition behind the agent's thinking time, on the one
+path in this service least worth adding a layer to (the same reasoning that
+made `ProxiedRedirectMiddleware` raw ASGI rather than `BaseHTTPMiddleware`).
+
+So: as soon as the response-start message identifies an excluded response,
+this forwards it and flips into pure passthrough for the rest of the
+exchange. Everything else — threshold, `Vary`, streaming compression,
+`http.response.pathsend` — is Starlette's, unmodified.
+
+Non-HTTP scopes (the voice WebSocket proxy) never reach a responder at all.
+"""
+
+from __future__ import annotations
+
+from starlette.datastructures import Headers
+from starlette.middleware.gzip import (
+    DEFAULT_EXCLUDED_CONTENT_TYPES,
+    GZipMiddleware,
+    GZipResponder,
+    IdentityResponder,
+)
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+#: Content-type prefixes this middleware never compresses.
+#:
+#: `text/event-stream` is the correctness entry — gzipping an SSE stream
+#: delays or breaks event flushes. It comes from Starlette's own default so
+#: the two can't drift; the rest are efficiency entries, payload types that
+#: arrive already compressed and would only burn CPU for ~0% gain. Matched
+#: with `str.startswith`, so a charset parameter (`text/event-stream;
+#: charset=utf-8`, which Starlette appends to every `text/*` media type)
+#: still matches.
+#:
+#: Deliberately *not* excluded: `application/pdf` and `image/svg+xml`, both
+#: of which compress usefully and both of which CloudFront itself lists as
+#: compressible.
+EXCLUDED_CONTENT_TYPES: tuple[str, ...] = (
+    *DEFAULT_EXCLUDED_CONTENT_TYPES,
+    "application/gzip",
+    "application/x-7z-compressed",
+    "application/x-bzip2",
+    "application/x-gzip",
+    "application/x-zip-compressed",
+    "application/zip",
+    "audio/",
+    "font/woff",
+    "image/avif",
+    "image/gif",
+    "image/jpeg",
+    "image/png",
+    "image/webp",
+    "video/",
+)
+
+
+def is_excluded(headers: Headers) -> bool:
+    """Whether this response must be passed through uncompressed.
+
+    True for an already-encoded body and for any content type in
+    :data:`EXCLUDED_CONTENT_TYPES`. A response with no `Content-Type` at all
+    is not excluded — Starlette's size threshold already covers the empty
+    bodies that case usually means.
+    """
+    if "content-encoding" in headers:
+        return True
+    return headers.get("content-type", "").startswith(EXCLUDED_CONTENT_TYPES)
+
+
+class _EagerPassthroughMixin:
+    """Forward an excluded response's start message without buffering it.
+
+    Mixed in ahead of Starlette's responders so `super()` still owns every
+    response this does *not* claim.
+    """
+
+    #: Class-level default; flipped on the instance by the first
+    #: `http.response.start` that turns out to be excluded.
+    _passthrough = False
+
+    async def send_with_compression(self, message: Message) -> None:
+        if not self._passthrough:
+            if message["type"] != "http.response.start":
+                await super().send_with_compression(message)
+                return
+            if not is_excluded(Headers(raw=message["headers"])):
+                await super().send_with_compression(message)
+                return
+            # Excluded: nothing downstream will alter the headers, so the
+            # client can have them now rather than when the first chunk
+            # lands. `started` keeps the base class consistent in case
+            # anything else consults it.
+            self._passthrough = True
+            self.started = True
+        await self.send(message)
+
+
+class _IdentityPassthroughResponder(_EagerPassthroughMixin, IdentityResponder):
+    """Non-gzip clients: adds `Vary`, compresses nothing."""
+
+
+class _GZipPassthroughResponder(_EagerPassthroughMixin, GZipResponder):
+    """gzip-capable clients: compresses everything not excluded."""
+
+
+class StreamSafeGZipMiddleware(GZipMiddleware):
+    """`GZipMiddleware` that never buffers an excluded response.
+
+    Same constructor and same behaviour for compressible responses; see the
+    module docstring for what changes and why.
+    """
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            # WebSockets (voice mode) and lifespan: not ours.
+            await self.app(scope, receive, send)
+            return
+
+        responder: ASGIApp
+        if "gzip" in Headers(scope=scope).get("Accept-Encoding", ""):
+            responder = _GZipPassthroughResponder(
+                self.app, self.minimum_size, compresslevel=self.compresslevel
+            )
+        else:
+            responder = _IdentityPassthroughResponder(self.app, self.minimum_size)
+
+        await responder(scope, receive, send)

backend/tests/apis/app_api/test_middleware_stack.py

@@ -0,0 +1,74 @@
+"""The app-api middleware stack is ordered on purpose — lock the order down.
+
+`main.py` documents the intended request-side order:
+
+    ProxiedRedirect → GZip → SessionRefresh → CSRF → AgentCoreContext
+    → CORS → router
+
+Starlette's `add_middleware` prepends, so that order is the reverse of the
+call order in the module and easy to break by moving a call. Two placements
+in particular carry a reason:
+
+* ProxiedRedirect stays outermost, as its own docstring requires — it has to
+  see the final response headers.
+* GZip sits directly inside it, so it compresses everything the app emits
+  (including the error bodies CSRF and SessionRefresh return) while leaving
+  ProxiedRedirect's `Location` rewriting untouched.
+"""
+
+from __future__ import annotations
+
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.testclient import TestClient
+
+from apis.app_api.main import app
+from apis.shared.middleware.agentcore_context import AgentCoreContextMiddleware
+from apis.shared.middleware.compression import StreamSafeGZipMiddleware
+from apis.shared.middleware.csrf import CSRFMiddleware
+from apis.shared.middleware.proxied_redirect import (
+    FORWARDED_PREFIX_HEADER,
+    ProxiedRedirectMiddleware,
+)
+from apis.shared.middleware.session_refresh import SessionRefreshMiddleware
+
+#: Outermost first — `app.user_middleware` is in request order.
+EXPECTED_ORDER = [
+    ProxiedRedirectMiddleware,
+    StreamSafeGZipMiddleware,
+    SessionRefreshMiddleware,
+    CSRFMiddleware,
+    AgentCoreContextMiddleware,
+    CORSMiddleware,
+]
+
+
+def test_middleware_order_matches_the_documented_stack() -> None:
+    assert [m.cls for m in app.user_middleware] == EXPECTED_ORDER
+
+
+def test_compression_is_configured_for_json_not_for_cpu_burn() -> None:
+    gzip_middleware = next(
+        m for m in app.user_middleware if m.cls is StreamSafeGZipMiddleware
+    )
+
+    # Level 9 (Starlette's default) costs ~4x the CPU of level 6 for ~3%
+    # more compression on this repo's largest payloads.
+    assert gzip_middleware.kwargs["compresslevel"] == 6
+    assert gzip_middleware.kwargs["minimum_size"] == 500
+
+
+def test_redirect_rewriting_survives_the_compression_layer() -> None:
+    """ProxiedRedirect still owns `Location` with GZip underneath it."""
+    client = TestClient(app, follow_redirects=False)
+
+    response = client.get(
+        "/agents/",
+        headers={
+            "Host": "api.dev.boisestate.ai",
+            "Accept-Encoding": "gzip",
+            FORWARDED_PREFIX_HEADER: "/api",
+        },
+    )
+
+    assert response.status_code == 307
+    assert response.headers["location"] == "/api/agents"

backend/tests/apis/inference_api/test_middleware_stack.py

@@ -0,0 +1,38 @@
+"""inference-api must not go back to stock `GZipMiddleware`.
+
+This service exists to serve one route — the `/invocations` SSE turn — and
+Starlette's stock gzip middleware withholds `http.response.start` until the
+first body chunk arrives. On an SSE turn that chunk is the model's first
+token, so the response headers landed behind the agent's whole thinking time,
+with or without `Accept-Encoding: gzip`. app-api's chat proxy reads this
+response's `content-type` before it can open its own stream to the SPA, so
+the delay reached the browser.
+
+The swap to `StreamSafeGZipMiddleware` is easy to undo by accident — the two
+have the same constructor and compress identically — so it is pinned here.
+"""
+
+from __future__ import annotations
+
+from starlette.middleware.gzip import GZipMiddleware
+
+from apis.inference_api.main import app
+from apis.shared.middleware.compression import StreamSafeGZipMiddleware
+
+
+def test_compression_is_the_stream_safe_variant() -> None:
+    installed = [m.cls for m in app.user_middleware]
+
+    assert StreamSafeGZipMiddleware in installed
+    # `is not` rather than membership: the stream-safe class *subclasses*
+    # `GZipMiddleware`, so an `in` check would pass either way.
+    assert not any(cls is GZipMiddleware for cls in installed)
+
+
+def test_compression_settings_are_unchanged_by_the_swap() -> None:
+    gzip_middleware = next(
+        m for m in app.user_middleware if m.cls is StreamSafeGZipMiddleware
+    )
+
+    assert gzip_middleware.kwargs["minimum_size"] == 1000
+    assert gzip_middleware.kwargs["compresslevel"] == 6