SDK: safe request-path defaults for verify() (1s timeout + fail-closed)

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.3.0
+
+Make the request-path defaults safe. `verify()` and `verify_batch()` now use a
+separate **1s timeout** (`verify_timeout=`, vs the unchanged 10s `timeout=` for
+control-plane calls), and on a timeout / connection error / 5xx they **fail
+closed** instead of raising: you get a synthetic `VerifyResult` with
+`degraded=True` and `approved=False` (fall through to your LLM). Pass
+`fail_open=True` to have those cases return `approved=True` instead. 4xx
+responses (bad key, bad request, rate limit) still raise `CacheVerifierError`.
+`VerifyResult` gains a `degraded` field; `model_version` is
+`"verify_unavailable"` on a synthesized result. The GPTCache adapter takes the
+same `verify_timeout` / `fail_open` arguments and returns `0.0` (don't reuse) on
+a verifier outage by default.
+
 ## 0.2.0
 
 Add a `cacheverifier` console script with a `healthcheck` subcommand:

README.md

@@ -59,8 +59,34 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
 |---|---|
 | `approved` | serve the cached answer (`True`) or fall through (`False`) |
 | `score` / `threshold` | `approved` is `score >= threshold` |
-| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), or `"cold_start_fail_closed"` |
+| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` |
 | `latency_ms` | server-side inference time |
+| `degraded` | `True` when the call failed and this result was synthesized client-side |
+
+## On your request path
+
+`verify()` runs inline with your traffic, so its defaults are conservative:
+
+- **1s timeout** (`verify_timeout=`), separate from the 10s `timeout=` used for
+  fine-tuning / feedback / monitoring calls. Warm verification is tens of
+  milliseconds server-side; 1s covers the network round trip and a cold model
+  load after a deploy without letting a stuck verifier stall your request.
+- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not
+  raise — it logs a warning on the `cacheverifier` logger and returns a
+  `VerifyResult` with `degraded=True` and `approved=False`, so you fall through
+  to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad
+  request, rate limit) still raises `CacheVerifierError`.
+
+```python
+cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False)  # the defaults
+
+# fail_open=True instead returns approved=True on an outage — only if a
+# stale-or-near-miss answer is acceptable for that traffic:
+cv = CacheVerifier(api_key="cv_...", fail_open=True)
+```
+
+There is no formal uptime SLA yet, which is the other reason the fallback path
+is a built-in default rather than left to you.
 
 ## GPTCache
 

cacheverifier/__init__.py

@@ -18,5 +18,5 @@
 
 from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"]

cacheverifier/client.py

@@ -8,16 +8,38 @@
 
 from __future__ import annotations
 
+import logging
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass
 from typing import Any
 
 import httpx
 
 DEFAULT_BASE_URL = "https://www.cacheverifier.com"
+#: Timeout for control-plane calls (feedback, fine-tuning, monitoring, usage).
+#: These are not on your request path, so they get room to breathe.
 DEFAULT_TIMEOUT = 10.0
+#: Timeout for ``verify()`` / ``verify_batch()`` -- these ARE on your request
+#: path. Warm calls are tens of milliseconds server-side; 1s leaves headroom
+#: for the network round trip and a cold model load after a deploy while still
+#: bounding the damage when the service is actually unreachable.
+DEFAULT_VERIFY_TIMEOUT = 1.0
 
-__all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult"]
+#: model_version on a synthetic result returned when the verify call itself
+#: failed (timeout / connection error / 5xx) and the client fell back.
+VERIFY_UNAVAILABLE = "verify_unavailable"
+
+_log = logging.getLogger(__name__)
+
+__all__ = [
+    "DEFAULT_BASE_URL",
+    "DEFAULT_TIMEOUT",
+    "DEFAULT_VERIFY_TIMEOUT",
+    "VERIFY_UNAVAILABLE",
+    "CacheVerifier",
+    "CacheVerifierError",
+    "VerifyResult",
+]
 
 
 class CacheVerifierError(RuntimeError):
@@ -41,16 +63,23 @@ class VerifyResult:
     - `score`: the verifier's raw score for this pair; `approved` is `score >= threshold`.
     - `threshold`: the cutoff this call was decided against (tenant-specific once
       you've fine-tuned; 0.0 on the shared stock model).
-    - `model_version`: `"stock"`, `"v<id>"` for a fine-tuned model, or
-      `"cold_start_fail_closed"` when no model ran (see `cold_start_mode`).
+    - `model_version`: `"stock"`, `"v<id>"` for a fine-tuned model,
+      `"cold_start_fail_closed"` / `"cold_start_auto_pending"` when no model ran
+      (see `cold_start_mode`), or `"verify_unavailable"` on a synthetic
+      fallback result (see `degraded`).
     - `latency_ms`: server-side model inference time, not round-trip time.
+    - `degraded`: True when the verify call failed (timeout / connection error /
+      5xx) and this result was synthesized by the client rather than returned by
+      the API. `approved` then reflects the client's `fail_open` setting
+      (default `False` -> `approved=False`, i.e. fall through to your LLM).
     """
 
     approved: bool
     score: float
     threshold: float
     model_version: str
     latency_ms: float
+    degraded: bool = False
 
     @classmethod
     def _from_json(cls, d: dict[str, Any]) -> VerifyResult:
@@ -77,6 +106,18 @@ class CacheVerifier:
 
     Usable as a context manager (`with CacheVerifier(...) as cv:`) to close
     the underlying HTTP connection pool deterministically.
+
+    Because `verify()` sits on your request path, it is treated differently
+    from every other call:
+
+    - `verify_timeout` (default 1s, vs 10s for everything else) bounds how long
+      a slow or overloaded verifier can stall your request.
+    - On a timeout, connection error, or 5xx, `verify()` does not raise -- it
+      logs a warning and returns a synthetic `VerifyResult` with
+      `degraded=True`. `fail_open=False` (the default) makes that result
+      `approved=False` so you fall through to your LLM; `fail_open=True` makes
+      it `approved=True` so you serve the cached answer anyway. 4xx responses
+      (bad key, bad request, rate limit) still raise `CacheVerifierError`.
     """
 
     def __init__(
@@ -85,10 +126,14 @@ def __init__(
         *,
         base_url: str = DEFAULT_BASE_URL,
         timeout: float = DEFAULT_TIMEOUT,
+        verify_timeout: float = DEFAULT_VERIFY_TIMEOUT,
+        fail_open: bool = False,
         transport: httpx.BaseTransport | None = None,
     ) -> None:
         if not api_key:
             raise ValueError("api_key is required -- get one at https://www.cacheverifier.com")
+        self._verify_timeout = verify_timeout
+        self._fail_open = fail_open
         self._client = httpx.Client(
             base_url=base_url.rstrip("/"),
             headers={"X-API-Key": api_key, "User-Agent": _user_agent()},
@@ -110,8 +155,16 @@ def __exit__(self, *_exc: object) -> None:
     # -- core: verify ----------------------------------------------------
 
     def verify(self, query: str, candidate_answer: str) -> VerifyResult:
-        """Approve or reject one gray-zone cache hit. `POST /v1/verify`."""
-        data = self._post("/v1/verify", json={"query": query, "candidate_answer": candidate_answer})
+        """Approve or reject one gray-zone cache hit. `POST /v1/verify`.
+
+        On a timeout / connection error / 5xx this returns a synthetic
+        `VerifyResult` (`degraded=True`, `approved` per `fail_open`) instead of
+        raising -- the verifier being unreachable should degrade to a normal
+        cache miss, not an exception on your request path.
+        """
+        data = self._verify_call("/v1/verify", {"query": query, "candidate_answer": candidate_answer})
+        if data is None:
+            return self._degraded_result()
         return VerifyResult._from_json(data)
 
     def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]:
@@ -120,11 +173,52 @@ def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]:
 
         Useful when your own retrieval returns several close candidates:
         send them in rank order and take the first `approved` one.
+
+        Degrades the same way as `verify()`: on failure every pair comes back
+        as a `degraded` result rather than the call raising. A large batch may
+        need `verify_timeout` raised above the 1s default.
         """
         items = [{"query": q, "candidate_answer": a} for q, a in pairs]
-        data = self._post("/v1/verify/batch", json={"items": items})
+        data = self._verify_call("/v1/verify/batch", {"items": items})
+        if data is None:
+            return [self._degraded_result() for _ in items]
         return [VerifyResult._from_json(r) for r in data["results"]]
 
+    # -- verify plumbing: tight timeout + fail-open/closed fallback --------
+
+    def _verify_call(self, path: str, payload: dict[str, Any]) -> Any | None:
+        """POST a verify request with the request-path timeout. Returns the
+        parsed body, or None to signal "unavailable, fall back" on a
+        timeout / connection error / 5xx. 4xx still raises.
+        """
+        try:
+            return self._post(path, json=payload, timeout=self._verify_timeout)
+        except httpx.TransportError as exc:
+            self._warn_degraded(path, exc)
+        except CacheVerifierError as exc:
+            if exc.status_code < 500:
+                raise
+            self._warn_degraded(path, exc)
+        return None
+
+    def _warn_degraded(self, path: str, exc: Exception) -> None:
+        _log.warning(
+            "cacheverifier: %s unavailable (%s) -- failing %s",
+            path,
+            exc,
+            "open" if self._fail_open else "closed",
+        )
+
+    def _degraded_result(self) -> VerifyResult:
+        return VerifyResult(
+            approved=self._fail_open,
+            score=0.0,
+            threshold=0.0,
+            model_version=VERIFY_UNAVAILABLE,
+            latency_ms=0.0,
+            degraded=True,
+        )
+
     # -- feedback ------------------------------------------------------
 
     def feedback(
@@ -252,8 +346,14 @@ def _post(
         json: Any | None = None,
         params: dict[str, Any] | None = None,
         headers: dict[str, str] | None = None,
+        timeout: float | None = None,
     ) -> Any:
-        return self._unwrap(self._client.post(path, json=json, params=params or None, headers=headers))
+        # httpx treats timeout=None as "no timeout" -- only pass it through
+        # when a caller (verify) explicitly asked for a per-request value.
+        extra: dict[str, Any] = {"timeout": timeout} if timeout is not None else {}
+        return self._unwrap(
+            self._client.post(path, json=json, params=params or None, headers=headers, **extra)
+        )
 
     @staticmethod
     def _unwrap(resp: httpx.Response) -> Any:

cacheverifier/integrations/gptcache.py

@@ -22,7 +22,12 @@
 
 from typing import Any
 
-from cacheverifier.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT, CacheVerifier
+from cacheverifier.client import (
+    DEFAULT_BASE_URL,
+    DEFAULT_TIMEOUT,
+    DEFAULT_VERIFY_TIMEOUT,
+    CacheVerifier,
+)
 
 try:
     from gptcache.similarity_evaluation import SimilarityEvaluation as _GPTCacheBase
@@ -37,6 +42,11 @@ class CacheVerifierEvaluation(_GPTCacheBase):
     hosted verifier already makes a binary approve/reject call per gray-zone
     hit rather than a softened similarity score. Callers who want GPTCache's
     own threshold logic on top can wrap this rather than replace it.
+
+    If the verify call times out or the service is unreachable, `evaluation()`
+    returns 0.0 (don't reuse) by default -- a verifier outage falls through to
+    a normal GPTCache miss. Pass `fail_open=True` to reuse the cached answer
+    instead in that case.
     """
 
     def __init__(
@@ -45,8 +55,16 @@ def __init__(
         *,
         base_url: str = DEFAULT_BASE_URL,
         timeout: float = DEFAULT_TIMEOUT,
+        verify_timeout: float = DEFAULT_VERIFY_TIMEOUT,
+        fail_open: bool = False,
     ) -> None:
-        self._cv = CacheVerifier(api_key=api_key, base_url=base_url, timeout=timeout)
+        self._cv = CacheVerifier(
+            api_key=api_key,
+            base_url=base_url,
+            timeout=timeout,
+            verify_timeout=verify_timeout,
+            fail_open=fail_open,
+        )
 
     def evaluation(self, src_dict: dict[str, Any], cache_dict: dict[str, Any], **_kwargs: Any) -> float:
         query = src_dict.get("question") or src_dict.get("query", "")

examples/quickstart.py

@@ -28,10 +28,15 @@ def main() -> None:
     query = "how do I cancel my subscription"
     candidate = "Go to Settings > Billing > Pause subscription for a month."
 
+    # verify() sits on your request path: it uses a 1s timeout and, if the
+    # service is unreachable, returns a degraded result (approved=False here)
+    # instead of raising. Pass fail_open=True to serve the cached answer in
+    # that case instead.
     with CacheVerifier(api_key=api_key) as cv:
         result = cv.verify(query, candidate)
         print(f"verify -> approved={result.approved} score={result.score:.3f} "
-              f"threshold={result.threshold} model={result.model_version}")
+              f"threshold={result.threshold} model={result.model_version} "
+              f"degraded={result.degraded}")
 
         if result.approved:
             answer = candidate

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cacheverifier"
-version = "0.2.0"
+version = "0.3.0"
 description = "Python client for the hosted CacheVerifier semantic-cache verification API"
 readme = "README.md"
 requires-python = ">=3.9"