From d5619a32cb979a32b02c5d027f513a7ca141b87b Mon Sep 17 00:00:00 2001 From: xin Date: Fri, 4 Sep 2026 13:02:05 +0800 Subject: [PATCH] Make verify() request-path defaults safe: 1s timeout + fail-closed verify()/verify_batch() sit inline with the caller's traffic, but the client applied the same 10s timeout as control-plane calls and raised on any failure -- so a slow or down verifier stalled requests for up to 10s and turned an outage into an exception on the hot path. The docs already told users to fix this by hand (timeout=1.0, catch-and-fall-through); the SDK default now matches that advice. - verify_timeout (default 1.0) separate from timeout (still 10.0 for feedback/finetune/monitoring). Passed per-request so control-plane calls are unaffected. - On timeout / connection error / 5xx, verify() logs a warning and returns a synthetic VerifyResult(degraded=True, approved=) instead of raising. fail_open defaults to False (fall through to the LLM, same as a cache miss). 4xx still raises CacheVerifierError. - VerifyResult gains `degraded`; model_version is "verify_unavailable" on a synthesized result. - GPTCache adapter takes verify_timeout / fail_open and returns 0.0 on an outage by default. v0.2.0 -> v0.3.0. Co-authored-by: xin --- CHANGELOG.md | 14 +++ README.md | 28 +++++- cacheverifier/__init__.py | 2 +- cacheverifier/client.py | 114 +++++++++++++++++++++++-- cacheverifier/integrations/gptcache.py | 22 ++++- examples/quickstart.py | 7 +- pyproject.toml | 2 +- tests/test_client.py | 98 ++++++++++++++++++++- tests/test_healthcheck.py | 4 +- 9 files changed, 275 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c006df0..c5b2b85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.3.0 + +Make the request-path defaults safe. `verify()` and `verify_batch()` now use a +separate **1s timeout** (`verify_timeout=`, vs the unchanged 10s `timeout=` for +control-plane calls), and on a timeout / connection error / 5xx they **fail +closed** instead of raising: you get a synthetic `VerifyResult` with +`degraded=True` and `approved=False` (fall through to your LLM). Pass +`fail_open=True` to have those cases return `approved=True` instead. 4xx +responses (bad key, bad request, rate limit) still raise `CacheVerifierError`. +`VerifyResult` gains a `degraded` field; `model_version` is +`"verify_unavailable"` on a synthesized result. The GPTCache adapter takes the +same `verify_timeout` / `fail_open` arguments and returns `0.0` (don't reuse) on +a verifier outage by default. + ## 0.2.0 Add a `cacheverifier` console script with a `healthcheck` subcommand: diff --git a/README.md b/README.md index ecc8c19..b6aefa2 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,34 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86) |---|---| | `approved` | serve the cached answer (`True`) or fall through (`False`) | | `score` / `threshold` | `approved` is `score >= threshold` | -| `model_version` | `"stock"`, `"v"` (fine-tuned), or `"cold_start_fail_closed"` | +| `model_version` | `"stock"`, `"v"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` | | `latency_ms` | server-side inference time | +| `degraded` | `True` when the call failed and this result was synthesized client-side | + +## On your request path + +`verify()` runs inline with your traffic, so its defaults are conservative: + +- **1s timeout** (`verify_timeout=`), separate from the 10s `timeout=` used for + fine-tuning / feedback / monitoring calls. Warm verification is tens of + milliseconds server-side; 1s covers the network round trip and a cold model + load after a deploy without letting a stuck verifier stall your request. +- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not + raise — it logs a warning on the `cacheverifier` logger and returns a + `VerifyResult` with `degraded=True` and `approved=False`, so you fall through + to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad + request, rate limit) still raises `CacheVerifierError`. + +```python +cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False) # the defaults + +# fail_open=True instead returns approved=True on an outage — only if a +# stale-or-near-miss answer is acceptable for that traffic: +cv = CacheVerifier(api_key="cv_...", fail_open=True) +``` + +There is no formal uptime SLA yet, which is the other reason the fallback path +is a built-in default rather than left to you. ## GPTCache diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index d1c1b15..4d34f65 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -18,5 +18,5 @@ from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult -__version__ = "0.2.0" +__version__ = "0.3.0" __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"] diff --git a/cacheverifier/client.py b/cacheverifier/client.py index 06cb14a..7617116 100644 --- a/cacheverifier/client.py +++ b/cacheverifier/client.py @@ -8,6 +8,7 @@ from __future__ import annotations +import logging from collections.abc import Iterable, Sequence from dataclasses import dataclass from typing import Any @@ -15,9 +16,30 @@ import httpx DEFAULT_BASE_URL = "https://www.cacheverifier.com" +#: Timeout for control-plane calls (feedback, fine-tuning, monitoring, usage). +#: These are not on your request path, so they get room to breathe. DEFAULT_TIMEOUT = 10.0 +#: Timeout for ``verify()`` / ``verify_batch()`` -- these ARE on your request +#: path. Warm calls are tens of milliseconds server-side; 1s leaves headroom +#: for the network round trip and a cold model load after a deploy while still +#: bounding the damage when the service is actually unreachable. +DEFAULT_VERIFY_TIMEOUT = 1.0 -__all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult"] +#: model_version on a synthetic result returned when the verify call itself +#: failed (timeout / connection error / 5xx) and the client fell back. +VERIFY_UNAVAILABLE = "verify_unavailable" + +_log = logging.getLogger(__name__) + +__all__ = [ + "DEFAULT_BASE_URL", + "DEFAULT_TIMEOUT", + "DEFAULT_VERIFY_TIMEOUT", + "VERIFY_UNAVAILABLE", + "CacheVerifier", + "CacheVerifierError", + "VerifyResult", +] class CacheVerifierError(RuntimeError): @@ -41,9 +63,15 @@ class VerifyResult: - `score`: the verifier's raw score for this pair; `approved` is `score >= threshold`. - `threshold`: the cutoff this call was decided against (tenant-specific once you've fine-tuned; 0.0 on the shared stock model). - - `model_version`: `"stock"`, `"v"` for a fine-tuned model, or - `"cold_start_fail_closed"` when no model ran (see `cold_start_mode`). + - `model_version`: `"stock"`, `"v"` for a fine-tuned model, + `"cold_start_fail_closed"` / `"cold_start_auto_pending"` when no model ran + (see `cold_start_mode`), or `"verify_unavailable"` on a synthetic + fallback result (see `degraded`). - `latency_ms`: server-side model inference time, not round-trip time. + - `degraded`: True when the verify call failed (timeout / connection error / + 5xx) and this result was synthesized by the client rather than returned by + the API. `approved` then reflects the client's `fail_open` setting + (default `False` -> `approved=False`, i.e. fall through to your LLM). """ approved: bool @@ -51,6 +79,7 @@ class VerifyResult: threshold: float model_version: str latency_ms: float + degraded: bool = False @classmethod def _from_json(cls, d: dict[str, Any]) -> VerifyResult: @@ -77,6 +106,18 @@ class CacheVerifier: Usable as a context manager (`with CacheVerifier(...) as cv:`) to close the underlying HTTP connection pool deterministically. + + Because `verify()` sits on your request path, it is treated differently + from every other call: + + - `verify_timeout` (default 1s, vs 10s for everything else) bounds how long + a slow or overloaded verifier can stall your request. + - On a timeout, connection error, or 5xx, `verify()` does not raise -- it + logs a warning and returns a synthetic `VerifyResult` with + `degraded=True`. `fail_open=False` (the default) makes that result + `approved=False` so you fall through to your LLM; `fail_open=True` makes + it `approved=True` so you serve the cached answer anyway. 4xx responses + (bad key, bad request, rate limit) still raise `CacheVerifierError`. """ def __init__( @@ -85,10 +126,14 @@ def __init__( *, base_url: str = DEFAULT_BASE_URL, timeout: float = DEFAULT_TIMEOUT, + verify_timeout: float = DEFAULT_VERIFY_TIMEOUT, + fail_open: bool = False, transport: httpx.BaseTransport | None = None, ) -> None: if not api_key: raise ValueError("api_key is required -- get one at https://www.cacheverifier.com") + self._verify_timeout = verify_timeout + self._fail_open = fail_open self._client = httpx.Client( base_url=base_url.rstrip("/"), headers={"X-API-Key": api_key, "User-Agent": _user_agent()}, @@ -110,8 +155,16 @@ def __exit__(self, *_exc: object) -> None: # -- core: verify ---------------------------------------------------- def verify(self, query: str, candidate_answer: str) -> VerifyResult: - """Approve or reject one gray-zone cache hit. `POST /v1/verify`.""" - data = self._post("/v1/verify", json={"query": query, "candidate_answer": candidate_answer}) + """Approve or reject one gray-zone cache hit. `POST /v1/verify`. + + On a timeout / connection error / 5xx this returns a synthetic + `VerifyResult` (`degraded=True`, `approved` per `fail_open`) instead of + raising -- the verifier being unreachable should degrade to a normal + cache miss, not an exception on your request path. + """ + data = self._verify_call("/v1/verify", {"query": query, "candidate_answer": candidate_answer}) + if data is None: + return self._degraded_result() return VerifyResult._from_json(data) def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]: @@ -120,11 +173,52 @@ def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]: Useful when your own retrieval returns several close candidates: send them in rank order and take the first `approved` one. + + Degrades the same way as `verify()`: on failure every pair comes back + as a `degraded` result rather than the call raising. A large batch may + need `verify_timeout` raised above the 1s default. """ items = [{"query": q, "candidate_answer": a} for q, a in pairs] - data = self._post("/v1/verify/batch", json={"items": items}) + data = self._verify_call("/v1/verify/batch", {"items": items}) + if data is None: + return [self._degraded_result() for _ in items] return [VerifyResult._from_json(r) for r in data["results"]] + # -- verify plumbing: tight timeout + fail-open/closed fallback -------- + + def _verify_call(self, path: str, payload: dict[str, Any]) -> Any | None: + """POST a verify request with the request-path timeout. Returns the + parsed body, or None to signal "unavailable, fall back" on a + timeout / connection error / 5xx. 4xx still raises. + """ + try: + return self._post(path, json=payload, timeout=self._verify_timeout) + except httpx.TransportError as exc: + self._warn_degraded(path, exc) + except CacheVerifierError as exc: + if exc.status_code < 500: + raise + self._warn_degraded(path, exc) + return None + + def _warn_degraded(self, path: str, exc: Exception) -> None: + _log.warning( + "cacheverifier: %s unavailable (%s) -- failing %s", + path, + exc, + "open" if self._fail_open else "closed", + ) + + def _degraded_result(self) -> VerifyResult: + return VerifyResult( + approved=self._fail_open, + score=0.0, + threshold=0.0, + model_version=VERIFY_UNAVAILABLE, + latency_ms=0.0, + degraded=True, + ) + # -- feedback ------------------------------------------------------ def feedback( @@ -252,8 +346,14 @@ def _post( json: Any | None = None, params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, + timeout: float | None = None, ) -> Any: - return self._unwrap(self._client.post(path, json=json, params=params or None, headers=headers)) + # httpx treats timeout=None as "no timeout" -- only pass it through + # when a caller (verify) explicitly asked for a per-request value. + extra: dict[str, Any] = {"timeout": timeout} if timeout is not None else {} + return self._unwrap( + self._client.post(path, json=json, params=params or None, headers=headers, **extra) + ) @staticmethod def _unwrap(resp: httpx.Response) -> Any: diff --git a/cacheverifier/integrations/gptcache.py b/cacheverifier/integrations/gptcache.py index 2a2b6fd..05169b3 100644 --- a/cacheverifier/integrations/gptcache.py +++ b/cacheverifier/integrations/gptcache.py @@ -22,7 +22,12 @@ from typing import Any -from cacheverifier.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT, CacheVerifier +from cacheverifier.client import ( + DEFAULT_BASE_URL, + DEFAULT_TIMEOUT, + DEFAULT_VERIFY_TIMEOUT, + CacheVerifier, +) try: from gptcache.similarity_evaluation import SimilarityEvaluation as _GPTCacheBase @@ -37,6 +42,11 @@ class CacheVerifierEvaluation(_GPTCacheBase): hosted verifier already makes a binary approve/reject call per gray-zone hit rather than a softened similarity score. Callers who want GPTCache's own threshold logic on top can wrap this rather than replace it. + + If the verify call times out or the service is unreachable, `evaluation()` + returns 0.0 (don't reuse) by default -- a verifier outage falls through to + a normal GPTCache miss. Pass `fail_open=True` to reuse the cached answer + instead in that case. """ def __init__( @@ -45,8 +55,16 @@ def __init__( *, base_url: str = DEFAULT_BASE_URL, timeout: float = DEFAULT_TIMEOUT, + verify_timeout: float = DEFAULT_VERIFY_TIMEOUT, + fail_open: bool = False, ) -> None: - self._cv = CacheVerifier(api_key=api_key, base_url=base_url, timeout=timeout) + self._cv = CacheVerifier( + api_key=api_key, + base_url=base_url, + timeout=timeout, + verify_timeout=verify_timeout, + fail_open=fail_open, + ) def evaluation(self, src_dict: dict[str, Any], cache_dict: dict[str, Any], **_kwargs: Any) -> float: query = src_dict.get("question") or src_dict.get("query", "") diff --git a/examples/quickstart.py b/examples/quickstart.py index fbf88fd..49a7de5 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -28,10 +28,15 @@ def main() -> None: query = "how do I cancel my subscription" candidate = "Go to Settings > Billing > Pause subscription for a month." + # verify() sits on your request path: it uses a 1s timeout and, if the + # service is unreachable, returns a degraded result (approved=False here) + # instead of raising. Pass fail_open=True to serve the cached answer in + # that case instead. with CacheVerifier(api_key=api_key) as cv: result = cv.verify(query, candidate) print(f"verify -> approved={result.approved} score={result.score:.3f} " - f"threshold={result.threshold} model={result.model_version}") + f"threshold={result.threshold} model={result.model_version} " + f"degraded={result.degraded}") if result.approved: answer = candidate diff --git a/pyproject.toml b/pyproject.toml index d71d4a0..e9b0ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.2.0" +version = "0.3.0" description = "Python client for the hosted CacheVerifier semantic-cache verification API" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_client.py b/tests/test_client.py index a7013b0..e547b37 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -9,8 +9,17 @@ from cacheverifier import CacheVerifier, CacheVerifierError, VerifyResult -def make_client(handler): - return CacheVerifier(api_key="cv_test", transport=httpx.MockTransport(handler)) +def make_client(handler, **kwargs): + return CacheVerifier(api_key="cv_test", transport=httpx.MockTransport(handler), **kwargs) + + +_OK_VERIFY = { + "approved": True, + "score": 3.5, + "latency_ms": 24.1, + "model_version": "stock", + "threshold": 0.0, +} def test_verify_parses_result_and_sends_api_key(): @@ -142,3 +151,88 @@ def handler(request: httpx.Request) -> httpx.Response: def test_empty_api_key_rejected(): with pytest.raises(ValueError): CacheVerifier(api_key="") + + +# -- request-path timeout + fail-open/closed -------------------------------- + + +def test_verify_uses_the_tight_verify_timeout_not_the_control_plane_one(): + seen = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["timeout"] = request.extensions.get("timeout") + return httpx.Response(200, json=_OK_VERIFY) + + with make_client(handler, timeout=10.0, verify_timeout=1.0) as cv: + cv.verify("q", "a") + + # httpx expands a scalar timeout into per-operation values + assert set(seen["timeout"].values()) == {1.0} + + +def test_verify_fails_closed_on_timeout(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.TimeoutException("read timed out", request=request) + + with make_client(handler) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is False + assert result.model_version == "verify_unavailable" + + +def test_verify_fails_open_when_configured(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused", request=request) + + with make_client(handler, fail_open=True) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is True + + +def test_verify_fails_closed_on_5xx(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, text="upstream overloaded") + + with make_client(handler) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is False + + +def test_verify_still_raises_on_4xx(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(429, json={"detail": "rate limited"}) + + with make_client(handler) as cv, pytest.raises(CacheVerifierError) as excinfo: + cv.verify("q", "a") + + assert excinfo.value.status_code == 429 + + +def test_verify_batch_degrades_every_pair_on_failure(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + with make_client(handler) as cv: + results = cv.verify_batch([("q1", "a1"), ("q2", "a2"), ("q3", "a3")]) + + assert len(results) == 3 + assert all(r.degraded and not r.approved for r in results) + + +def test_control_plane_call_still_uses_the_10s_default_and_raises(): + seen = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["timeout"] = request.extensions.get("timeout") + return httpx.Response(200, json={"status": "ok"}) + + with make_client(handler, timeout=10.0, verify_timeout=1.0) as cv: + cv.usage() + + assert set(seen["timeout"].values()) == {10.0} diff --git a/tests/test_healthcheck.py b/tests/test_healthcheck.py index ebc87d7..7c41923 100644 --- a/tests/test_healthcheck.py +++ b/tests/test_healthcheck.py @@ -32,7 +32,9 @@ def test_version(self, capsys): with pytest.raises(SystemExit) as e: main(["--version"]) assert e.value.code == 0 - assert "cacheverifier 0.2.0" in capsys.readouterr().out + from cacheverifier import __version__ + + assert f"cacheverifier {__version__}" in capsys.readouterr().out def test_no_command_prints_help_and_returns_1(self, capsys): assert main([]) == 1