diff --git a/CHANGELOG.md b/CHANGELOG.md index c006df0..c5b2b85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.3.0 + +Make the request-path defaults safe. `verify()` and `verify_batch()` now use a +separate **1s timeout** (`verify_timeout=`, vs the unchanged 10s `timeout=` for +control-plane calls), and on a timeout / connection error / 5xx they **fail +closed** instead of raising: you get a synthetic `VerifyResult` with +`degraded=True` and `approved=False` (fall through to your LLM). Pass +`fail_open=True` to have those cases return `approved=True` instead. 4xx +responses (bad key, bad request, rate limit) still raise `CacheVerifierError`. +`VerifyResult` gains a `degraded` field; `model_version` is +`"verify_unavailable"` on a synthesized result. The GPTCache adapter takes the +same `verify_timeout` / `fail_open` arguments and returns `0.0` (don't reuse) on +a verifier outage by default. + ## 0.2.0 Add a `cacheverifier` console script with a `healthcheck` subcommand: diff --git a/README.md b/README.md index ecc8c19..b6aefa2 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,34 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86) |---|---| | `approved` | serve the cached answer (`True`) or fall through (`False`) | | `score` / `threshold` | `approved` is `score >= threshold` | -| `model_version` | `"stock"`, `"v"` (fine-tuned), or `"cold_start_fail_closed"` | +| `model_version` | `"stock"`, `"v"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` | | `latency_ms` | server-side inference time | +| `degraded` | `True` when the call failed and this result was synthesized client-side | + +## On your request path + +`verify()` runs inline with your traffic, so its defaults are conservative: + +- **1s timeout** (`verify_timeout=`), separate from the 10s `timeout=` used for + fine-tuning / feedback / monitoring calls. Warm verification is tens of + milliseconds server-side; 1s covers the network round trip and a cold model + load after a deploy without letting a stuck verifier stall your request. +- **Fails closed.** On a timeout, connection error, or 5xx, `verify()` does not + raise — it logs a warning on the `cacheverifier` logger and returns a + `VerifyResult` with `degraded=True` and `approved=False`, so you fall through + to your LLM exactly as you would on a cache miss. A `4xx` (bad key, bad + request, rate limit) still raises `CacheVerifierError`. + +```python +cv = CacheVerifier(api_key="cv_...", verify_timeout=1.0, fail_open=False) # the defaults + +# fail_open=True instead returns approved=True on an outage — only if a +# stale-or-near-miss answer is acceptable for that traffic: +cv = CacheVerifier(api_key="cv_...", fail_open=True) +``` + +There is no formal uptime SLA yet, which is the other reason the fallback path +is a built-in default rather than left to you. ## GPTCache diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index d1c1b15..4d34f65 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -18,5 +18,5 @@ from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult -__version__ = "0.2.0" +__version__ = "0.3.0" __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"] diff --git a/cacheverifier/client.py b/cacheverifier/client.py index 06cb14a..7617116 100644 --- a/cacheverifier/client.py +++ b/cacheverifier/client.py @@ -8,6 +8,7 @@ from __future__ import annotations +import logging from collections.abc import Iterable, Sequence from dataclasses import dataclass from typing import Any @@ -15,9 +16,30 @@ import httpx DEFAULT_BASE_URL = "https://www.cacheverifier.com" +#: Timeout for control-plane calls (feedback, fine-tuning, monitoring, usage). +#: These are not on your request path, so they get room to breathe. DEFAULT_TIMEOUT = 10.0 +#: Timeout for ``verify()`` / ``verify_batch()`` -- these ARE on your request +#: path. Warm calls are tens of milliseconds server-side; 1s leaves headroom +#: for the network round trip and a cold model load after a deploy while still +#: bounding the damage when the service is actually unreachable. +DEFAULT_VERIFY_TIMEOUT = 1.0 -__all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult"] +#: model_version on a synthetic result returned when the verify call itself +#: failed (timeout / connection error / 5xx) and the client fell back. +VERIFY_UNAVAILABLE = "verify_unavailable" + +_log = logging.getLogger(__name__) + +__all__ = [ + "DEFAULT_BASE_URL", + "DEFAULT_TIMEOUT", + "DEFAULT_VERIFY_TIMEOUT", + "VERIFY_UNAVAILABLE", + "CacheVerifier", + "CacheVerifierError", + "VerifyResult", +] class CacheVerifierError(RuntimeError): @@ -41,9 +63,15 @@ class VerifyResult: - `score`: the verifier's raw score for this pair; `approved` is `score >= threshold`. - `threshold`: the cutoff this call was decided against (tenant-specific once you've fine-tuned; 0.0 on the shared stock model). - - `model_version`: `"stock"`, `"v"` for a fine-tuned model, or - `"cold_start_fail_closed"` when no model ran (see `cold_start_mode`). + - `model_version`: `"stock"`, `"v"` for a fine-tuned model, + `"cold_start_fail_closed"` / `"cold_start_auto_pending"` when no model ran + (see `cold_start_mode`), or `"verify_unavailable"` on a synthetic + fallback result (see `degraded`). - `latency_ms`: server-side model inference time, not round-trip time. + - `degraded`: True when the verify call failed (timeout / connection error / + 5xx) and this result was synthesized by the client rather than returned by + the API. `approved` then reflects the client's `fail_open` setting + (default `False` -> `approved=False`, i.e. fall through to your LLM). """ approved: bool @@ -51,6 +79,7 @@ class VerifyResult: threshold: float model_version: str latency_ms: float + degraded: bool = False @classmethod def _from_json(cls, d: dict[str, Any]) -> VerifyResult: @@ -77,6 +106,18 @@ class CacheVerifier: Usable as a context manager (`with CacheVerifier(...) as cv:`) to close the underlying HTTP connection pool deterministically. + + Because `verify()` sits on your request path, it is treated differently + from every other call: + + - `verify_timeout` (default 1s, vs 10s for everything else) bounds how long + a slow or overloaded verifier can stall your request. + - On a timeout, connection error, or 5xx, `verify()` does not raise -- it + logs a warning and returns a synthetic `VerifyResult` with + `degraded=True`. `fail_open=False` (the default) makes that result + `approved=False` so you fall through to your LLM; `fail_open=True` makes + it `approved=True` so you serve the cached answer anyway. 4xx responses + (bad key, bad request, rate limit) still raise `CacheVerifierError`. """ def __init__( @@ -85,10 +126,14 @@ def __init__( *, base_url: str = DEFAULT_BASE_URL, timeout: float = DEFAULT_TIMEOUT, + verify_timeout: float = DEFAULT_VERIFY_TIMEOUT, + fail_open: bool = False, transport: httpx.BaseTransport | None = None, ) -> None: if not api_key: raise ValueError("api_key is required -- get one at https://www.cacheverifier.com") + self._verify_timeout = verify_timeout + self._fail_open = fail_open self._client = httpx.Client( base_url=base_url.rstrip("/"), headers={"X-API-Key": api_key, "User-Agent": _user_agent()}, @@ -110,8 +155,16 @@ def __exit__(self, *_exc: object) -> None: # -- core: verify ---------------------------------------------------- def verify(self, query: str, candidate_answer: str) -> VerifyResult: - """Approve or reject one gray-zone cache hit. `POST /v1/verify`.""" - data = self._post("/v1/verify", json={"query": query, "candidate_answer": candidate_answer}) + """Approve or reject one gray-zone cache hit. `POST /v1/verify`. + + On a timeout / connection error / 5xx this returns a synthetic + `VerifyResult` (`degraded=True`, `approved` per `fail_open`) instead of + raising -- the verifier being unreachable should degrade to a normal + cache miss, not an exception on your request path. + """ + data = self._verify_call("/v1/verify", {"query": query, "candidate_answer": candidate_answer}) + if data is None: + return self._degraded_result() return VerifyResult._from_json(data) def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]: @@ -120,11 +173,52 @@ def verify_batch(self, pairs: Sequence[tuple[str, str]]) -> list[VerifyResult]: Useful when your own retrieval returns several close candidates: send them in rank order and take the first `approved` one. + + Degrades the same way as `verify()`: on failure every pair comes back + as a `degraded` result rather than the call raising. A large batch may + need `verify_timeout` raised above the 1s default. """ items = [{"query": q, "candidate_answer": a} for q, a in pairs] - data = self._post("/v1/verify/batch", json={"items": items}) + data = self._verify_call("/v1/verify/batch", {"items": items}) + if data is None: + return [self._degraded_result() for _ in items] return [VerifyResult._from_json(r) for r in data["results"]] + # -- verify plumbing: tight timeout + fail-open/closed fallback -------- + + def _verify_call(self, path: str, payload: dict[str, Any]) -> Any | None: + """POST a verify request with the request-path timeout. Returns the + parsed body, or None to signal "unavailable, fall back" on a + timeout / connection error / 5xx. 4xx still raises. + """ + try: + return self._post(path, json=payload, timeout=self._verify_timeout) + except httpx.TransportError as exc: + self._warn_degraded(path, exc) + except CacheVerifierError as exc: + if exc.status_code < 500: + raise + self._warn_degraded(path, exc) + return None + + def _warn_degraded(self, path: str, exc: Exception) -> None: + _log.warning( + "cacheverifier: %s unavailable (%s) -- failing %s", + path, + exc, + "open" if self._fail_open else "closed", + ) + + def _degraded_result(self) -> VerifyResult: + return VerifyResult( + approved=self._fail_open, + score=0.0, + threshold=0.0, + model_version=VERIFY_UNAVAILABLE, + latency_ms=0.0, + degraded=True, + ) + # -- feedback ------------------------------------------------------ def feedback( @@ -252,8 +346,14 @@ def _post( json: Any | None = None, params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, + timeout: float | None = None, ) -> Any: - return self._unwrap(self._client.post(path, json=json, params=params or None, headers=headers)) + # httpx treats timeout=None as "no timeout" -- only pass it through + # when a caller (verify) explicitly asked for a per-request value. + extra: dict[str, Any] = {"timeout": timeout} if timeout is not None else {} + return self._unwrap( + self._client.post(path, json=json, params=params or None, headers=headers, **extra) + ) @staticmethod def _unwrap(resp: httpx.Response) -> Any: diff --git a/cacheverifier/integrations/gptcache.py b/cacheverifier/integrations/gptcache.py index 2a2b6fd..05169b3 100644 --- a/cacheverifier/integrations/gptcache.py +++ b/cacheverifier/integrations/gptcache.py @@ -22,7 +22,12 @@ from typing import Any -from cacheverifier.client import DEFAULT_BASE_URL, DEFAULT_TIMEOUT, CacheVerifier +from cacheverifier.client import ( + DEFAULT_BASE_URL, + DEFAULT_TIMEOUT, + DEFAULT_VERIFY_TIMEOUT, + CacheVerifier, +) try: from gptcache.similarity_evaluation import SimilarityEvaluation as _GPTCacheBase @@ -37,6 +42,11 @@ class CacheVerifierEvaluation(_GPTCacheBase): hosted verifier already makes a binary approve/reject call per gray-zone hit rather than a softened similarity score. Callers who want GPTCache's own threshold logic on top can wrap this rather than replace it. + + If the verify call times out or the service is unreachable, `evaluation()` + returns 0.0 (don't reuse) by default -- a verifier outage falls through to + a normal GPTCache miss. Pass `fail_open=True` to reuse the cached answer + instead in that case. """ def __init__( @@ -45,8 +55,16 @@ def __init__( *, base_url: str = DEFAULT_BASE_URL, timeout: float = DEFAULT_TIMEOUT, + verify_timeout: float = DEFAULT_VERIFY_TIMEOUT, + fail_open: bool = False, ) -> None: - self._cv = CacheVerifier(api_key=api_key, base_url=base_url, timeout=timeout) + self._cv = CacheVerifier( + api_key=api_key, + base_url=base_url, + timeout=timeout, + verify_timeout=verify_timeout, + fail_open=fail_open, + ) def evaluation(self, src_dict: dict[str, Any], cache_dict: dict[str, Any], **_kwargs: Any) -> float: query = src_dict.get("question") or src_dict.get("query", "") diff --git a/examples/quickstart.py b/examples/quickstart.py index fbf88fd..49a7de5 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -28,10 +28,15 @@ def main() -> None: query = "how do I cancel my subscription" candidate = "Go to Settings > Billing > Pause subscription for a month." + # verify() sits on your request path: it uses a 1s timeout and, if the + # service is unreachable, returns a degraded result (approved=False here) + # instead of raising. Pass fail_open=True to serve the cached answer in + # that case instead. with CacheVerifier(api_key=api_key) as cv: result = cv.verify(query, candidate) print(f"verify -> approved={result.approved} score={result.score:.3f} " - f"threshold={result.threshold} model={result.model_version}") + f"threshold={result.threshold} model={result.model_version} " + f"degraded={result.degraded}") if result.approved: answer = candidate diff --git a/pyproject.toml b/pyproject.toml index d71d4a0..e9b0ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.2.0" +version = "0.3.0" description = "Python client for the hosted CacheVerifier semantic-cache verification API" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_client.py b/tests/test_client.py index a7013b0..e547b37 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -9,8 +9,17 @@ from cacheverifier import CacheVerifier, CacheVerifierError, VerifyResult -def make_client(handler): - return CacheVerifier(api_key="cv_test", transport=httpx.MockTransport(handler)) +def make_client(handler, **kwargs): + return CacheVerifier(api_key="cv_test", transport=httpx.MockTransport(handler), **kwargs) + + +_OK_VERIFY = { + "approved": True, + "score": 3.5, + "latency_ms": 24.1, + "model_version": "stock", + "threshold": 0.0, +} def test_verify_parses_result_and_sends_api_key(): @@ -142,3 +151,88 @@ def handler(request: httpx.Request) -> httpx.Response: def test_empty_api_key_rejected(): with pytest.raises(ValueError): CacheVerifier(api_key="") + + +# -- request-path timeout + fail-open/closed -------------------------------- + + +def test_verify_uses_the_tight_verify_timeout_not_the_control_plane_one(): + seen = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["timeout"] = request.extensions.get("timeout") + return httpx.Response(200, json=_OK_VERIFY) + + with make_client(handler, timeout=10.0, verify_timeout=1.0) as cv: + cv.verify("q", "a") + + # httpx expands a scalar timeout into per-operation values + assert set(seen["timeout"].values()) == {1.0} + + +def test_verify_fails_closed_on_timeout(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.TimeoutException("read timed out", request=request) + + with make_client(handler) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is False + assert result.model_version == "verify_unavailable" + + +def test_verify_fails_open_when_configured(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused", request=request) + + with make_client(handler, fail_open=True) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is True + + +def test_verify_fails_closed_on_5xx(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, text="upstream overloaded") + + with make_client(handler) as cv: + result = cv.verify("q", "a") + + assert result.degraded is True + assert result.approved is False + + +def test_verify_still_raises_on_4xx(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(429, json={"detail": "rate limited"}) + + with make_client(handler) as cv, pytest.raises(CacheVerifierError) as excinfo: + cv.verify("q", "a") + + assert excinfo.value.status_code == 429 + + +def test_verify_batch_degrades_every_pair_on_failure(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + with make_client(handler) as cv: + results = cv.verify_batch([("q1", "a1"), ("q2", "a2"), ("q3", "a3")]) + + assert len(results) == 3 + assert all(r.degraded and not r.approved for r in results) + + +def test_control_plane_call_still_uses_the_10s_default_and_raises(): + seen = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["timeout"] = request.extensions.get("timeout") + return httpx.Response(200, json={"status": "ok"}) + + with make_client(handler, timeout=10.0, verify_timeout=1.0) as cv: + cv.usage() + + assert set(seen["timeout"].values()) == {10.0} diff --git a/tests/test_healthcheck.py b/tests/test_healthcheck.py index ebc87d7..7c41923 100644 --- a/tests/test_healthcheck.py +++ b/tests/test_healthcheck.py @@ -32,7 +32,9 @@ def test_version(self, capsys): with pytest.raises(SystemExit) as e: main(["--version"]) assert e.value.code == 0 - assert "cacheverifier 0.2.0" in capsys.readouterr().out + from cacheverifier import __version__ + + assert f"cacheverifier {__version__}" in capsys.readouterr().out def test_no_command_prints_help_and_returns_1(self, capsys): assert main([]) == 1