diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a22492..6187130 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,27 @@ jobs: - run: mypy cacheverifier if: matrix.python-version != '3.9' # mypy config targets 3.10+ + # `cacheverifier healthcheck` -- the base `test` job above installs only + # `.[dev]`, so it exercises the CLI dispatch and the missing-extra path; + # this job installs `.[healthcheck,dev]` and runs the real fine-tune + # end-to-end. One Python version is enough (the training path isn't + # version-sensitive) and CPU-only torch keeps the install lean -- the + # default wheel drags in ~3 GB of unused CUDA libraries whose import-time + # mmap alone can OOM a small runner. + healthcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: python -m pip install --upgrade pip + - run: pip install torch --index-url https://download.pytorch.org/whl/cpu + - run: pip install -e ".[healthcheck,dev]" + - run: ruff check . + - run: mypy cacheverifier + - run: pytest -q tests/test_healthcheck.py + build: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3113b02..c006df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.2.0 + +Add a `cacheverifier` console script with a `healthcheck` subcommand: +`cacheverifier healthcheck traffic.jsonl` runs the hosted Health Check +Report's stock-vs-fine-tuned held-out AUC evaluation (`POST /v1/finetune/dry-run`) +entirely offline — no queries or answers leave the machine. Needs the new +`healthcheck` extra (`pip install "cacheverifier[healthcheck]"`: torch, +sentence-transformers, numpy); the base client stays `httpx`-only. +`--emit-summary` writes an aggregate-only JSON file with no text. + ## 0.1.0 Initial release: thin `httpx`-based client for `/v1/verify`, `/v1/verify/batch`, diff --git a/README.md b/README.md index 394754f..ecc8c19 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,12 @@ zone", where a plain threshold match might be wrong. pip install cacheverifier # with the GPTCache adapter: pip install "cacheverifier[gptcache]" +# with the offline Health Check (adds torch + sentence-transformers): +pip install "cacheverifier[healthcheck]" ``` -Requires Python 3.9+. The only runtime dependency is `httpx`. +Requires Python 3.9+. The only runtime dependency is `httpx` — the extras above +are opt-in. ## Quickstart @@ -95,6 +98,41 @@ if job.get("result_model_version"): `cv.dry_run([...])` reports the same baseline-vs-tuned AUC on examples you pass directly, without writing anything or deploying a model. +## Local Health Check (offline) + +`cv.dry_run()` still uploads your examples to the API. If that's a blocker — a +compliance review, or just not wanting production traffic to leave your network — +run the identical stock-vs-fine-tuned evaluation entirely on your own machine: + +```bash +pip install "cacheverifier[healthcheck]" + +cacheverifier healthcheck traffic.jsonl +cacheverifier healthcheck traffic.jsonl --emit-summary summary.json +``` + +`traffic.jsonl` is a JSON array or JSONL of `{"query", "candidate_answer", "was_correct"}` +rows **in arrival order** (the train/calibrate/test split is chronological, matching the +hosted service so the numbers are comparable). Optional per row: `"stale": true`. + +Nothing is sent anywhere — the base model downloads once from Hugging Face, then it's +fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts, rates — +no query or answer text) that's safe to share for a human read. + +``` +results +------------------------------------------------------------------ + train / calibrate / test: 3349 / 419 / 419 + stock verifier held-out AUC: 0.6120 + fine-tuned held-out AUC: 0.7080 (delta +0.0960) + label-noise proxy (disagreement): 11.4% + ceiling status: still_improvable + +verdict +------------------------------------------------------------------ + IMPROVED -- fine-tuning on your own data helps this traffic +``` + ## API surface | method | endpoint | diff --git a/cacheverifier/__init__.py b/cacheverifier/__init__.py index afb72d6..d1c1b15 100644 --- a/cacheverifier/__init__.py +++ b/cacheverifier/__init__.py @@ -18,5 +18,5 @@ from cacheverifier.client import CacheVerifier, CacheVerifierError, VerifyResult -__version__ = "0.1.0" +__version__ = "0.2.0" __all__ = ["CacheVerifier", "CacheVerifierError", "VerifyResult", "__version__"] diff --git a/cacheverifier/__main__.py b/cacheverifier/__main__.py new file mode 100644 index 0000000..7c9a7ca --- /dev/null +++ b/cacheverifier/__main__.py @@ -0,0 +1,44 @@ +"""`python -m cacheverifier` / the `cacheverifier` console script. + +A thin argparse dispatcher. The base install (httpx only) provides +`--version` and `--help`; the `healthcheck` subcommand additionally needs +the `healthcheck` extra and imports nothing heavy until it runs. +""" + +from __future__ import annotations + +import argparse +import sys + +from cacheverifier import __version__ + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="cacheverifier", + description="CacheVerifier -- hosted semantic-cache verification (https://www.cacheverifier.com).", + ) + parser.add_argument("--version", action="version", version=f"cacheverifier {__version__}") + subparsers = parser.add_subparsers(dest="command", metavar="") + + # Import lazily and defensively: a missing healthcheck extra must not + # break `cacheverifier --help` or `--version`. cli.add_subparser itself + # only touches argparse. + from cacheverifier._healthcheck import cli as healthcheck_cli + + healthcheck_cli.add_subparser(subparsers) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + if not getattr(args, "command", None): + parser.print_help() + return 1 + return int(args.func(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cacheverifier/_healthcheck/__init__.py b/cacheverifier/_healthcheck/__init__.py new file mode 100644 index 0000000..69fdbc4 --- /dev/null +++ b/cacheverifier/_healthcheck/__init__.py @@ -0,0 +1,7 @@ +"""`cacheverifier healthcheck` -- run the hosted Health Check Report's +stock-vs-fine-tuned AUC evaluation entirely on your own machine. + +Everything under here needs the `healthcheck` extra +(`pip install "cacheverifier[healthcheck]"`: torch, sentence-transformers, +scikit-learn, numpy) and is imported only when the subcommand runs. +""" diff --git a/cacheverifier/_healthcheck/_finetune.py b/cacheverifier/_healthcheck/_finetune.py new file mode 100644 index 0000000..0655a9e --- /dev/null +++ b/cacheverifier/_healthcheck/_finetune.py @@ -0,0 +1,274 @@ +"""Local fine-tune + held-out AUC evaluation for `cacheverifier healthcheck`. + +This is a trimmed, self-contained port of the hosted service's +`verifier_core.finetune` / `verifier_core.cross_encoder` (the code behind +`POST /v1/finetune/dry-run`). It keeps the parts a Health Check needs -- +the chronological train/calibrate/test split, held-out AUC for the stock +vs. fine-tuned model, a Youden's-J operating point, and the label-noise +and ceiling diagnostics -- and drops everything specific to running a +production tenant (Conformal Risk Control certification, cost-ratio +threshold grids, drift monitoring, operating-point-jump detection). + +Nothing here is imported unless the `healthcheck` extra is installed and +the subcommand is actually run -- see `cacheverifier.__main__`. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any, cast + +import numpy as np +import torch +from sentence_transformers import CrossEncoder, InputExample +from torch.utils.data import DataLoader + +# All four imports above are from the `healthcheck` extra. This module is +# only ever imported from inside `cacheverifier._healthcheck.cli.run()`, +# which catches the ImportError and points the user at +# `pip install "cacheverifier[healthcheck]"` -- so a plain +# `import cacheverifier` / `cacheverifier --help` never pays this cost. + +DEFAULT_BASE_MODEL = "cross-encoder/ms-marco-MiniLM-L6-v2" +"""The stock off-the-shelf verifier every tenant starts on -- the same base +model the hosted service fine-tunes from.""" + +MAX_SEQUENCE_LENGTH = 128 +"""Token cap applied identically at train and score time. Matches the +hosted service; BERT-style attention memory is O(n^2) in sequence length, +and this keeps a CPU run well-behaved.""" + +_CONTENT_TOKEN_BUDGET = MAX_SEQUENCE_LENGTH - 3 # [CLS] + 2x [SEP] +_MIN_QUERY_TOKENS = 32 + +TRAIN_BATCH_SIZE = 4 +MIN_TRAIN_EXAMPLES = 20 +"""Hard floor -- below this there aren't enough rows for one meaningful +epoch plus a held-out split.""" + +COLD_START_WARNING_THRESHOLD = 1000 +"""The hosted service's research (Research page, Q2) found fine-tuning only +reliably positive across every dataset above roughly this many rows. Below +it a result is real but noisy.""" + +_MIN_TEST_FOR_THRESHOLD = 10 +_MIN_TEST_FOR_CEILING_DIAGNOSIS = 50 +_CEILING_TAIL_MARGIN = 0.15 +_CEILING_AUC_CAP = 0.75 + +NOISE_WARNING_DISAGREEMENT_RATE = 0.20 +"""`train_label_disagreement_rate` above this: worth auditing how the +correctness signal is produced.""" +NOISE_HOLD_BACK_DISAGREEMENT_RATE = 0.30 +"""Above this the hosted service holds a new model back for review.""" + + +@dataclass(frozen=True) +class GrayZoneExample: + query: str + candidate_answer: str + was_correct: bool + + +@dataclass(frozen=True) +class HealthCheckResult: + n_train: int + n_calibrate: int + n_test: int + train_positive_rate: float + test_positive_rate: float + auc_baseline: float + auc_tuned: float + train_time_seconds: float + threshold: float | None + threshold_hit_rate: float | None + threshold_error_rate: float | None + train_label_disagreement_rate: float | None + ceiling_status: str | None + model_path: str + + @property + def auc_delta(self) -> float: + return self.auc_tuned - self.auc_baseline + + +def _head_tail_truncate(ids: list[int], budget: int, head_ratio: float = 0.6) -> list[int]: + if len(ids) <= budget: + return ids + head_n = int(budget * head_ratio) + tail_n = budget - head_n + return ids[:head_n] + (ids[-tail_n:] if tail_n > 0 else []) + + +def smart_truncate_pair(query: str, candidate_answer: str, tokenizer: Any) -> tuple[str, str]: + """Tokenizer-aware head+tail truncation, applied identically at train + and score time (a model trained on one truncation policy and scored + with another sees a different token window than it was optimized + against). A no-op whenever the pair already fits.""" + query_ids = tokenizer.encode(query, add_special_tokens=False) + answer_ids = tokenizer.encode(candidate_answer, add_special_tokens=False) + if len(query_ids) + len(answer_ids) <= _CONTENT_TOKEN_BUDGET: + return query, candidate_answer + + query_budget = min(len(query_ids), max(_MIN_QUERY_TOKENS, _CONTENT_TOKEN_BUDGET // 2)) + answer_budget = _CONTENT_TOKEN_BUDGET - query_budget + query_ids = _head_tail_truncate(query_ids, query_budget) + answer_ids = _head_tail_truncate(answer_ids, answer_budget) + return ( + tokenizer.decode(query_ids, skip_special_tokens=True), + tokenizer.decode(answer_ids, skip_special_tokens=True), + ) + + +def roc_auc(scores: np.ndarray, labels: np.ndarray) -> float: + """Rank-based AUC, no sklearn dependency. NaN if either class is absent.""" + order = np.argsort(scores) + ranks = np.empty_like(order, dtype=float) + ranks[order] = np.arange(len(scores)) + pos_ranks = ranks[labels == 1] + n_pos, n_neg = int((labels == 1).sum()), int((labels == 0).sum()) + if n_pos == 0 or n_neg == 0: + return float("nan") + return float((pos_ranks.sum() - n_pos * (n_pos - 1) / 2) / (n_pos * n_neg)) + + +def select_threshold(scores: np.ndarray, labels: np.ndarray) -> float | None: + """Youden's J: the threshold maximizing (TPR - FPR). None when the + held-out split is too small or single-class to calibrate from -- the + caller decides the fallback.""" + n_pos = int((labels == 1).sum()) + n_neg = int((labels == 0).sum()) + if n_pos == 0 or n_neg == 0 or (n_pos + n_neg) < _MIN_TEST_FOR_THRESHOLD: + return None + + order = np.argsort(-scores) + sorted_scores = scores[order] + sorted_labels = labels[order] + tp = np.cumsum(sorted_labels == 1) + fp = np.cumsum(sorted_labels == 0) + youden_j = tp / n_pos - fp / n_neg + return float(sorted_scores[int(np.argmax(youden_j))]) + + +def _hit_and_error_rate(scores: np.ndarray, labels: np.ndarray, threshold: float) -> tuple[float, float | None]: + approved = scores >= threshold + n_approved = int(approved.sum()) + hit_rate = n_approved / len(scores) if len(scores) else 0.0 + if n_approved == 0: + return hit_rate, None + return hit_rate, float(np.mean(labels[approved] == 0)) + + +def _diagnose_ceiling(scores: np.ndarray, labels: np.ndarray) -> str | None: + """"insufficient_data" | "possible_ceiling" | "still_improvable" -- is + the fine-tuned verifier plausibly undertrained (more data would help) + or capped by the domain's own ambiguity? Precision in the top score + decile vs. the base rate, same shape of check as the hosted service.""" + n = len(labels) + if n < _MIN_TEST_FOR_CEILING_DIAGNOSIS: + return "insufficient_data" + base_rate = float(labels.mean()) + top_decile_n = max(1, n // 10) + top_decile_idx = np.argsort(-scores)[:top_decile_n] + precision_at_top_decile = float(labels[top_decile_idx].mean()) + auc = roc_auc(scores, labels) + if precision_at_top_decile - base_rate >= _CEILING_TAIL_MARGIN and auc < _CEILING_AUC_CAP: + return "possible_ceiling" + return "still_improvable" + + +def run_healthcheck( + examples: list[GrayZoneExample], + output_dir: str, + *, + base_model: str = DEFAULT_BASE_MODEL, + epochs: int = 1, +) -> HealthCheckResult: + """Fine-tune `base_model` on a chronological prefix of `examples` and + measure held-out AUC for the stock vs. fine-tuned model. + + `examples` must be in arrival (stream) order -- the train/calibrate/test + split is positional, not shuffled, matching the hosted service so the + numbers are comparable. + """ + if len(examples) < MIN_TRAIN_EXAMPLES: + raise ValueError(f"need at least {MIN_TRAIN_EXAMPLES} examples, got {len(examples)}") + + split = max(1, int(len(examples) * 0.8)) + train_rows, holdout_rows = examples[:split], examples[split:] + if not holdout_rows: + train_rows, holdout_rows = examples[:-1], examples[-1:] + calib_split = max(1, len(holdout_rows) // 2) + calib_rows, test_rows = holdout_rows[:calib_split], holdout_rows[calib_split:] + if not test_rows: + calib_rows, test_rows = holdout_rows[:-1], holdout_rows[-1:] + + device = "cuda" if torch.cuda.is_available() else "cpu" + test_pairs = [(e.query, e.candidate_answer) for e in test_rows] + test_labels = np.array([1 if e.was_correct else 0 for e in test_rows]) + calib_pairs = [(e.query, e.candidate_answer) for e in calib_rows] + calib_labels = np.array([1 if e.was_correct else 0 for e in calib_rows]) + train_positive_rate = float(np.mean([1 if e.was_correct else 0 for e in train_rows])) + test_positive_rate = float(test_labels.mean()) if len(test_labels) else 0.0 + + baseline = CrossEncoder(base_model, device=device, max_length=MAX_SEQUENCE_LENGTH) + test_pairs_scored = [smart_truncate_pair(q, a, baseline.tokenizer) for q, a in test_pairs] + baseline_scores = np.array(baseline.predict(test_pairs_scored, batch_size=32, show_progress_bar=False)) + auc_baseline = roc_auc(baseline_scores, test_labels) + del baseline + + tuned = CrossEncoder(base_model, device=device, max_length=MAX_SEQUENCE_LENGTH) + train_examples = [ + InputExample( + texts=list(smart_truncate_pair(e.query, e.candidate_answer, tuned.tokenizer)), + label=1.0 if e.was_correct else 0.0, + ) + for e in train_rows + ] + # A list[InputExample] is a valid PyTorch map-style dataset at runtime; + # sentence-transformers' pre-4.0 training API is built around exactly + # this, but the two libraries' stubs don't compose (InputExample isn't a + # typed Dataset). cast rather than `# type: ignore` so this stays clean + # whether or not torch's stubs are installed (base CI has no torch). + loader: DataLoader[Any] = DataLoader(cast(Any, train_examples), shuffle=True, batch_size=TRAIN_BATCH_SIZE) + t0 = time.time() + tuned.fit(train_dataloader=loader, epochs=epochs, show_progress_bar=False) + train_time_seconds = time.time() - t0 + tuned.save(output_dir) + + calib_scored = [smart_truncate_pair(q, a, tuned.tokenizer) for q, a in calib_pairs] + calib_scores = np.array(tuned.predict(calib_scored, batch_size=32, show_progress_bar=False)) + threshold = select_threshold(calib_scores, calib_labels) + + tuned_scores = np.array(tuned.predict(test_pairs_scored, batch_size=32, show_progress_bar=False)) + auc_tuned = roc_auc(tuned_scores, test_labels) + ceiling_status = _diagnose_ceiling(tuned_scores, test_labels) + + threshold_hit_rate: float | None = None + threshold_error_rate: float | None = None + train_label_disagreement_rate: float | None = None + if threshold is not None: + threshold_hit_rate, threshold_error_rate = _hit_and_error_rate(tuned_scores, test_labels, threshold) + train_pairs_scored = [smart_truncate_pair(e.query, e.candidate_answer, tuned.tokenizer) for e in train_rows] + train_scores = np.array(tuned.predict(train_pairs_scored, batch_size=32, show_progress_bar=False)) + train_labels = np.array([1 if e.was_correct else 0 for e in train_rows]) + train_pred = (train_scores >= threshold).astype(int) + train_label_disagreement_rate = float(np.mean(train_pred != train_labels)) + + return HealthCheckResult( + n_train=len(train_rows), + n_calibrate=len(calib_rows), + n_test=len(test_rows), + train_positive_rate=train_positive_rate, + test_positive_rate=test_positive_rate, + auc_baseline=auc_baseline, + auc_tuned=auc_tuned, + train_time_seconds=train_time_seconds, + threshold=threshold, + threshold_hit_rate=threshold_hit_rate, + threshold_error_rate=threshold_error_rate, + train_label_disagreement_rate=train_label_disagreement_rate, + ceiling_status=ceiling_status, + model_path=output_dir, + ) diff --git a/cacheverifier/_healthcheck/cli.py b/cacheverifier/_healthcheck/cli.py new file mode 100644 index 0000000..8a5de27 --- /dev/null +++ b/cacheverifier/_healthcheck/cli.py @@ -0,0 +1,209 @@ +"""Argument parsing, IO, and the printed report for `cacheverifier healthcheck`. + +The heavy work (torch, sentence-transformers) lives in `._finetune` and is +imported only inside `run()`, so `cacheverifier --help` stays light and +works without the `healthcheck` extra installed. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +_MISSING_EXTRA_HINT = ( + "the healthcheck extra is not installed -- run:\n" + ' pip install "cacheverifier[healthcheck]"\n' + "(this pulls in torch + sentence-transformers; it is deliberately not a\n" + "dependency of the base client)" +) + + +def add_subparser(subparsers: argparse._SubParsersAction) -> None: + p = subparsers.add_parser( + "healthcheck", + help="run the Health Check evaluation locally (no data leaves your machine)", + description=( + "Fine-tune the stock verifier on a chronological prefix of your own " + "gray-zone cache hits and measure held-out AUC before and after -- the " + "same evaluation as POST /v1/finetune/dry-run, but entirely offline. " + "Your queries and answers never leave this host; only the optional " + "--emit-summary file (floats and counts, no text) can." + ), + ) + p.add_argument( + "input", + type=Path, + help="JSON array or JSONL of {query, candidate_answer, was_correct} rows, in arrival order", + ) + p.add_argument( + "--emit-summary", + type=Path, + default=None, + metavar="PATH", + help="also write an aggregate-only JSON summary (no query/answer text) to PATH", + ) + p.add_argument( + "--epochs", + type=int, + default=1, + help="fine-tuning epochs (default: 1, matching the hosted service)", + ) + p.add_argument( + "--keep-model", + action="store_true", + help="keep the locally fine-tuned model directory instead of deleting it", + ) + p.set_defaults(func=run) + + +def _load_rows(path: Path) -> list[dict[str, Any]]: + text = path.read_text() + if text.lstrip().startswith("["): + rows = json.loads(text) + else: + rows = [json.loads(line) for line in text.splitlines() if line.strip()] + if not isinstance(rows, list): + # ValueError, not TypeError: this is bad input data, not a caller + # bug -- run() catches it alongside JSONDecodeError and prints it. + raise ValueError("input must be a JSON array or JSONL of objects") # noqa: TRY004 + return rows + + +def _to_examples(rows: list[dict[str, Any]], example_cls: Any) -> tuple[list[Any], int]: + examples = [] + n_stale = 0 + for i, row in enumerate(rows): + missing = {"query", "candidate_answer", "was_correct"} - row.keys() + if missing: + raise ValueError(f"row {i}: missing key(s) {sorted(missing)}") + if row.get("stale"): + n_stale += 1 + continue + examples.append( + example_cls( + query=str(row["query"]), + candidate_answer=str(row["candidate_answer"]), + was_correct=bool(row["was_correct"]), + ) + ) + return examples, n_stale + + +def _fmt(x: float | None, *, pct: bool = False) -> str: + if x is None: + return "n/a" + return f"{x * 100:.1f}%" if pct else f"{x:.4f}" + + +def _verdict(delta: float) -> str: + if delta >= 0.02: + return "IMPROVED -- fine-tuning on your own data helps this traffic" + if delta <= -0.02: + return "WORSE -- fine-tuning hurt; your signal may be too noisy or too sparse" + return "NO CHANGE -- fine-tuning neither helped nor hurt measurably" + + +def run(args: argparse.Namespace) -> int: + try: + from cacheverifier._healthcheck import _finetune + except ImportError as e: # torch / sentence-transformers / numpy absent + print(f"error: {_MISSING_EXTRA_HINT}\n\n(import failed: {e})", file=sys.stderr) + return 2 + + input_path: Path = args.input + if not input_path.exists(): + print(f"error: input file not found: {input_path}", file=sys.stderr) + return 2 + + try: + rows = _load_rows(input_path) + examples, n_stale = _to_examples(rows, _finetune.GrayZoneExample) + except (ValueError, json.JSONDecodeError, OSError) as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + n = len(examples) + print(f"\nCacheVerifier local Health Check -- {input_path}") + print("=" * 66) + print(f"rows read: {len(rows)}") + print(f"usable (non-stale): {n}") + if n_stale: + print(f"stale, excluded: {n_stale}") + + if n < _finetune.MIN_TRAIN_EXAMPLES: + print(f"\nNeed at least {_finetune.MIN_TRAIN_EXAMPLES} non-stale rows to run; have {n}.", file=sys.stderr) + return 2 + if n < _finetune.COLD_START_WARNING_THRESHOLD: + print( + f"\nNote: {n} rows is below the ~{_finetune.COLD_START_WARNING_THRESHOLD} the research " + "(cacheverifier.com/research, Q2) found fine-tuning reliably positive across every\n" + " dataset. A small-sample result here is real but noisy -- re-run as feedback grows." + ) + + import tempfile + + out_dir = tempfile.mkdtemp(prefix="cacheverifier_healthcheck_") + print(f"\nfine-tuning locally (nothing sent) -> {out_dir}") + print("this takes a few minutes on CPU; the base model downloads once on first run...\n") + + result = _finetune.run_healthcheck(examples, out_dir, base_model=_finetune.DEFAULT_BASE_MODEL, epochs=args.epochs) + + print("results") + print("-" * 66) + print(f" train / calibrate / test: {result.n_train} / {result.n_calibrate} / {result.n_test}") + print(f" stock verifier held-out AUC: {_fmt(result.auc_baseline)}") + print(f" fine-tuned held-out AUC: {_fmt(result.auc_tuned)} (delta {result.auc_delta:+.4f})") + print(f" label-noise proxy (disagreement): {_fmt(result.train_label_disagreement_rate, pct=True)}") + if result.train_label_disagreement_rate is not None: + if result.train_label_disagreement_rate > _finetune.NOISE_HOLD_BACK_DISAGREEMENT_RATE: + print(" -> HIGH: the hosted service would hold this model back for review") + elif result.train_label_disagreement_rate > _finetune.NOISE_WARNING_DISAGREEMENT_RATE: + print(" -> elevated: worth auditing how your correctness signal is produced") + print(f" train / test positive rate: {_fmt(result.train_positive_rate, pct=True)} / {_fmt(result.test_positive_rate, pct=True)}") + print(f" ceiling status: {result.ceiling_status or 'n/a'}") + if result.threshold is not None: + print( + f" operating point @ picked threshold: hit rate {_fmt(result.threshold_hit_rate, pct=True)}, " + f"error rate {_fmt(result.threshold_error_rate, pct=True)}" + ) + print("\nverdict") + print("-" * 66) + print(f" {_verdict(result.auc_delta)}\n") + + if args.emit_summary is not None: + summary = { + "n_rows_read": len(rows), + "n_usable": n, + "n_stale_excluded": n_stale, + "n_train": result.n_train, + "n_calibrate": result.n_calibrate, + "n_test": result.n_test, + "auc_stock": round(result.auc_baseline, 4), + "auc_finetuned": round(result.auc_tuned, 4), + "auc_delta": round(result.auc_delta, 4), + "train_label_disagreement_rate": ( + round(result.train_label_disagreement_rate, 4) + if result.train_label_disagreement_rate is not None + else None + ), + "train_positive_rate": round(result.train_positive_rate, 4), + "test_positive_rate": round(result.test_positive_rate, 4), + "ceiling_status": result.ceiling_status, + "threshold_hit_rate": result.threshold_hit_rate, + "threshold_error_rate": result.threshold_error_rate, + } + args.emit_summary.write_text(json.dumps(summary, indent=2) + "\n") + print(f"aggregate summary written to {args.emit_summary}") + print("(floats and counts only -- no query or answer text -- safe to share for a human read)\n") + + if args.keep_model: + print(f"fine-tuned model kept at {result.model_path}") + else: + import shutil + + shutil.rmtree(out_dir, ignore_errors=True) + + return 0 diff --git a/pyproject.toml b/pyproject.toml index 43ada5a..d71d4a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "cacheverifier" -version = "0.1.0" +version = "0.2.0" description = "Python client for the hosted CacheVerifier semantic-cache verification API" readme = "README.md" requires-python = ">=3.9" @@ -29,6 +29,9 @@ classifiers = [ ] dependencies = ["httpx>=0.24"] +[project.scripts] +cacheverifier = "cacheverifier.__main__:main" + [project.urls] Homepage = "https://www.cacheverifier.com" Documentation = "https://www.cacheverifier.com/docs" @@ -37,6 +40,20 @@ Research = "https://github.com/imxinchengyou/CacheVerifier" [project.optional-dependencies] gptcache = ["gptcache>=0.1.30"] +# `cacheverifier healthcheck` -- runs the hosted Health Check Report's +# stock-vs-fine-tuned evaluation locally. Deliberately NOT a base +# dependency: torch + sentence-transformers dwarf the httpx-only client. +# sentence-transformers is pinned <4 for the pre-4.0 CrossEncoder.fit() +# API this uses (the >=4 line needs torch>=2.4, which drops older Pythons). +# Effectively wants Python >=3.10. +healthcheck = [ + # numpy <2: the torch builds these versions pull are compiled against + # the numpy 1.x ABI ("Failed to initialize NumPy: _ARRAY_API not found" + # on numpy 2). Same constraint the hosted service uses. + "numpy>=1.26,<2", + "torch>=2.2", + "sentence-transformers>=3.0,<4", +] dev = ["pytest>=7", "mypy>=1.5", "ruff>=0.4"] [tool.hatch.build.targets.wheel] @@ -60,3 +77,11 @@ warn_redundant_casts = true [[tool.mypy.overrides]] module = "gptcache.*" ignore_missing_imports = true + +[[tool.mypy.overrides]] +# The healthcheck extra's deps: not installed in the base test/type-check +# env (CI runs `pip install -e ".[dev]"`, not `[dev,healthcheck]`), and the +# `_healthcheck` subpackage is only importable when that extra is present +# anyway. numpy ships inline types but only when installed. +module = ["torch.*", "sentence_transformers.*", "numpy.*"] +ignore_missing_imports = true diff --git a/tests/test_healthcheck.py b/tests/test_healthcheck.py new file mode 100644 index 0000000..ebc87d7 --- /dev/null +++ b/tests/test_healthcheck.py @@ -0,0 +1,98 @@ +"""CLI dispatch + Health Check runner. + +The dispatch/parsing tests run everywhere (base install). The end-to-end +training test needs the `healthcheck` extra and is skipped when torch is +absent -- CI's base matrix skips it; the dedicated `healthcheck` job runs +it. +""" + +from __future__ import annotations + +import importlib.util +import json +from dataclasses import dataclass + +import pytest + +from cacheverifier.__main__ import main +from cacheverifier._healthcheck import cli + +HAS_TORCH = importlib.util.find_spec("torch") is not None + + +@dataclass(frozen=True) +class _DummyExample: + query: str + candidate_answer: str + was_correct: bool + + +class TestDispatch: + def test_version(self, capsys): + with pytest.raises(SystemExit) as e: + main(["--version"]) + assert e.value.code == 0 + assert "cacheverifier 0.2.0" in capsys.readouterr().out + + def test_no_command_prints_help_and_returns_1(self, capsys): + assert main([]) == 1 + assert "healthcheck" in capsys.readouterr().out + + def test_healthcheck_help(self): + with pytest.raises(SystemExit) as e: + main(["healthcheck", "--help"]) + assert e.value.code == 0 + + @pytest.mark.skipif(HAS_TORCH, reason="torch present -- the missing-extra path can't be exercised") + def test_healthcheck_without_extra_points_at_pip_install(self, capsys, tmp_path): + f = tmp_path / "t.jsonl" + f.write_text("") + assert main(["healthcheck", str(f)]) == 2 + assert 'pip install "cacheverifier[healthcheck]"' in capsys.readouterr().err + + +class TestParsing: + def test_load_rows_accepts_json_array_and_jsonl(self, tmp_path): + rows = [{"a": 1}, {"a": 2}] + arr = tmp_path / "a.json" + arr.write_text(json.dumps(rows)) + jsonl = tmp_path / "a.jsonl" + jsonl.write_text("\n".join(json.dumps(r) for r in rows) + "\n") + assert cli._load_rows(arr) == rows + assert cli._load_rows(jsonl) == rows + + def test_to_examples_skips_stale_and_counts_them(self): + rows = [ + {"query": "q1", "candidate_answer": "a1", "was_correct": True}, + {"query": "q2", "candidate_answer": "a2", "was_correct": False, "stale": True}, + ] + examples, n_stale = cli._to_examples(rows, _DummyExample) + assert len(examples) == 1 + assert n_stale == 1 + assert examples[0] == _DummyExample("q1", "a1", True) + + def test_to_examples_rejects_missing_keys(self): + with pytest.raises(ValueError, match="missing key"): + cli._to_examples([{"query": "q"}], _DummyExample) + + +@pytest.mark.skipif(not HAS_TORCH, reason="needs the healthcheck extra (torch, sentence-transformers)") +def test_run_healthcheck_end_to_end(tmp_path): + from cacheverifier._healthcheck._finetune import GrayZoneExample, HealthCheckResult, run_healthcheck + + groups = [ + ("how do I cancel my subscription", "Settings > Billing > Cancel.", "Settings > Billing > Pause a month."), + ("how do I get a refund", "Orders > Return > Request refund.", "Orders > Return > Exchange size."), + ("how do I reset my password", "Account > Security > Reset password.", "Account > Security > Enable 2FA."), + ] + examples = [] + for i in range(36): + q, good, bad = groups[i % len(groups)] + correct = i % 2 == 0 + examples.append(GrayZoneExample(q, good if correct else bad, correct)) + + result = run_healthcheck(examples, str(tmp_path / "model"), epochs=1) + assert isinstance(result, HealthCheckResult) + assert result.n_train + result.n_calibrate + result.n_test == 36 + assert 0.0 <= result.auc_tuned <= 1.0 + assert (tmp_path / "model").is_dir()