diff --git a/CHANGELOG.md b/CHANGELOG.md index 6044b7d..305c98f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- New `plexus pick "REQUEST"` picks a tool for a plain-language request from what each manifest emits and consumes, with a probability for every tool, and abstains when no tool matches or the top probability is under `--threshold`. On a 51-request held-out set it picked the right tool 57% of the time, abstained on 10%, and abstained requests carried 2.7 times the error of picked ones. Details and the bar set before the run: `docs/PICK.md`. - The Claude plugin folder now carries the server code at `client-plugin/server/src/plexus/`, so an install of the folder alone starts. `python scripts/build_client_package.py --sync-vendored` rewrites that copy from `src/`, and a test fails when it drifts. The launcher no longer looks outside the plugin folder; a missing copy prints a one-line message and exits with status 1. A test keeps every code and text file under 256 KiB, images under 2 MB, and the folder at 512 files or fewer. - The skill and the MCP tool descriptions say "tool" where they said "organ". - The Claude plugin manifest adds the plugin directory listing fields: display name, keywords, homepage, repository, documentation, support, privacy and terms links, and a 1024 px icon. Plexus takes no launch settings, so the manifest declares no `userConfig`. Portable and Codex manifests are unchanged. diff --git a/README.md b/README.md index 5683d84..bd3f772 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,8 @@ body cannot make it agree with itself. And the receipt carries a method version that has to match before anything else is compared, so a plan written by an older plexus is reported as failing rather than silently re-interpreted under new rules. +Not sure which tool a request belongs to? `plexus pick "REQUEST"` returns a probability for each tool and abstains when none fits. Measured results are in [docs/PICK.md](docs/PICK.md). + ## How a tool plugs in A manifest is plain JSON. A tool ships one and it joins the mesh. Drop diff --git a/client-plugin/server/src/plexus/cli.py b/client-plugin/server/src/plexus/cli.py index a2c2f9c..457c600 100644 --- a/client-plugin/server/src/plexus/cli.py +++ b/client-plugin/server/src/plexus/cli.py @@ -4,6 +4,7 @@ plexus wiring [...] # capability -> (producer, consumer) pairs plexus plan --goal ORGAN [...] # upstream pipeline that feeds ORGAN plexus route --from A --to B [...] # capability path from A to B + plexus pick "REQUEST" [--threshold P] # pick a tool for a request, or abstain plexus validate [...] # manifest problems (exit 1 if any) """ from __future__ import annotations @@ -71,6 +72,11 @@ def main(argv: "list[str] | None" = None) -> int: rp = sub.add_parser("route"); _add_source_flags(rp) rp.add_argument("--from", dest="src", required=True) rp.add_argument("--to", dest="dst", required=True) + kp = sub.add_parser("pick", help="pick a tool for a request, or abstain") + _add_source_flags(kp) + kp.add_argument("request", help="the request in plain language") + kp.add_argument("--threshold", type=float, default=0.23, + help="abstain when the top tool's probability is under this") vp = sub.add_parser("verify"); _add_source_flags(vp) vp.add_argument("--plan", required=True, help="a plan/route JSON file (from `plexus plan`/`route`) to re-derive " @@ -106,6 +112,11 @@ def main(argv: "list[str] | None" = None) -> int: if args.cmd == "route": print(json.dumps(route(discover(mans), args.src, args.dst), indent=2)) return 0 + if args.cmd == "pick": + from .pick import pick + result = pick(discover(mans), args.request, threshold=args.threshold) + print(json.dumps(result, indent=2)) + return 0 if args.cmd == "verify": from .receipt import verify_plan with open(args.plan, encoding="utf-8") as f: diff --git a/client-plugin/server/src/plexus/pick.py b/client-plugin/server/src/plexus/pick.py new file mode 100644 index 0000000..dfdab04 --- /dev/null +++ b/client-plugin/server/src/plexus/pick.py @@ -0,0 +1,103 @@ +"""pick.py: choose a tool for a plain-language request, or abstain. + +Each tool in the mesh becomes a short document built from its manifest: its +name, the title and summary of every capability it emits or consumes, and the +words of each capability id. A request is scored against every tool with +BM25, scores become probabilities with a softmax, and the top tool is picked +only when its probability reaches ``threshold``. Otherwise the result is +ABSTAIN: ``picked`` is None and ``abstained`` is True, so a person or a larger +model decides. Deterministic: ties break by tool name. +""" +from __future__ import annotations + +import math +import re + +from .mesh import Mesh + +SCHEMA = "plexus.pick/1" +_TOKEN = re.compile(r"[a-z0-9]+") +_STOP = frozenset( + "a an and are as at be by can do for from how i in into is it its me my of on " + "or our please so that the this to up us we what when which why will with you " + "your v1 v2 1 2".split() +) + + +def _stem(word: str) -> str: + for suffix in ("ing", "ies", "ed", "s"): + if word.endswith(suffix) and len(word) - len(suffix) >= 3: + return word[: -len(suffix)] + ("y" if suffix == "ies" else "") + return word + + +def terms(text: str) -> list[str]: + """Lowercase word tokens, stop words dropped, light suffix stemming.""" + return [_stem(t) for t in _TOKEN.findall(text.lower()) if t not in _STOP] + + +def tool_document(manifest) -> list[str]: + parts = [manifest.organ.replace("-", " ")] + for port in list(manifest.emits) + list(manifest.consumes): + parts.extend([port.title, port.summary, port.capability]) + return terms(" ".join(parts)) + + +def _bm25(docs: list[list[str]], query: list[str], k1: float = 1.2, + b: float = 0.75) -> list[float]: + n = len(docs) + avgdl = sum(len(d) for d in docs) / n if n else 0.0 + df: dict[str, int] = {} + for d in docs: + for t in set(d): + df[t] = df.get(t, 0) + 1 + out = [] + for doc in docs: + total = 0.0 + for t in set(query): + f = doc.count(t) + if f: + idf = math.log(1 + (n - df[t] + 0.5) / (df[t] + 0.5)) + total += idf * f * (k1 + 1) / (f + k1 * (1 - b + b * len(doc) / (avgdl or 1))) + out.append(total) + return out + + +def _softmax(scores: list[float], temperature: float) -> list[float]: + if not scores: + return [] + top = max(scores) + exps = [math.exp((s - top) / temperature) for s in scores] + z = sum(exps) + return [e / z for e in exps] + + +def rank_tools(mesh: Mesh, request: str, temperature: float = 1.0) -> list[dict]: + """Every tool with its raw score and probability, best first.""" + organs = sorted(mesh.manifests) + raw = _bm25([tool_document(mesh.manifests[o]) for o in organs], terms(request)) + probs = _softmax(raw, temperature) + rows = [{"tool": o, "score": round(r, 6), "probability": round(p, 6)} + for o, r, p in zip(organs, raw, probs)] + rows.sort(key=lambda row: (-row["probability"], row["tool"])) + return rows + + +DEFAULT_THRESHOLD = 0.23 # the dev-half calibration on the labelled set, docs/PICK.md + + +def pick(mesh: Mesh, request: str, *, threshold: float = DEFAULT_THRESHOLD, + temperature: float = 1.0) -> dict: + """Pick a tool for ``request`` or abstain. Returns a JSON-ready dict. + + The default threshold is the value the dev-half calibration chose. A + threshold of 0.0 abstains only when no tool shares a word with the request. + """ + if not 0.0 <= threshold <= 1.0 or temperature <= 0: + raise ValueError("threshold must be in [0, 1] and temperature > 0") + rows = rank_tools(mesh, request, temperature) + top = rows[0] if rows else None + abstained = top is None or top["score"] <= 0 or top["probability"] < threshold + return {"schema": SCHEMA, "request": request, "threshold": threshold, + "picked": None if abstained else top["tool"], "abstained": abstained, + "confidence": top["probability"] if top else 0.0, "candidates": rows} diff --git a/client-plugin/server/src/plexus/pick_bench.py b/client-plugin/server/src/plexus/pick_bench.py new file mode 100644 index 0000000..b4fb12e --- /dev/null +++ b/client-plugin/server/src/plexus/pick_bench.py @@ -0,0 +1,110 @@ +"""pick_bench.py: measure `plexus pick` on a labelled request set. + +Items split in half by a seeded hash of their id. The abstain threshold is +calibrated on the dev half; numbers come from the test half. The baseline is +the dev half's majority tool, always picked. A shuffled-label control checks +that "abstained items carry more error" is not an artifact of the arithmetic. +""" +from __future__ import annotations + +import hashlib +import json +import math +import random +from collections import Counter +from pathlib import Path + +from .mesh import discover +from .pick import pick +from .registry import builtin_manifests + +SEED = 20261003 + + +def split(items: list[dict], seed: int = SEED) -> tuple[list[dict], list[dict]]: + dev, test = [], [] + for item in items: + digest = hashlib.sha256(f"{seed}:{item['id']}".encode()).digest() + (dev if digest[0] % 2 == 0 else test).append(item) + return dev, test + + +def wilson(k: int, n: int, z: float = 1.96) -> list[float]: + if n == 0: + return [0.0, 0.0] + p, d = k / n, 1 + z * z / n + c = p + z * z / (2 * n) + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) + return [round((c - h) / d, 3), round((c + h) / d, 3)] + + +def _rate(k: int, n: int) -> float: + return round(k / n, 4) if n else 0.0 + + +def abstain_stats(rows: list[tuple]) -> dict: + """rows: (top-1 tool, picked tool or None, gold). Error is top-1 vs gold.""" + dec = [r for r in rows if r[1] is not None] + abst = [r for r in rows if r[1] is None] + dec_rate = _rate(sum(1 for t, _, g in dec if t != g), len(dec)) + abs_rate = _rate(sum(1 for t, _, g in abst if t != g), len(abst)) + return {"decided": len(dec), "abstained": len(abst), + "abstain_rate": _rate(len(abst), len(rows)), + "decided_error_rate": dec_rate, "abstained_error_rate": abs_rate, + "error_ratio": round(abs_rate / dec_rate, 3) if dec_rate else None} + + +def calibrate(mesh, items: list[dict], share: float = 0.10) -> float: + """Lowest threshold whose abstain rate on ``items`` is at least ``share``.""" + tops = [] + for item in items: + top = pick(mesh, item["text"])["candidates"][0] + tops.append(top["probability"] if top["score"] > 0 else -1.0) + need = math.ceil(share * len(tops)) + forced = sum(1 for t in tops if t < 0) + if forced >= need: + return 0.0 + ranked = sorted(t for t in tops if t >= 0) + return min(1.0, round(ranked[need - forced - 1] + 1e-6, 6)) + + +def run(items: list[dict], mesh=None, seed: int = SEED) -> dict: + mesh = mesh or discover(builtin_manifests()) + dev, test = split(items, seed) + threshold = calibrate(mesh, dev) + rows = [] + for item in test: + r = pick(mesh, item["text"], threshold=threshold) + rows.append((r["candidates"][0]["tool"], r["picked"], item["gold"])) + correct = sum(1 for t, _, g in rows if t == g) + majority = Counter(i["gold"] for i in dev).most_common(1)[0][0] + base = sum(1 for i in test if i["gold"] == majority) + stats = abstain_stats(rows) + golds = [g for _, _, g in rows] + random.Random(seed).shuffle(golds) + shuffled = abstain_stats([(t, p, g) for (t, p, _), g in zip(rows, golds)]) + acc, base_acc = _rate(correct, len(rows)), _rate(base, len(test)) + bar = {"P1_accuracy_baseline_plus_0_20": acc >= base_acc + 0.20, + "P2_abstain_under_15pct": stats["abstain_rate"] < 0.15, + "P3_error_ratio_at_least_2": (stats["error_ratio"] or 0.0) >= 2.0, + "control_fails_P3": (shuffled["error_ratio"] or 0.0) < 2.0} + return {"schema": "plexus.pick-bench/1", "seed": seed, "dev": len(dev), + "test": len(test), "threshold": threshold, + "top1_accuracy": acc, "top1_wilson": wilson(correct, len(rows)), + "baseline": {"tool": majority, "accuracy": base_acc}, **stats, + "shuffled_label_control": shuffled, "bar": bar} + + +def main(argv: list[str] | None = None) -> int: + import sys + args = argv if argv is not None else sys.argv[1:] + if not args: + print("usage: python -m plexus.pick_bench LABELS.json", file=sys.stderr) + return 2 + items = json.loads(Path(args[0]).read_text(encoding="utf-8"))["items"] + print(json.dumps(run(items), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/PICK.md b/docs/PICK.md new file mode 100644 index 0000000..7610ef6 --- /dev/null +++ b/docs/PICK.md @@ -0,0 +1,52 @@ +# Pick a tool for a request, or abstain + +`plexus pick` reads a plain-language request and returns a probability for each tool in the mesh, scored against what each tool's manifest says it emits and consumes. When no tool matches, or the top probability is under a threshold, it abstains, so the request goes to a person or a larger model in place of a guessed tool. `plexus route` still answers the wiring question, how one tool's output reaches another. + +## Bar, set before the run + +This bar was committed before the picker was written and before its author saw the labelled set. + +**Labels.** 100 requests written by a separate labeller who saw only tool names, their capability titles and a one-line purpose, never code, each with one gold tool (`tests/fixtures/plexus_pick_labels.json`). The local routing-accuracy records hold 2 records, too few to use. + +**Split.** Items are split in half by a seeded hash of their id. The threshold is chosen on the dev half only: the lowest threshold whose dev abstain rate is at least 10%. All bar numbers come from the test half. + +**Baseline.** Plexus had no request picker. The baseline is the majority tool on the dev half, always picked. + +- **P1, accuracy.** Top-1 accuracy is at least the baseline's plus 0.20. +- **P2, abstain rate.** Under 15% of test items abstain. +- **P3, abstain carries error.** The top-1 error rate on abstained items is at least twice the error rate on decided items. +- **Control.** With the gold labels shuffled, P3 must fail. +- **Ship rule.** The picker ships as a new command whatever the result, with the numbers below. + +## Try it + +```bash +plexus pick "verify a sealed wiki pinned to a commit" +plexus pick --threshold 0.5 "check this memory still matches its source" +python -m plexus.pick_bench tests/fixtures/plexus_pick_labels.json +``` + +Each tool is scored with BM25 against a document built from its manifest: its name, the title and summary of each capability, and the words of each capability id. A softmax turns the scores into probabilities. The default threshold, 0.23, is the calibrated value from the run below. A threshold of 0.0 abstains only when no tool shares a word with the request. + +## Results + +Run on 2026-10-03, seed 20261003. Dev half 49 items, test half 51. The calibration rule picked a threshold of 0.23. + +| Test half, 51 items | Value | +|---|---| +| Top-1 accuracy | 0.57 (29 of 51), Wilson 0.43 to 0.70 | +| Baseline, dev majority tool (learn) | 0.02 | +| Abstain rate | 0.10 (5 of 51) | +| Error rate when picked | 0.37 | +| Error rate when abstained | 1.00 | +| Error ratio | 2.7 | +| Shuffled-label control ratio | 1.15 | + +- **P1 passes.** 0.57 against 0.02. The baseline is weak because the dev majority tool appeared once in the test half; uniform guessing over 10 tools would score about 0.10, and the picker clears that too. +- **P2 passes.** 10% of requests abstain. +- **P3 passes.** Abstained requests err at 1.00 against 0.37 for picked ones. +- **The control fails P3 as required.** Shuffled labels give 1.15. + +## Limits + +The labels come from one labeller, a Claude subagent that saw tool names, capability titles and a one-line purpose written for the task. 51 test items give wide intervals, and 4 in 10 picks are still wrong. Treat a pick as a suggestion to confirm. A tool whose manifest has sparse titles will rarely be picked. diff --git a/src/plexus/cli.py b/src/plexus/cli.py index a2c2f9c..457c600 100644 --- a/src/plexus/cli.py +++ b/src/plexus/cli.py @@ -4,6 +4,7 @@ plexus wiring [...] # capability -> (producer, consumer) pairs plexus plan --goal ORGAN [...] # upstream pipeline that feeds ORGAN plexus route --from A --to B [...] # capability path from A to B + plexus pick "REQUEST" [--threshold P] # pick a tool for a request, or abstain plexus validate [...] # manifest problems (exit 1 if any) """ from __future__ import annotations @@ -71,6 +72,11 @@ def main(argv: "list[str] | None" = None) -> int: rp = sub.add_parser("route"); _add_source_flags(rp) rp.add_argument("--from", dest="src", required=True) rp.add_argument("--to", dest="dst", required=True) + kp = sub.add_parser("pick", help="pick a tool for a request, or abstain") + _add_source_flags(kp) + kp.add_argument("request", help="the request in plain language") + kp.add_argument("--threshold", type=float, default=0.23, + help="abstain when the top tool's probability is under this") vp = sub.add_parser("verify"); _add_source_flags(vp) vp.add_argument("--plan", required=True, help="a plan/route JSON file (from `plexus plan`/`route`) to re-derive " @@ -106,6 +112,11 @@ def main(argv: "list[str] | None" = None) -> int: if args.cmd == "route": print(json.dumps(route(discover(mans), args.src, args.dst), indent=2)) return 0 + if args.cmd == "pick": + from .pick import pick + result = pick(discover(mans), args.request, threshold=args.threshold) + print(json.dumps(result, indent=2)) + return 0 if args.cmd == "verify": from .receipt import verify_plan with open(args.plan, encoding="utf-8") as f: diff --git a/src/plexus/pick.py b/src/plexus/pick.py new file mode 100644 index 0000000..dfdab04 --- /dev/null +++ b/src/plexus/pick.py @@ -0,0 +1,103 @@ +"""pick.py: choose a tool for a plain-language request, or abstain. + +Each tool in the mesh becomes a short document built from its manifest: its +name, the title and summary of every capability it emits or consumes, and the +words of each capability id. A request is scored against every tool with +BM25, scores become probabilities with a softmax, and the top tool is picked +only when its probability reaches ``threshold``. Otherwise the result is +ABSTAIN: ``picked`` is None and ``abstained`` is True, so a person or a larger +model decides. Deterministic: ties break by tool name. +""" +from __future__ import annotations + +import math +import re + +from .mesh import Mesh + +SCHEMA = "plexus.pick/1" +_TOKEN = re.compile(r"[a-z0-9]+") +_STOP = frozenset( + "a an and are as at be by can do for from how i in into is it its me my of on " + "or our please so that the this to up us we what when which why will with you " + "your v1 v2 1 2".split() +) + + +def _stem(word: str) -> str: + for suffix in ("ing", "ies", "ed", "s"): + if word.endswith(suffix) and len(word) - len(suffix) >= 3: + return word[: -len(suffix)] + ("y" if suffix == "ies" else "") + return word + + +def terms(text: str) -> list[str]: + """Lowercase word tokens, stop words dropped, light suffix stemming.""" + return [_stem(t) for t in _TOKEN.findall(text.lower()) if t not in _STOP] + + +def tool_document(manifest) -> list[str]: + parts = [manifest.organ.replace("-", " ")] + for port in list(manifest.emits) + list(manifest.consumes): + parts.extend([port.title, port.summary, port.capability]) + return terms(" ".join(parts)) + + +def _bm25(docs: list[list[str]], query: list[str], k1: float = 1.2, + b: float = 0.75) -> list[float]: + n = len(docs) + avgdl = sum(len(d) for d in docs) / n if n else 0.0 + df: dict[str, int] = {} + for d in docs: + for t in set(d): + df[t] = df.get(t, 0) + 1 + out = [] + for doc in docs: + total = 0.0 + for t in set(query): + f = doc.count(t) + if f: + idf = math.log(1 + (n - df[t] + 0.5) / (df[t] + 0.5)) + total += idf * f * (k1 + 1) / (f + k1 * (1 - b + b * len(doc) / (avgdl or 1))) + out.append(total) + return out + + +def _softmax(scores: list[float], temperature: float) -> list[float]: + if not scores: + return [] + top = max(scores) + exps = [math.exp((s - top) / temperature) for s in scores] + z = sum(exps) + return [e / z for e in exps] + + +def rank_tools(mesh: Mesh, request: str, temperature: float = 1.0) -> list[dict]: + """Every tool with its raw score and probability, best first.""" + organs = sorted(mesh.manifests) + raw = _bm25([tool_document(mesh.manifests[o]) for o in organs], terms(request)) + probs = _softmax(raw, temperature) + rows = [{"tool": o, "score": round(r, 6), "probability": round(p, 6)} + for o, r, p in zip(organs, raw, probs)] + rows.sort(key=lambda row: (-row["probability"], row["tool"])) + return rows + + +DEFAULT_THRESHOLD = 0.23 # the dev-half calibration on the labelled set, docs/PICK.md + + +def pick(mesh: Mesh, request: str, *, threshold: float = DEFAULT_THRESHOLD, + temperature: float = 1.0) -> dict: + """Pick a tool for ``request`` or abstain. Returns a JSON-ready dict. + + The default threshold is the value the dev-half calibration chose. A + threshold of 0.0 abstains only when no tool shares a word with the request. + """ + if not 0.0 <= threshold <= 1.0 or temperature <= 0: + raise ValueError("threshold must be in [0, 1] and temperature > 0") + rows = rank_tools(mesh, request, temperature) + top = rows[0] if rows else None + abstained = top is None or top["score"] <= 0 or top["probability"] < threshold + return {"schema": SCHEMA, "request": request, "threshold": threshold, + "picked": None if abstained else top["tool"], "abstained": abstained, + "confidence": top["probability"] if top else 0.0, "candidates": rows} diff --git a/src/plexus/pick_bench.py b/src/plexus/pick_bench.py new file mode 100644 index 0000000..b4fb12e --- /dev/null +++ b/src/plexus/pick_bench.py @@ -0,0 +1,110 @@ +"""pick_bench.py: measure `plexus pick` on a labelled request set. + +Items split in half by a seeded hash of their id. The abstain threshold is +calibrated on the dev half; numbers come from the test half. The baseline is +the dev half's majority tool, always picked. A shuffled-label control checks +that "abstained items carry more error" is not an artifact of the arithmetic. +""" +from __future__ import annotations + +import hashlib +import json +import math +import random +from collections import Counter +from pathlib import Path + +from .mesh import discover +from .pick import pick +from .registry import builtin_manifests + +SEED = 20261003 + + +def split(items: list[dict], seed: int = SEED) -> tuple[list[dict], list[dict]]: + dev, test = [], [] + for item in items: + digest = hashlib.sha256(f"{seed}:{item['id']}".encode()).digest() + (dev if digest[0] % 2 == 0 else test).append(item) + return dev, test + + +def wilson(k: int, n: int, z: float = 1.96) -> list[float]: + if n == 0: + return [0.0, 0.0] + p, d = k / n, 1 + z * z / n + c = p + z * z / (2 * n) + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) + return [round((c - h) / d, 3), round((c + h) / d, 3)] + + +def _rate(k: int, n: int) -> float: + return round(k / n, 4) if n else 0.0 + + +def abstain_stats(rows: list[tuple]) -> dict: + """rows: (top-1 tool, picked tool or None, gold). Error is top-1 vs gold.""" + dec = [r for r in rows if r[1] is not None] + abst = [r for r in rows if r[1] is None] + dec_rate = _rate(sum(1 for t, _, g in dec if t != g), len(dec)) + abs_rate = _rate(sum(1 for t, _, g in abst if t != g), len(abst)) + return {"decided": len(dec), "abstained": len(abst), + "abstain_rate": _rate(len(abst), len(rows)), + "decided_error_rate": dec_rate, "abstained_error_rate": abs_rate, + "error_ratio": round(abs_rate / dec_rate, 3) if dec_rate else None} + + +def calibrate(mesh, items: list[dict], share: float = 0.10) -> float: + """Lowest threshold whose abstain rate on ``items`` is at least ``share``.""" + tops = [] + for item in items: + top = pick(mesh, item["text"])["candidates"][0] + tops.append(top["probability"] if top["score"] > 0 else -1.0) + need = math.ceil(share * len(tops)) + forced = sum(1 for t in tops if t < 0) + if forced >= need: + return 0.0 + ranked = sorted(t for t in tops if t >= 0) + return min(1.0, round(ranked[need - forced - 1] + 1e-6, 6)) + + +def run(items: list[dict], mesh=None, seed: int = SEED) -> dict: + mesh = mesh or discover(builtin_manifests()) + dev, test = split(items, seed) + threshold = calibrate(mesh, dev) + rows = [] + for item in test: + r = pick(mesh, item["text"], threshold=threshold) + rows.append((r["candidates"][0]["tool"], r["picked"], item["gold"])) + correct = sum(1 for t, _, g in rows if t == g) + majority = Counter(i["gold"] for i in dev).most_common(1)[0][0] + base = sum(1 for i in test if i["gold"] == majority) + stats = abstain_stats(rows) + golds = [g for _, _, g in rows] + random.Random(seed).shuffle(golds) + shuffled = abstain_stats([(t, p, g) for (t, p, _), g in zip(rows, golds)]) + acc, base_acc = _rate(correct, len(rows)), _rate(base, len(test)) + bar = {"P1_accuracy_baseline_plus_0_20": acc >= base_acc + 0.20, + "P2_abstain_under_15pct": stats["abstain_rate"] < 0.15, + "P3_error_ratio_at_least_2": (stats["error_ratio"] or 0.0) >= 2.0, + "control_fails_P3": (shuffled["error_ratio"] or 0.0) < 2.0} + return {"schema": "plexus.pick-bench/1", "seed": seed, "dev": len(dev), + "test": len(test), "threshold": threshold, + "top1_accuracy": acc, "top1_wilson": wilson(correct, len(rows)), + "baseline": {"tool": majority, "accuracy": base_acc}, **stats, + "shuffled_label_control": shuffled, "bar": bar} + + +def main(argv: list[str] | None = None) -> int: + import sys + args = argv if argv is not None else sys.argv[1:] + if not args: + print("usage: python -m plexus.pick_bench LABELS.json", file=sys.stderr) + return 2 + items = json.loads(Path(args[0]).read_text(encoding="utf-8"))["items"] + print(json.dumps(run(items), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/plexus_pick_labels.json b/tests/fixtures/plexus_pick_labels.json new file mode 100644 index 0000000..7dd3b83 --- /dev/null +++ b/tests/fixtures/plexus_pick_labels.json @@ -0,0 +1,606 @@ +{ + "schema": "plexus.pick-labels/1", + "author": "independent labeller, tool descriptions only", + "items": [ + { + "id": "p001", + "text": "pull the last two weeks of arxiv papers on speculative decoding and give me a digest I can cite", + "gold": "gather", + "alt": null + }, + { + "id": "p002", + "text": "I need the FastAPI docs plus a couple of blog posts collected, with a note of where each one came from", + "gold": "gather", + "alt": null + }, + { + "id": "p003", + "text": "scrape a few pages about the new EU AI Act guidance and keep a receipt for every page", + "gold": "gather", + "alt": null + }, + { + "id": "p004", + "text": "is this quote really from the paper it says it is from, or did someone invent it?", + "gold": "gather", + "alt": "crucible" + }, + { + "id": "p005", + "text": "set up one run that hits the web, vendor docs and arxiv for RAG evaluation methods", + "gold": "gather", + "alt": null + }, + { + "id": "p006", + "text": "collect background reading on CRDTs", + "gold": "gather", + "alt": null + }, + { + "id": "p007", + "text": "lock down the research digest so nobody can quietly swap a source later", + "gold": "gather", + "alt": null + }, + { + "id": "p008", + "text": "find me papers that cite the Chinchilla scaling result", + "gold": "gather", + "alt": null + }, + { + "id": "p009", + "text": "grab the Postgres 17 release notes and changelog, and keep track of the URLs", + "gold": "gather", + "alt": null + }, + { + "id": "p010", + "text": "what are competitors saying publicly about pricing for agent tracing, with sources", + "gold": "gather", + "alt": null + }, + { + "id": "p011", + "text": "I claim our cache cuts p95 latency by 30 percent, check that against the benchmark numbers", + "gold": "crucible", + "alt": null + }, + { + "id": "p012", + "text": "judge whether this hypothesis holds up given the numbers in results.csv", + "gold": "crucible", + "alt": null + }, + { + "id": "p013", + "text": "write my hypothesis up as a structured thesis so it can be scored", + "gold": "crucible", + "alt": null + }, + { + "id": "p014", + "text": "make a replay bundle so a reviewer can rerun the whole assessment on their machine", + "gold": "crucible", + "alt": null + }, + { + "id": "p015", + "text": "block the release unless measured accuracy clears 0.9", + "gold": "crucible", + "alt": "flywheel-infra" + }, + { + "id": "p016", + "text": "use the sources I pulled yesterday as evidence for the main claim in the blog draft", + "gold": "crucible", + "alt": "gather" + }, + { + "id": "p017", + "text": "give me a measurement for each statement in the README benchmark section", + "gold": "crucible", + "alt": null + }, + { + "id": "p018", + "text": "does the data honestly support saying model B beats model A?", + "gold": "crucible", + "alt": null + }, + { + "id": "p019", + "text": "publish the claim we're testing before the run so outsiders can see what we committed to", + "gold": "crucible", + "alt": null + }, + { + "id": "p020", + "text": "rerun last month's assessment from its template and tell me if the verdict changed", + "gold": "crucible", + "alt": null + }, + { + "id": "p021", + "text": "split this refactor across three agents and keep a ledger of who did what", + "gold": "forum", + "alt": null + }, + { + "id": "p022", + "text": "turn our LangSmith traces into something tamper-evident", + "gold": "forum", + "alt": null + }, + { + "id": "p023", + "text": "import the OpenTelemetry trace from yesterday's agent run and make it verifiable", + "gold": "forum", + "alt": null + }, + { + "id": "p024", + "text": "route this ticket to whichever agent fits best and record the decision", + "gold": "forum", + "alt": null + }, + { + "id": "p025", + "text": "I want a receipt every time an agent submits its work", + "gold": "forum", + "alt": null + }, + { + "id": "p026", + "text": "package the planning room's context so the next agent sees what was decided", + "gold": "forum", + "alt": "canon" + }, + { + "id": "p027", + "text": "run a debate between a planner agent and a critic agent", + "gold": "forum", + "alt": null + }, + { + "id": "p028", + "text": "AgentOps export -> ledger please", + "gold": "forum", + "alt": null + }, + { + "id": "p029", + "text": "coordinate a reviewer agent and a coder agent on PR 412", + "gold": "forum", + "alt": null + }, + { + "id": "p030", + "text": "who changed the plan mid-run? I need a witnessed log of the multi-agent session", + "gold": "forum", + "alt": "relay" + }, + { + "id": "p031", + "text": "build a wiki of this repo pinned to the current commit", + "gold": "index", + "alt": null + }, + { + "id": "p032", + "text": "give me a context bundle for the auth module that fits in 8k tokens", + "gold": "index", + "alt": null + }, + { + "id": "p033", + "text": "is it true that every handler goes through the rate-limit middleware?", + "gold": "index", + "alt": "crucible" + }, + { + "id": "p034", + "text": "has the code drifted since we generated the wiki?", + "gold": "index", + "alt": null + }, + { + "id": "p035", + "text": "map out the repo structure for a new hire", + "gold": "index", + "alt": null + }, + { + "id": "p036", + "text": "where is parse_config defined and who calls it", + "gold": "index", + "alt": null + }, + { + "id": "p037", + "text": "re-check that the sealed wiki from the last release still matches the code", + "gold": "index", + "alt": null + }, + { + "id": "p038", + "text": "pin a snapshot so I can diff it against the working tree later", + "gold": "index", + "alt": null + }, + { + "id": "p039", + "text": "feed repo context into the envelopes the other tools consume", + "gold": "index", + "alt": "telos" + }, + { + "id": "p040", + "text": "index the monorepo", + "gold": "index", + "alt": null + }, + { + "id": "p041", + "text": "remember that we decided to drop Python 3.9 support", + "gold": "mneme", + "alt": null + }, + { + "id": "p042", + "text": "what did I tell the agent about the deploy window last week, and where did that come from?", + "gold": "mneme", + "alt": null + }, + { + "id": "p043", + "text": "check whether the agent's stored memories still match the source docs", + "gold": "mneme", + "alt": null + }, + { + "id": "p044", + "text": "save these chat turns so the agent can recall them later", + "gold": "mneme", + "alt": null + }, + { + "id": "p045", + "text": "the agent keeps remembering a config value that changed months ago, find the drift", + "gold": "mneme", + "alt": null + }, + { + "id": "p046", + "text": "turn this remembered fact into a thesis we can test", + "gold": "mneme", + "alt": "crucible" + }, + { + "id": "p047", + "text": "is the file behind this memory still current?", + "gold": "mneme", + "alt": null + }, + { + "id": "p048", + "text": "recall the key rotation policy we discussed, and show me proof of where it came from", + "gold": "mneme", + "alt": null + }, + { + "id": "p049", + "text": "audit the agent's memory for things it made up", + "gold": "mneme", + "alt": null + }, + { + "id": "p050", + "text": "ingest the meeting notes into agent memory", + "gold": "mneme", + "alt": "canon" + }, + { + "id": "p051", + "text": "carry my working context from Claude into a Codex session tomorrow", + "gold": "canon", + "alt": null + }, + { + "id": "p052", + "text": "snapshot where I am before I close this session so I can pick it back up", + "gold": "canon", + "alt": null + }, + { + "id": "p053", + "text": "is this host ready to load a context capsule?", + "gold": "canon", + "alt": null + }, + { + "id": "p054", + "text": "verify the markdown capsule file hasn't been edited", + "gold": "canon", + "alt": null + }, + { + "id": "p055", + "text": "build a provider-neutral context record from these three spec files", + "gold": "canon", + "alt": null + }, + { + "id": "p056", + "text": "expose our context records to other clients over read-only MCP", + "gold": "canon", + "alt": null + }, + { + "id": "p057", + "text": "start a fresh session and prove it booted from the right context", + "gold": "canon", + "alt": null + }, + { + "id": "p058", + "text": "switching from GPT to Claude halfway through the project, how do I not lose state?", + "gold": "canon", + "alt": null + }, + { + "id": "p059", + "text": "check whether the capsule is ready to hand off", + "gold": "canon", + "alt": null + }, + { + "id": "p060", + "text": "hand off context between sessions", + "gold": "canon", + "alt": "mneme" + }, + { + "id": "p061", + "text": "run a coding agent against my local llama.cpp server", + "gold": "relay", + "alt": null + }, + { + "id": "p062", + "text": "fall back to an online endpoint if the local model is down", + "gold": "relay", + "alt": null + }, + { + "id": "p063", + "text": "reopen yesterday's agent session", + "gold": "relay", + "alt": null + }, + { + "id": "p064", + "text": "list my saved agent sessions", + "gold": "relay", + "alt": null + }, + { + "id": "p065", + "text": "verify that certificate offline, no network", + "gold": "relay", + "alt": null + }, + { + "id": "p066", + "text": "expose the local agent as a remote MCP endpoint", + "gold": "relay", + "alt": null + }, + { + "id": "p067", + "text": "run the agent with a bounded budget and only accept the result if it passes the gate", + "gold": "relay", + "alt": null + }, + { + "id": "p068", + "text": "hash-chained log of a single local agent session", + "gold": "relay", + "alt": "forum" + }, + { + "id": "p069", + "text": "try Qwen on ollama for this task", + "gold": "relay", + "alt": null + }, + { + "id": "p070", + "text": "I want a correctness certificate for the patch the agent wrote", + "gold": "relay", + "alt": "crucible" + }, + { + "id": "p071", + "text": "teach me the pumping lemma step by step", + "gold": "learn", + "alt": null + }, + { + "id": "p072", + "text": "turn this proof into a lesson", + "gold": "learn", + "alt": null + }, + { + "id": "p073", + "text": "did the student really master recursion or just guess right?", + "gold": "learn", + "alt": null + }, + { + "id": "p074", + "text": "which misconceptions about pointers are most common across the class", + "gold": "learn", + "alt": null + }, + { + "id": "p075", + "text": "record that I finished the linear algebra unit", + "gold": "learn", + "alt": null + }, + { + "id": "p076", + "text": "make a lesson out of the Cauchy-Schwarz proof and check the claims in it", + "gold": "learn", + "alt": "crucible" + }, + { + "id": "p077", + "text": "tutor me on Bayes", + "gold": "learn", + "alt": null + }, + { + "id": "p078", + "text": "lock the next module until I pass the quiz", + "gold": "learn", + "alt": null + }, + { + "id": "p079", + "text": "rank the mistakes students make on induction proofs", + "gold": "learn", + "alt": null + }, + { + "id": "p080", + "text": "log a mastery credential for today's tutoring session", + "gold": "learn", + "alt": null + }, + { + "id": "p081", + "text": "what's the state of all my tools and what should I do next", + "gold": "telos", + "alt": null + }, + { + "id": "p082", + "text": "summarize what the five flagship tools are up to", + "gold": "telos", + "alt": null + }, + { + "id": "p083", + "text": "run the end-to-end golden workflow check", + "gold": "telos", + "alt": null + }, + { + "id": "p084", + "text": "do one thing that touches gather, crucible and index together", + "gold": "telos", + "alt": null + }, + { + "id": "p085", + "text": "is my workbench ready", + "gold": "telos", + "alt": null + }, + { + "id": "p086", + "text": "one action spanning several tools with a single receipt", + "gold": "telos", + "alt": "forum" + }, + { + "id": "p087", + "text": "dashboard of everything", + "gold": "telos", + "alt": null + }, + { + "id": "p088", + "text": "after the upgrade, verify the main workflow still works across the tools", + "gold": "telos", + "alt": null + }, + { + "id": "p089", + "text": "next actions?", + "gold": "telos", + "alt": null + }, + { + "id": "p090", + "text": "room summary", + "gold": "telos", + "alt": null + }, + { + "id": "p091", + "text": "did the agent try to send data to a host it shouldn't?", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p092", + "text": "a token showed up in the logs, give me a sealed exposure report", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p093", + "text": "halt the pipeline only when two independent monitors agree something is wrong", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p094", + "text": "produce a bill of materials for this run", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p095", + "text": "receipt every tool call the agent makes", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p096", + "text": "test that the sandbox boundary holds", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p097", + "text": "correlate alerts across the network and process layers", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p098", + "text": "write up an organizational lesson from last night's incident", + "gold": "flywheel-infra", + "alt": "learn" + }, + { + "id": "p099", + "text": "governance state across lanes", + "gold": "flywheel-infra", + "alt": null + }, + { + "id": "p100", + "text": "grade these failures so we can learn from them", + "gold": "flywheel-infra", + "alt": "learn" + } + ] +} diff --git a/tests/test_plexus_pick.py b/tests/test_plexus_pick.py new file mode 100644 index 0000000..421ee9f --- /dev/null +++ b/tests/test_plexus_pick.py @@ -0,0 +1,103 @@ +"""plexus pick: tool probabilities, the abstain outcome and the bench. + +Each behaviour is asserted with a paired mutation that must flip it. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from plexus.cli import main # noqa: E402 +from plexus.manifest import Manifest # noqa: E402 +from plexus.mesh import discover # noqa: E402 +from plexus.pick import pick, terms, tool_document # noqa: E402 +from plexus.pick_bench import abstain_stats, calibrate, run, split # noqa: E402 + +FIXTURE = Path(__file__).parent / "fixtures" / "plexus_pick_labels.json" + + +def _mesh(): + return discover([ + Manifest.from_dict({"organ": "memory", "emits": [ + {"capability": "mem.recall/1", "title": "recall receipt for stored notes"}]}), + Manifest.from_dict({"organ": "judge", "consumes": [ + {"capability": "judge.thesis/1", "title": "thesis measured against evidence"}]}), + ]) + + +def test_clear_request_is_picked_and_unmatched_request_abstains(): + mesh = _mesh() + hit = pick(mesh, "recall my stored notes") + assert hit["picked"] == "memory" and hit["abstained"] is False + miss = pick(mesh, "compose a symphony") + assert miss["picked"] is None and miss["abstained"] is True + + +def test_threshold_turns_a_weak_match_into_an_abstain(): + weak = "notes about the thesis" + assert pick(_mesh(), weak, threshold=0.0)["picked"] is not None + assert pick(_mesh(), weak, threshold=0.99)["picked"] is None + + +def test_candidates_are_probabilities_best_first(): + r = pick(_mesh(), "measure the thesis against evidence") + probs = [c["probability"] for c in r["candidates"]] + assert abs(sum(probs) - 1.0) < 1e-5 and probs == sorted(probs, reverse=True) + assert r["candidates"][0]["tool"] == "judge" + + +def test_tool_document_reads_titles_and_capability_words(): + doc = tool_document(_mesh().manifests["memory"]) + assert "receipt" in doc and "mem" in doc and "memory" in doc + assert "thesis" not in doc + + +def test_terms_stem_and_drop_stop_words(): + assert terms("The receipts") == ["receipt"] + assert terms("the") == [] + + +def test_invalid_threshold_is_refused(): + with pytest.raises(ValueError): + pick(_mesh(), "x", threshold=2.0) + + +def test_calibration_reaches_the_requested_share(): + items = [{"text": t} for t in ("recall notes", "thesis evidence", "notes thesis", + "symphony", "recall", "evidence", "stored", + "measured", "receipt", "notes")] + mesh = _mesh() + t = calibrate(mesh, items, share=0.30) + assert sum(pick(mesh, i["text"], threshold=t)["abstained"] for i in items) >= 3 + assert calibrate(mesh, items, share=0.30) >= calibrate(mesh, items, share=0.10) + + +def test_abstain_stats_and_undefined_ratio(): + s = abstain_stats([("a", "a", "a"), ("a", "a", "b"), ("a", None, "b")]) + assert s["decided_error_rate"] == 0.5 and s["error_ratio"] == 2.0 + assert abstain_stats([("a", "a", "a"), ("a", None, "b")])["error_ratio"] is None + + +def test_split_is_deterministic_and_complete(): + items = [{"id": f"p{i:03d}"} for i in range(30)] + dev, test = split(items) + assert (dev, test) == split(items) + assert len(dev) + len(test) == 30 and split(items, seed=7) != (dev, test) + + +def test_bench_reproduces_the_reported_result(): + items = json.loads(FIXTURE.read_text(encoding="utf-8"))["items"] + out = run(items) + assert all(out["bar"].values()), out["bar"] + assert out == run(items) + + +def test_cli_pick_prints_a_pick(capsys): + assert main(["pick", "verify a sealed wiki pinned to a commit"]) == 0 + out = json.loads(capsys.readouterr().out) + assert out["schema"] == "plexus.pick/1" and out["picked"] == "index"