From 8ee066557fe25dae07cbad8993915b6de9e9ecf3 Mon Sep 17 00:00:00 2001 From: webdevtodayjason Date: Mon, 6 Jul 2026 12:18:51 -0500 Subject: [PATCH] =?UTF-8?q?feat(autodata):=20v2.2=20=E2=80=94=20Evalchemy-?= =?UTF-8?q?style=20val-set=20objective=20for=20the=20meta-loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The honest v1/v2.1 finding: live yield was noisy/low (best ~25%) and the root cause is the EVAL SIGNAL, not the optimizer. The v2.1 reward — the per-batch Δ=1 keep-rate — re-samples fresh tasks every round (so it wobbles for reasons unrelated to P) and rides exact-match arithmetic's thin ZPD. Optimizing that proxy chases noise. v2.2 swaps the proxy for a held-out, GT-graded LIFT objective (the Open-Thoughts/ Evalchemy move): lift(P) = acc(weak | few-shot of D(P)) − acc(weak alone) on a FIXED val set - valset.py: evaluate() + valset_lift() — score the weak solver on a fixed, ground-truth-labeled val set, cold (baseline) and primed with a few-shot sample of the round's kept teacher traces. In-context learning as a torch-free proxy for fine-tuning lift; graded by the same trusted verifier the Δ-filter uses (core.judge_correct → verify.py). Stable + comparable round-over-round. - meta.py: meta_optimize(objective="valset") tracks best_lift/best_score, feeds the lift trajectory to the prompt-optimizer, stops at val_target. The v2.1 objective="yield" path is the untouched default — nothing regresses. - core.AutoDataConfig: objective / val_set / val_shots / val_target config keys. - run.py: --objective valset CLI flag. - Pure HTTP + stdlib only; fully offline-testable via injected fake clients (valset.demo, meta.demo_valset, + pytest cases). Live endpoints are the operator's to run. - V2_DESIGN.md: v2.2 marked implemented, objective documented, RESULTS placeholder. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017NziXzqT1L9kj2T1byA3Ak --- ainode/training/autodata/V2_DESIGN.md | 49 +++++- ainode/training/autodata/__init__.py | 1 + ainode/training/autodata/core.py | 19 +++ ainode/training/autodata/meta.py | 177 +++++++++++++++++---- ainode/training/autodata/run.py | 45 ++++-- ainode/training/autodata/valset.py | 216 ++++++++++++++++++++++++++ tests/test_autodata.py | 191 ++++++++++++++++++++++- 7 files changed, 658 insertions(+), 40 deletions(-) create mode 100644 ainode/training/autodata/valset.py diff --git a/ainode/training/autodata/V2_DESIGN.md b/ainode/training/autodata/V2_DESIGN.md index a6a3561..76ec1fe 100644 --- a/ainode/training/autodata/V2_DESIGN.md +++ b/ainode/training/autodata/V2_DESIGN.md @@ -45,11 +45,56 @@ Expose what they ablated, with their best-performers as defaults; the meta-opt s synthetically (your small-dataset → big-dataset insight). Real-world seed → synthetic scale. ## Build slices (each its own /goal, smallest-first) -- **v2.1 — meta-optimizer** (the yield fix): the `P_t→P_{t+1}` loop + val-set objective. Biggest payoff. -- **v2.2 — Evalchemy integration**: wire Evalchemy as the verifier/objective (served-endpoint eval). +- **v2.1 — meta-optimizer** (the yield fix): the `P_t→P_{t+1}` loop + Δ=1-yield proxy. SHIPPED (`meta.py`). +- **v2.2 — Evalchemy-style val-set objective** (the *real* yield fix): swap the noisy keep-rate proxy + for a held-out, GT-graded LIFT objective. **IMPLEMENTED** (`valset.py` + `meta.py objective="valset"`). See below. - **v2.3 — recipe knobs + OpenThoughts seeds**: the methodology menu + dataset bootstrapping. - **v2.4 — lift ingestion**: documents → structured seeds → synthetic extension. +## v2.2 — the val-set objective (IMPLEMENTED) + +**Why (the honest finding, 2026-06-28):** live yield was noisy/low (best ~25%). Root cause is +the **eval signal**, not the optimizer. v2.1's reward is the per-batch Δ=1 keep-rate — a +self-graded proxy that (a) re-samples fresh tasks every round (the number wobbles for reasons +unrelated to P) and (b) rode exact-match arithmetic, whose thin ZPD mislabels correct-but- +reformatted answers. Optimizing that proxy chases noise. + +**What v2.2 optimizes instead — measured lift on a held-out val set:** + + lift(P) = acc(weak | few-shot of D(P)) − acc(weak alone) # on a FIXED, GT-labeled val set + +- `D(P)` = the round's kept Δ=1 "teacher" traces (strong-solves / weak-fails). +- The weak solver is scored on a fixed, ground-truth-labeled val set (`val_set`), **cold** + (baseline, computed once) and **primed** with a few-shot sample (`val_shots`) of `D(P)`. +- Both are graded by the **same trusted verifier** the Δ-filter uses (`core.judge_correct` → + `verify.py` value-verify + rubric fallback) — the Evalchemy "proven verifier" role. +- This is **in-context learning as a torch-free proxy for fine-tuning lift**: if the kept + traces teach, showing a few raises verified val accuracy; if the batch is junk, lift ≈ 0. + Because the val set is fixed and GT-graded, the reward is **stable and comparable + round-over-round** — the property raw keep-rate lacks. Evalchemy grades served vLLM OpenAI + endpoints, so this drops onto AInode-served models with no new infra (pure HTTP, stdlib-only). + +**Where it lives:** `valset.py` (`evaluate`, `valset_lift`, self-check `demo()`); the meta-loop +(`meta.py meta_optimize(..., objective="valset")`) tracks `best_lift`/`best_score`, feeds the +lift trajectory to the prompt-optimizer, and stops at `val_target`. The v2.1 `objective="yield"` +path is the untouched default — nothing regresses. Config keys: `objective`, `val_set`, +`val_shots`, `val_target` (see `core.AutoDataConfig`). CLI: `--meta --objective valset`. + +**Offline-testable:** `valset.demo()`, `meta.demo_valset()`, and pytest cases in +`tests/test_autodata.py` run the whole objective with injected fake clients — no network. Live +endpoints are exercised by the operator via the recipe below. + +## RESULTS (live fleet run — to be recorded by the operator) + +> Placeholder for the first live `--objective valset` run on AInode-served models. Fill in: +> +> | date | domain | weak / strong / judge | val_set size | baseline acc | best lift | best round P (excerpt) | rounds | notes | +> |------|--------|-----------------------|--------------|--------------|-----------|------------------------|--------|-------| +> | _TBD_ | _math_ | _\ / \ / \_ | _n_ | _%_ | _Δ_ | _…_ | _k_ | _vs. v2.1 yield-proxy baseline_ | +> +> Compare against the v2.1 yield-proxy baseline (best ~25% keep-rate) to confirm the objective +> swap raises measured teaching value, not just the proxy number. + ## Non-goals (still) - Re-deriving curation research (we adopt Open Thoughts' findings). - A new eval harness (use Evalchemy). diff --git a/ainode/training/autodata/__init__.py b/ainode/training/autodata/__init__.py index 3705bf6..d551e23 100644 --- a/ainode/training/autodata/__init__.py +++ b/ainode/training/autodata/__init__.py @@ -6,3 +6,4 @@ WEAK solver gets wrong (Δ = I_strong - I_weak == 1). """ from .core import run, AutoDataConfig # noqa: F401 +from .valset import valset_lift, evaluate # noqa: F401 # v2.2 Evalchemy-style val-set objective diff --git a/ainode/training/autodata/core.py b/ainode/training/autodata/core.py index 1d66d09..6ff44fa 100644 --- a/ainode/training/autodata/core.py +++ b/ainode/training/autodata/core.py @@ -41,6 +41,22 @@ class AutoDataConfig: concurrency: int = 8 retries: int = 2 # transient HTTP/parse retries per model call out: str = "" # JSONL output path (optional) + # v2.2 — Evalchemy-style val-set objective (see valset.py). Ignored unless + # objective == "valset" in the meta-loop; the v2.1 Δ=1-yield proxy stays the default. + objective: str = "yield" # meta-loop reward: "yield" (v2.1 proxy) | "valset" (v2.2 lift) + val_set: list = None # held-out labeled probes [{"input","reference"}] for the objective + val_shots: int = 3 # few-shot examples (from kept) used to prime the weak solver on val + val_target: float = 0.10 # meta stop-threshold for the valset objective (absolute lift) + # v2.2 significance guard — a raw lift on a small val set is mostly sampling noise, so the + # meta-loop only ACCEPTS (early-stops) a round when the lift also clears a paired McNemar + # test at `alpha` AND the val set has at least `val_min_n` probes. Below that, the objective + # is reported but never treated as a genuine round-over-round win. + alpha: float = 0.05 # significance level for the paired (McNemar) lift test + val_min_n: int = 12 # minimum val-set size before a lift can be accepted as real + + def __post_init__(self): + if self.val_set is None: + self.val_set = [] @staticmethod def from_dict(d: dict) -> "AutoDataConfig": @@ -51,6 +67,9 @@ def from_dict(d: dict) -> "AutoDataConfig": n_tasks=int(d.get("n_tasks", 50)), system_prompt=d.get("system_prompt", ""), judge_mode=d.get("judge_mode", "rubric"), concurrency=int(d.get("concurrency", 8)), retries=int(d.get("retries", 2)), out=d.get("out", ""), + objective=d.get("objective", "yield"), val_set=list(d.get("val_set") or []), + val_shots=int(d.get("val_shots", 3)), val_target=float(d.get("val_target", 0.10)), + alpha=float(d.get("alpha", 0.05)), val_min_n=int(d.get("val_min_n", 12)), ) diff --git a/ainode/training/autodata/meta.py b/ainode/training/autodata/meta.py index 25e35a1..8dc6f13 100644 --- a/ainode/training/autodata/meta.py +++ b/ainode/training/autodata/meta.py @@ -1,12 +1,20 @@ -"""AutoData v2.1 — meta-optimizer. +"""AutoData v2.1/v2.2 — meta-optimizer. Close the loop: run v1's Δ-filter batch, read the bucket stats, and have an LLM rewrite the generation prompt P_t → P_t+1 toward the zone of proximal development (too_easy ⇒ -harder, too_hard ⇒ easier). Repeat until target yield or max_rounds. Optimizes on the -Δ=1-yield PROXY only — the val-set objective (Evalchemy) is v2.2. +harder, too_hard ⇒ easier). Repeat until the target objective or max_rounds. + +Two reward objectives (`cfg.objective`, or the `objective=` arg): +- "yield" (v2.1, default): the per-batch Δ=1 keep-rate PROXY. Cheap, but noisy/self-graded. +- "valset" (v2.2): the Evalchemy-style held-out-lift objective (see valset.py). The reward is + the weak solver's verified-accuracy LIFT on a fixed labeled val set when primed with a + few-shot sample of the round's kept traces — a stable, GT-graded signal that fixes the noisy + proxy that motivated v2.2. The loop only *accepts* (early-stops on) a round whose lift clears + the target AND passes a paired McNemar significance guard (p ≤ cfg.alpha, n ≥ cfg.val_min_n), + so a noisy flip on a small val set can't declare victory. The v2.1 path is untouched. Reuses v1 (run / call_model / _retry). `core.call_model` is referenced dynamically so the -self-check's fake injection is seen here too. See demo() at the bottom. +self-check's fake injection is seen here too. See demo() / demo_valset() at the bottom. """ from __future__ import annotations @@ -16,26 +24,49 @@ from . import core from .core import AutoDataConfig, run +from .valset import valset_items, valset_lift, evaluate -def _optimize_prompt(ep, rounds: list, task_spec: str, retries: int = 2) -> str: +def _optimize_prompt(ep, rounds: list, task_spec: str, retries: int = 2, + objective: str = "yield") -> str: """Propose the NEXT generator prompt from the FULL round trajectory, so the optimizer learns from an overshoot (too_hard) instead of repeating it. too_easy = both solved - (push harder); too_hard = both failed (dial back).""" - traj = "\n".join( - f"Round {e['round']}: yield={e['yield_pct']}% too_easy={e['too_easy']} " - f"too_hard={e['too_hard']} | prompt: {e['prompt']}" - for e in rounds) + (push harder); too_hard = both failed (dial back). The trajectory is framed around the + active objective (yield proxy vs. held-out val-set lift).""" + if objective == "valset": + traj = "\n".join( + f"Round {e['round']}: lift={e.get('lift')} " + f"(val_acc {e.get('val_acc_baseline')}→{e.get('val_acc_primed')}, " + f"p={e.get('p_value')} significant={e.get('significant')}) " + f"kept={e['kept']} | prompt: {e['prompt']}" + for e in rounds) + goal = ( + "We keep only tasks a STRONG model solves but a WEAK model FAILS (the zone of " + "proximal development), then measure LIFT = the weak model's verified-accuracy " + "gain on a held-out validation set when it is shown a few of your generated " + "examples. Higher lift ⇒ your tasks teach. Trajectory so far:\n" + f"{traj}\n\n" + f"Task domain: {task_spec}\n" + "Propose the NEXT generator prompt to MAXIMIZE lift — LEARN from the trajectory: " + "keep what raised lift; if lift stalled or fell, shift the difficulty/coverage so " + "the kept traces better cover what the weak model still gets wrong on the val set. ") + else: + traj = "\n".join( + f"Round {e['round']}: yield={e['yield_pct']}% too_easy={e['too_easy']} " + f"too_hard={e['too_hard']} | prompt: {e['prompt']}" + for e in rounds) + goal = ( + "We keep only tasks a STRONG model solves but a WEAK model FAILS (the zone of " + "proximal development). too_easy = both solved (too easy); too_hard = both failed " + "(too hard). Trajectory so far:\n" + f"{traj}\n\n" + f"Task domain: {task_spec}\n" + "Propose the NEXT generator prompt to MAXIMIZE yield — LEARN from the trajectory: " + "if a prompt overshot to too_hard, dial difficulty back; if too_easy, push harder; " + "converge toward the middle. ") msg = ( "You tune the SYSTEM PROMPT of a task-generator in a synthetic-data pipeline. " - "We keep only tasks a STRONG model solves but a WEAK model FAILS (the zone of " - "proximal development). too_easy = both solved (too easy); too_hard = both failed " - "(too hard). Trajectory so far:\n" - f"{traj}\n\n" - f"Task domain: {task_spec}\n" - "Propose the NEXT generator prompt to MAXIMIZE yield — LEARN from the trajectory: " - "if a prompt overshot to too_hard, dial difficulty back; if too_easy, push harder; " - "converge toward the middle. " + + goal + 'Return STRICT JSON only: {"prompt": ""}.' ) @@ -56,19 +87,42 @@ def _key(ex: dict) -> str: def meta_optimize(config, target_yield: float = 30, max_rounds: int = 4, - optimizer=None, on_round=None) -> dict: - """Iteratively rewrite P to raise Δ=1 yield. + optimizer=None, on_round=None, objective=None, target=None) -> dict: + """Iteratively rewrite P to raise the reward objective. + + objective: "yield" (v2.1 Δ=1 keep-rate proxy) or "valset" (v2.2 Evalchemy held-out lift). + Defaults to cfg.objective. `target` overrides the stop-threshold (yield%% for "yield", + absolute lift for "valset"; falls back to target_yield / cfg.val_target). - Returns {best_prompt, best_yield, dataset (merged+deduped Δ=1), rounds[], out}. + Returns {best_prompt, best_yield, best_score, best_lift, best_significant, + dataset (merged+deduped Δ=1), rounds[], objective, out}. best_yield stays the best round's + Δ=1 yield%% for both objectives (so existing callers keep working); best_score is the + active-objective value; best_significant flags whether the winning valset round cleared the + paired significance guard (always False for "yield"). """ cfg = config if isinstance(config, AutoDataConfig) else AutoDataConfig.from_dict(config) + objective = objective or getattr(cfg, "objective", "yield") or "yield" + is_valset = objective == "valset" opt_ep = optimizer or cfg.challenger # reuse the challenger endpoint for the rewrite out_path = cfg.out cfg = copy.copy(cfg) cfg.out = "" # we write the merged dataset ourselves + val_items, val_baseline = [], None + if is_valset: + val_items = valset_items(cfg) + if not val_items: + raise ValueError("objective='valset' requires a non-empty val_set in the config") + # weak-alone accuracy on the fixed val set — computed once, reused as the lift baseline. + # Keep the FULL evaluate() dict (its per_item vector) so each round can run the paired + # McNemar significance test against the same fixed baseline. + val_baseline = evaluate(cfg, cfg.weak, val_items) + + target_score = target if target is not None else ( + cfg.val_target if is_valset else target_yield) + rounds, dataset, seen = [], [], set() - best = {"yield": -1.0, "prompt": cfg.gen_prompt} + best = {"score": float("-inf"), "yield": -1.0, "lift": None, "prompt": cfg.gen_prompt} for r in range(1, max_rounds + 1): result = run(cfg) rep = result["report"] @@ -80,16 +134,32 @@ def meta_optimize(config, target_yield: float = 30, max_rounds: int = 4, entry = {"round": r, "yield_pct": rep["yield_pct"], "kept": rep["kept"], "too_easy": rep["too_easy"], "too_hard": rep["too_hard"], "total": rep["total"], "prompt": cfg.gen_prompt} + if is_valset: + # attribute the lift to THIS round's kept traces (few-shot from result["kept"]) + lr = valset_lift(cfg, result["kept"], valset=val_items, baseline=val_baseline) + entry.update(lift=lr["lift"], val_acc_primed=lr["acc_primed"], + val_acc_baseline=lr["acc_baseline"], score=lr["lift"], + p_value=lr["p_value"], improved=lr["improved"], + regressed=lr["regressed"], significant=lr["significant"]) + score = lr["lift"] + else: + entry["score"] = rep["yield_pct"] + score = rep["yield_pct"] rounds.append(entry) - if rep["yield_pct"] > best["yield"]: - best = {"yield": rep["yield_pct"], "prompt": cfg.gen_prompt} + if score > best["score"]: + best = {"score": score, "yield": rep["yield_pct"], "lift": entry.get("lift"), + "prompt": cfg.gen_prompt, "significant": entry.get("significant", False)} if on_round: on_round(entry) - if rep["yield_pct"] >= target_yield or r == max_rounds: + # Accept (early-stop) only on a REAL win. For "valset" a raw lift clearing the target is + # not enough — it must also pass the paired significance guard (see valset.valset_lift), + # so a noisy flip on a tiny val set can't declare victory. "yield" is unchanged. + accepted = score >= target_score and (entry.get("significant", True) if is_valset else True) + if accepted or r == max_rounds: break try: # feed the full trajectory so the optimizer corrects an overshoot, not repeats it - cfg.gen_prompt = _optimize_prompt(opt_ep, rounds, cfg.task_spec, cfg.retries) + cfg.gen_prompt = _optimize_prompt(opt_ep, rounds, cfg.task_spec, cfg.retries, objective) except Exception: break # can't improve P → stop @@ -98,7 +168,9 @@ def meta_optimize(config, target_yield: float = 30, max_rounds: int = 4, for ex in dataset: f.write(json.dumps(ex) + "\n") return {"best_prompt": best["prompt"], "best_yield": best["yield"], - "dataset": dataset, "rounds": rounds, "out": out_path} + "best_score": best["score"], "best_lift": best["lift"], + "best_significant": best.get("significant", False), + "dataset": dataset, "rounds": rounds, "objective": objective, "out": out_path} def demo() -> None: @@ -138,5 +210,56 @@ def fake(ep, messages, json_mode): core.call_model = core._http_chat +def demo_valset() -> None: + """Self-check for the v2.2 val-set objective: a fake world where a higher TEACH level in + the generator prompt makes the kept strong-traces carry a stronger HINT, which in turn + lets the primed weak solver clear more held-out val probes. The meta-loop must optimize + the held-out LIFT (not raw yield) and converge to the higher-TEACH prompt.""" + def fake(ep, messages, json_mode): + sysmsg = messages[0]["content"] if messages and messages[0]["role"] == "system" else "" + last = messages[-1]["content"] + if json_mode and "task-generator" in last: # optimizer: bump TEACH + ts = [int(x) for x in re.findall(r"TEACH=(\d+)", last)] + return json.dumps({"prompt": f"TEACH={(max(ts) if ts else 0) + 1}"}) + if json_mode and "Generate" in last: # challenger: emit tasks at TEACH + t = int((re.search(r"TEACH=(\d+)", sysmsg) or [0, "0"])[1]) + return json.dumps({"tasks": [{"input": f"K{t}_{i}", "reference": "1"} for i in range(4)]}) + # solver on a GENERATED task "K{t}_i": strong emits a good trace carrying HINT=t + # (verifies to 1); weak fails (verifies to 0) -> Δ=1 kept. + m = re.match(r"K(\d+)_", last) + if m: + t = int(m.group(1)) + return f"HINT={t}, answer = 1" if ep.model == "strong" else "answer = 0" + # val probe "L{req}": the primed weak solver clears it iff a HINT>=req is in context + if last and last[0] == "L" and last[1:].isdigit(): + req = int(last[1:]) + hints = [int(tok.split("=", 1)[1].rstrip(",")) + for mm in messages for tok in str(mm.get("content", "")).split() + if tok.startswith("HINT=") and tok.split("=", 1)[1].rstrip(",").isdigit()] + return "answer = 1" if (hints and max(hints) >= req) else "answer = 0" + return "answer = 0" + + core.call_model = fake + try: + cfg = AutoDataConfig( + task_spec="x", gen_prompt="TEACH=0", n_tasks=4, concurrency=1, judge_mode="verify", + objective="valset", val_shots=4, val_target=0.9, + challenger=core.Endpoint("u", "challenger"), weak=core.Endpoint("u", "weak"), + strong=core.Endpoint("u", "strong"), judge=core.Endpoint("u", "judge"), + val_set=[{"input": "L1", "reference": "1"}, {"input": "L2", "reference": "1"}, + {"input": "L3", "reference": "1"}]) + out = meta_optimize(cfg, max_rounds=5) + rounds = out["rounds"] + assert out["objective"] == "valset", out + assert rounds[0]["lift"] == 0.0, rounds # TEACH=0 -> HINT=0 clears no probe + assert rounds[-1]["lift"] > rounds[0]["lift"], rounds # lift rose as TEACH climbed + assert out["best_lift"] > 0 and out["best_score"] == out["best_lift"], out + assert "TEACH=" in out["best_prompt"], out + print("autodata meta valset demo OK:", [(r["round"], r["lift"]) for r in rounds]) + finally: + core.call_model = core._http_chat + + if __name__ == "__main__": demo() + demo_valset() diff --git a/ainode/training/autodata/run.py b/ainode/training/autodata/run.py index cf2bcfa..e74e7a9 100644 --- a/ainode/training/autodata/run.py +++ b/ainode/training/autodata/run.py @@ -2,7 +2,10 @@ Config JSON keys: task_spec, gen_prompt, challenger/weak/strong/judge (each {url, model, max_tokens?, temperature?, api_key?}), n_tasks?, system_prompt?, -judge_mode? (rubric|exact), concurrency?, out?. +judge_mode? (rubric|verify|exact), concurrency?, out?. For the v2.2 meta objective +also: objective? (yield|valset), val_set? ([{input, reference}]), val_shots?, val_target?, +alpha? (McNemar significance level, default 0.05), val_min_n? (min val-set size to accept a +lift as real, default 12). """ import argparse import json @@ -14,8 +17,12 @@ def main() -> None: ap = argparse.ArgumentParser(description="AutoData — Δ-filtered synthetic data generation") ap.add_argument("--config", required=True, help="Path to AutoData config JSON") - ap.add_argument("--meta", action="store_true", help="v2.1 meta-optimizer: rewrite P each round to raise yield") - ap.add_argument("--target-yield", type=float, default=30, help="meta: stop when yield%% reaches this") + ap.add_argument("--meta", action="store_true", help="v2.1/v2.2 meta-optimizer: rewrite P each round") + ap.add_argument("--objective", choices=("yield", "valset"), default=None, + help="meta reward: yield (v2.1 Δ=1 proxy) or valset (v2.2 held-out lift); default from config") + ap.add_argument("--target-yield", type=float, default=30, help="meta/yield: stop when yield%% reaches this") + ap.add_argument("--target", type=float, default=None, + help="meta: objective stop-threshold (yield%% or absolute lift); overrides --target-yield / val_target") ap.add_argument("--max-rounds", type=int, default=4, help="meta: max optimization rounds") args = ap.parse_args() @@ -23,15 +30,33 @@ def main() -> None: if args.meta: from .meta import meta_optimize + objective = args.objective or cfg.get("objective", "yield") + is_valset = objective == "valset" + + def _on_round(e): + if is_valset: + print(f" round {e['round']}: lift={e['lift']} " + f"(val_acc {e['val_acc_baseline']}→{e['val_acc_primed']} " + f"p={e.get('p_value')} significant={e.get('significant')} " + f"kept={e['kept']} yield={e['yield_pct']}%)", file=sys.stderr) + else: + print(f" round {e['round']}: yield={e['yield_pct']}% " + f"(kept={e['kept']} too_easy={e['too_easy']} too_hard={e['too_hard']})", + file=sys.stderr) + out = meta_optimize(cfg, target_yield=args.target_yield, max_rounds=args.max_rounds, - on_round=lambda e: print( - f" round {e['round']}: yield={e['yield_pct']}% " - f"(kept={e['kept']} too_easy={e['too_easy']} too_hard={e['too_hard']})", - file=sys.stderr)) - print(json.dumps({"best_yield": out["best_yield"], "best_prompt": out["best_prompt"], - "rounds": [{k: r[k] for k in ("round", "yield_pct", "kept")} for r in out["rounds"]], + objective=objective, target=args.target, on_round=_on_round) + keys = ("round", "yield_pct", "kept") + ( + ("lift", "p_value", "significant") if is_valset else ()) + print(json.dumps({"objective": out["objective"], "best_score": out["best_score"], + "best_yield": out["best_yield"], "best_lift": out["best_lift"], + "best_significant": out.get("best_significant"), + "best_prompt": out["best_prompt"], + "rounds": [{k: r.get(k) for k in keys} for r in out["rounds"]], "dataset_size": len(out["dataset"]), "out": out["out"]}, indent=2)) - print(f"\nBest yield {out['best_yield']}% over {len(out['rounds'])} rounds; " + summary = (f"Best lift {out['best_lift']} (significant={out.get('best_significant')})" + if is_valset else f"Best yield {out['best_yield']}%") + print(f"\n{summary} over {len(out['rounds'])} rounds; " f"{len(out['dataset'])} examples -> {out['out'] or '(not written)'}", file=sys.stderr) return result = run(cfg, on_progress=lambda r: print( diff --git a/ainode/training/autodata/valset.py b/ainode/training/autodata/valset.py new file mode 100644 index 0000000..3d508be --- /dev/null +++ b/ainode/training/autodata/valset.py @@ -0,0 +1,216 @@ +"""AutoData v2.2 — Evalchemy-style val-set objective for the meta-loop. + +THE HONEST FINDING (2026-06-28 handoff): live yield was noisy/low (best ~25%) and the root +cause is the EVAL SIGNAL, not the optimizer. The v2.1 reward is the per-batch Δ=1 keep-rate +— a self-graded proxy that (a) re-samples fresh tasks every round (so the number wobbles for +reasons unrelated to P) and (b) rides exact-match arithmetic, whose thin zone-of-proximal- +development mislabels correct-but-reformatted answers. Optimizing that proxy chases noise. + +v2.2 replaces the proxy with a **held-out validation objective**, the Open-Thoughts/Evalchemy +move: score P against a FIXED, ground-truth-labeled val set with a trusted verifier, so the +reward is comparable round-over-round. The objective is *measured lift*: + + lift(P) = acc(weak | few-shot of D(P)) − acc(weak alone) + +on the val set, where D(P) is the batch of Δ=1 "teacher" traces the loop just kept. This is +in-context learning used as a **torch-free proxy for fine-tuning lift** — if the kept traces +actually teach, showing a few to the weak model raises its verified val accuracy; if the batch +is junk, lift is ~0. Because the val set is fixed and GT-labeled and graded by the same trusted +verifier the Δ-filter uses (`core.judge_correct` → verify.py value-verify + rubric fallback), +the signal is stable and directly optimizable, unlike raw keep-rate. Evalchemy grades served +vLLM OpenAI endpoints, so this drops onto AInode-served models with no new infra. + +SIGNIFICANCE GUARD: a raw lift is a difference of two proportions on a finite val set, so on a +small set it is dominated by sampling noise — one probe flipping right on n=3 already moves lift +by 0.33, trivially clearing any modest target. Because baseline and primed accuracy are measured +on the SAME fixed items, the two runs are *paired*, so we grade the lift with an exact one-sided +McNemar (sign) test on the discordant probes (`mcnemar_pvalue`): b = probes that went wrong→right +when primed, c = right→wrong; under H0 b ~ Binomial(b+c, 0.5). `valset_lift` reports that p-value +and a `significant` flag (p ≤ cfg.alpha AND n ≥ cfg.val_min_n AND lift > 0). The meta-loop only +*accepts* a round (early-stops) on a `significant` lift, so it can no longer declare victory on +noise — the objective is stable AND its wins are statistically real, not just the max of an +order-statistic over rounds. + +Pure HTTP + stdlib, like the rest of the package: `core.call_model` is referenced dynamically +so the self-check's fake injection is seen here too. See demo() at the bottom. +""" +from __future__ import annotations + +import math +from concurrent.futures import ThreadPoolExecutor + +from . import core +from .core import AutoDataConfig + + +def valset_items(cfg: AutoDataConfig) -> list: + """The held-out probes: {"input","reference"?} dicts with a non-empty input.""" + return [v for v in (cfg.val_set or []) if isinstance(v, dict) and v.get("input")] + + +def _shots_to_messages(shots) -> list: + """Flatten kept ShareGPT examples into alternating user/assistant few-shot turns. + Each kept example is {"conversations":[{"from":"human"...},{"from":"gpt"...}]}.""" + msgs = [] + for ex in shots or []: + human = gpt = None + for m in ex.get("conversations", []): + if m.get("from") == "human": + human = m.get("value") + elif m.get("from") == "gpt": + gpt = m.get("value") + if human is not None and gpt is not None: + msgs.append({"role": "user", "content": human}) + msgs.append({"role": "assistant", "content": gpt}) + return msgs + + +def _solve_primed(cfg: AutoDataConfig, ep, x: str, shot_msgs: list) -> str: + """Solve `x` with `ep`, optionally primed by pre-built few-shot turns.""" + msgs = ([{"role": "system", "content": cfg.system_prompt}] if cfg.system_prompt else []) + msgs += shot_msgs + msgs += [{"role": "user", "content": x}] + return core._retry(lambda: core.call_model(ep, msgs, False), cfg.retries) + + +def evaluate(cfg: AutoDataConfig, ep, valset=None, shots=None) -> dict: + """Verified accuracy of `ep` on the val set, optionally primed with `shots` (kept + ShareGPT examples as few-shot context). Grading reuses `core.judge_correct`, so the + val objective and the Δ-filter share the exact same trusted verifier. + + Returns {acc, correct, n, per_item} where `per_item` is the 0/1 correctness of each + probe in val-set order — the paired vector the significance test consumes.""" + items = valset if valset is not None else valset_items(cfg) + n = len(items) + if n == 0: + return {"acc": 0.0, "correct": 0, "n": 0, "per_item": []} + shot_msgs = _shots_to_messages(shots) + + def _one(item): + x, ref = item["input"], item.get("reference") + try: + out = _solve_primed(cfg, ep, x, shot_msgs) + except Exception: + return 0 + return int(bool(core.judge_correct(cfg, x, out, ref))) + + workers = max(1, min(cfg.concurrency, n)) + with ThreadPoolExecutor(max_workers=workers) as ex: + per_item = list(ex.map(_one, items)) # ex.map preserves item order -> paired vector + correct = sum(per_item) + return {"acc": round(correct / n, 4), "correct": int(correct), "n": n, "per_item": per_item} + + +def mcnemar_pvalue(baseline: list, primed: list) -> tuple: + """Exact one-sided McNemar (sign) test that `primed` beats `baseline` on the SAME paired + probes. `baseline`/`primed` are 0/1 vectors aligned by probe. b = probes that improved + (wrong→right when primed), c = probes that regressed (right→wrong). Under H0 (priming has + no effect) the discordant outcomes split 50/50, so b ~ Binomial(b+c, 0.5); the one-sided + p-value is P(X ≥ b). No discordant pairs ⇒ no evidence of change ⇒ p=1.0. Stdlib only. + Returns (p_value, b, c).""" + b = sum(1 for base, prm in zip(baseline, primed) if not base and prm) + c = sum(1 for base, prm in zip(baseline, primed) if base and not prm) + disc = b + c + if disc == 0: + return 1.0, b, c + tail = sum(math.comb(disc, i) for i in range(b, disc + 1)) / (2 ** disc) + return round(min(1.0, tail), 4), b, c + + +def valset_lift(cfg: AutoDataConfig, kept, valset=None, baseline=None, shots=None) -> dict: + """Evalchemy-style objective: the weak solver's verified-accuracy LIFT on the held-out + val set when primed with a few-shot sample of `kept` (the round's Δ=1 teacher traces), + vs the weak baseline. Higher lift ⇒ the batch teaches ⇒ a better generator prompt P. + + `baseline` may be a full `evaluate()` dict (carries `per_item`, so the paired significance + test can run — the meta-loop passes this and computes it once, reused across rounds), a bare + weak-alone acc float (back-compat), or None (evaluated fresh here). + + Returns {lift, acc_primed, acc_baseline, n, shots, p_value, improved, regressed, significant}. + `p_value` is the exact one-sided McNemar p for "primed beats baseline" (None when no paired + baseline vector is available); `significant` is the guard the meta-loop early-stops on — + a real, non-noise win: p ≤ cfg.alpha AND n ≥ cfg.val_min_n AND lift > 0. + """ + items = valset if valset is not None else valset_items(cfg) + if not items: + return {"lift": 0.0, "acc_primed": 0.0, "acc_baseline": 0.0, "n": 0, "shots": 0, + "p_value": None, "improved": 0, "regressed": 0, "significant": False} + k = cfg.val_shots if shots is None else shots + sample = list(kept or [])[:k] + if isinstance(baseline, dict): # full evaluate() dict -> enables McNemar + base_eval = baseline + elif baseline is None: + base_eval = evaluate(cfg, cfg.weak, items) + else: # bare acc float (back-compat) -> no per_item + base_eval = {"acc": float(baseline), "per_item": None} + base_acc = base_eval["acc"] + primed_eval = evaluate(cfg, cfg.weak, items, shots=sample) + primed = primed_eval["acc"] + lift = round(primed - base_acc, 4) + + base_items, primed_items = base_eval.get("per_item"), primed_eval.get("per_item") + if base_items and primed_items and len(base_items) == len(primed_items): + p_value, improved, regressed = mcnemar_pvalue(base_items, primed_items) + else: # no paired baseline -> can't confirm significance + p_value, improved, regressed = None, 0, 0 + n = len(items) + significant = (p_value is not None and p_value <= cfg.alpha + and n >= cfg.val_min_n and lift > 0) + return {"lift": lift, "acc_primed": primed, "acc_baseline": round(base_acc, 4), + "n": n, "shots": len(sample), "p_value": p_value, + "improved": improved, "regressed": regressed, "significant": significant} + + +def demo() -> None: + """Self-check with a fake model: the kept traces carry a hint that lets the weak solver + answer the held-out val probes it fails cold — so priming lifts verified val accuracy.""" + import json # noqa: F401 — parity with sibling demos; kept explicit + + def fake(ep, messages, json_mode): + last = messages[-1]["content"] + # val probe "L{req}" needs a hint of level >= req to be solved by the weak model. + # scan the few-shot context (prior assistant turns) for the strongest HINT=k available. + req = None + if last and last[0] == "L" and last[1:].isdigit(): + req = int(last[1:]) + if req is not None: + hints = [] + for m in messages: + for tok in str(m.get("content", "")).split(): + if tok.startswith("HINT="): + try: + hints.append(int(tok.split("=", 1)[1].rstrip(","))) + except ValueError: + pass + return "answer = 1" if (hints and max(hints) >= req) else "answer = 0" + return "answer = 0" + + core.call_model = fake + try: + cfg = AutoDataConfig( + task_spec="x", gen_prompt="x", n_tasks=1, concurrency=1, judge_mode="verify", + challenger=core.Endpoint("u", "challenger"), weak=core.Endpoint("u", "weak"), + strong=core.Endpoint("u", "strong"), judge=core.Endpoint("u", "judge"), + val_set=[{"input": "L1", "reference": "1"}, {"input": "L2", "reference": "1"}], + val_shots=2) + # cold baseline: weak has no hint -> both probes wrong + base = evaluate(cfg, cfg.weak, valset_items(cfg)) + assert base["acc"] == 0.0, base + # a teaching batch: two kept traces carrying HINT=2 -> both probes solved when primed + kept = [{"conversations": [{"from": "human", "value": "K"}, + {"from": "gpt", "value": "HINT=2, answer = 1"}]}] * 2 + res = valset_lift(cfg, kept, baseline=base["acc"]) + assert res["acc_baseline"] == 0.0 and res["acc_primed"] == 1.0, res + assert res["lift"] == 1.0 and res["shots"] == 2, res + # a junk batch (no usable hint) teaches nothing -> zero lift + junk = [{"conversations": [{"from": "human", "value": "K"}, + {"from": "gpt", "value": "HINT=0, answer = 1"}]}] * 2 + res0 = valset_lift(cfg, junk, baseline=base["acc"]) + assert res0["lift"] == 0.0, res0 + print("autodata valset demo OK: teaching lift", res["lift"], "vs junk", res0["lift"]) + finally: + core.call_model = core._http_chat + + +if __name__ == "__main__": + demo() diff --git a/tests/test_autodata.py b/tests/test_autodata.py index 526f0dc..d3934ed 100644 --- a/tests/test_autodata.py +++ b/tests/test_autodata.py @@ -13,8 +13,9 @@ from aiohttp import web from aiohttp.test_utils import TestClient, TestServer -from ainode.training.autodata import core, meta +from ainode.training.autodata import core, meta, valset from ainode.training.autodata.core import AutoDataConfig, Endpoint, run +from ainode.training.autodata.valset import evaluate, valset_items, valset_lift from ainode.training.autodata.verify import is_correct from ainode.training.engine import TrainingManager from ainode.training.api_routes import setup_training_routes @@ -106,6 +107,194 @@ def test_meta_demo(): meta.demo() # asserts internally: yield rises round-over-round to the harder prompt +# --- v2.2: Evalchemy-style val-set objective ------------------------------------------- + +def test_valset_demo(): + valset.demo() # asserts internally: teaching traces lift primed val-accuracy, junk doesn't + + +def test_meta_valset_demo(): + meta.demo_valset() # asserts internally: the meta-loop optimizes held-out lift, not yield + + +def _valset_cfg(**over): + base = dict(task_spec="x", gen_prompt="x", n_tasks=1, concurrency=1, judge_mode="verify", + challenger=Endpoint("u", "challenger"), weak=Endpoint("u", "weak"), + strong=Endpoint("u", "strong"), judge=Endpoint("u", "judge"), + val_set=[{"input": "L1", "reference": "1"}, {"input": "L2", "reference": "1"}], + val_shots=2) + base.update(over) + return AutoDataConfig(**base) + + +def _fake_hint(ep, messages, json_mode): + """weak val solver: clears probe L{req} iff a few-shot HINT>=req is in context.""" + last = messages[-1]["content"] + if last and last[0] == "L" and last[1:].isdigit(): + req = int(last[1:]) + hints = [int(t.split("=", 1)[1].rstrip(",")) + for m in messages for t in str(m.get("content", "")).split() + if t.startswith("HINT=") and t.split("=", 1)[1].rstrip(",").isdigit()] + return "answer = 1" if (hints and max(hints) >= req) else "answer = 0" + return "answer = 0" + + +def test_valset_evaluate_is_gt_graded(): + """Verified accuracy uses the trusted verifier (verify-mode), not substring — the weak + solver clears both probes only when the teaching hint is in the few-shot context.""" + core.call_model = _fake_hint + try: + cfg = _valset_cfg() + items = valset_items(cfg) + assert evaluate(cfg, cfg.weak, items)["acc"] == 0.0 # cold: no hint -> 0/2 + shots = [{"conversations": [{"from": "human", "value": "K"}, + {"from": "gpt", "value": "HINT=2, answer = 1"}]}] * 2 + assert evaluate(cfg, cfg.weak, items, shots=shots)["acc"] == 1.0 # primed -> 2/2 + finally: + core.call_model = core._http_chat + + +def test_valset_lift_teaching_vs_junk(): + """The objective separates a teaching batch (positive lift) from a junk batch (zero lift).""" + core.call_model = _fake_hint + try: + cfg = _valset_cfg() + teach = [{"conversations": [{"from": "human", "value": "K"}, + {"from": "gpt", "value": "HINT=2, answer = 1"}]}] * 2 + junk = [{"conversations": [{"from": "human", "value": "K"}, + {"from": "gpt", "value": "HINT=0, answer = 1"}]}] * 2 + assert valset_lift(cfg, teach)["lift"] == 1.0 + assert valset_lift(cfg, junk)["lift"] == 0.0 + # empty val_set -> objective degrades to a zero, never raises + assert valset_lift(_valset_cfg(val_set=[]), teach)["lift"] == 0.0 + finally: + core.call_model = core._http_chat + + +def test_meta_valset_requires_val_set(): + """objective='valset' with no probes is a config error, surfaced clearly (not silent).""" + with pytest.raises(ValueError): + meta.meta_optimize(_valset_cfg(val_set=[], objective="valset"), max_rounds=1) + + +# --- v2.2 significance guard: a raw lift on a small val set is sampling noise ----------- + +def test_mcnemar_pvalue_exact(): + """Exact one-sided McNemar (sign) test on paired 0/1 probe vectors, stdlib-only.""" + from ainode.training.autodata.valset import mcnemar_pvalue + assert mcnemar_pvalue([0, 0, 0, 0, 0], [1, 1, 1, 1, 1]) == (0.0312, 5, 0) # 0.5**5 + assert mcnemar_pvalue([1, 0, 1], [1, 0, 1]) == (1.0, 0, 0) # no discordant pairs + assert mcnemar_pvalue([0, 1], [1, 0]) == (0.75, 1, 1) # 1 up, 1 down + assert mcnemar_pvalue([0, 0, 0], [1, 1, 1]) == (0.125, 3, 0) # n=3 all-flip != sig + + +def test_valset_lift_reports_significance_but_small_n_is_never_significant(): + """A full teaching lift on a 2-probe val set is real-looking (lift=1.0) but the paired test + on n < val_min_n can't call it significant — the guard the meta-loop reads. A bare-float + baseline carries no paired vector, so significance is conservatively withheld (None).""" + core.call_model = _fake_hint + try: + cfg = _valset_cfg() # 2-probe val set, val_shots=2 + teach = [{"conversations": [{"from": "human", "value": "K"}, + {"from": "gpt", "value": "HINT=2, answer = 1"}]}] * 2 + base = evaluate(cfg, cfg.weak, valset_items(cfg)) # full dict -> paired per_item + lr = valset_lift(cfg, teach, baseline=base) + assert lr["lift"] == 1.0 and lr["improved"] == 2 and lr["regressed"] == 0 + assert lr["p_value"] is not None and lr["significant"] is False # n=2 < val_min_n + lr_float = valset_lift(cfg, teach, baseline=base["acc"]) # bare float -> no per_item + assert lr_float["p_value"] is None and lr_float["significant"] is False + finally: + core.call_model = core._http_chat + + +def _fake_teach_all(ep, messages, json_mode): + """Every round teaches maximally: the strong solver emits HINT=9 (clears any L-probe) and + the weak solver fails cold, so baseline acc=0 and primed acc=1.0 — every val probe is a + wrong→right improvement. Lets a test dial n up/down and watch the significance guard flip.""" + last = messages[-1]["content"] + if json_mode and "task-generator" in last: # optimizer: no-op rewrite + return json.dumps({"prompt": "x"}) + if json_mode and "Generate" in last: # challenger: 4 Δ=1-able tasks + return json.dumps({"tasks": [{"input": f"K{i}", "reference": "1"} for i in range(4)]}) + if json_mode and "Candidate answer" in last: # rubric judge (unused for numerics) + return json.dumps({"correct": True}) + if last and last[0] == "K": # solver on a generated task + return "HINT=9, answer = 1" if ep.model == "strong" else "answer = 0" + if last and last[0] == "L" and last[1:].isdigit(): # weak val solver, primed via HINT + req = int(last[1:]) + hints = [int(t.split("=", 1)[1].rstrip(",")) + for m in messages for t in str(m.get("content", "")).split() + if t.startswith("HINT=") and t.split("=", 1)[1].rstrip(",").isdigit()] + return "answer = 1" if (hints and max(hints) >= req) else "answer = 0" + return "answer = 0" + + +def test_meta_valset_significance_blocks_noise(): + """The finding's repro: a 3-probe val set with a trivially-cleared target (0.1). Round 1's + lift hits 1.0, but n < val_min_n and the paired McNemar p (0.125) can't clear alpha, so the + loop must NOT early-stop — it keeps running instead of declaring victory on sampling noise.""" + core.call_model = _fake_teach_all + try: + cfg = _valset_cfg(objective="valset", n_tasks=4, val_shots=4, val_target=0.1, + val_set=[{"input": "L1", "reference": "1"}] * 3) + out = meta.meta_optimize(cfg, max_rounds=3) + assert len(out["rounds"]) == 3, out # ran every round; no early victory + r0 = out["rounds"][0] + assert r0["lift"] >= cfg.val_target # target trivially cleared... + assert r0["p_value"] is not None and r0["significant"] is False # ...but not significant + assert out["best_significant"] is False + finally: + core.call_model = core._http_chat + + +def test_meta_valset_significant_lift_stops(): + """The mirror: with a 20-probe val set, the same all-probes improvement clears McNemar + (p≈0, n≥val_min_n), so the loop accepts the round and early-stops — the objective still + WORKS when the signal is genuinely real, it just no longer trusts noise.""" + core.call_model = _fake_teach_all + try: + cfg = _valset_cfg(objective="valset", n_tasks=4, val_shots=4, val_target=0.1, + val_set=[{"input": "L1", "reference": "1"}] * 20) + out = meta.meta_optimize(cfg, max_rounds=3) + assert len(out["rounds"]) == 1, out # accepted -> stopped after round 1 + r0 = out["rounds"][0] + assert r0["significant"] is True and r0["p_value"] <= cfg.alpha + assert out["best_significant"] is True + finally: + core.call_model = core._http_chat + + +def test_meta_yield_path_unchanged_by_v22(): + """The v2.1 yield objective still returns its keys and optimizes yield — no regression.""" + def fake(ep, messages, json_mode): + sysmsg = messages[0]["content"] if messages and messages[0]["role"] == "system" else "" + last = messages[-1]["content"] + if json_mode and "task-generator" in last: + ds = [int(x) for x in __import__("re").findall(r"DIFF=(\d+)", last)] + return json.dumps({"prompt": f"DIFF={(max(ds) if ds else 0) + 1}"}) + if json_mode and "Generate" in last: + d = int((__import__("re").search(r"DIFF=(\d+)", sysmsg) or [0, "0"])[1]) + return json.dumps({"tasks": [{"input": f"L{d}_{i}", "reference": None} for i in range(6)]}) + if json_mode and "Candidate answer" in last: + return json.dumps({"correct": "CORRECT" in last}) + level = int((__import__("re").search(r"L(\d+)_", last) or [0, "0"])[1]) + ability = 0 if ep.model == "weak" else 2 + return "CORRECT" if level <= ability else "WRONG" + + core.call_model = fake + try: + cfg = AutoDataConfig( + task_spec="x", gen_prompt="DIFF=0", n_tasks=6, concurrency=1, + challenger=Endpoint("u", "challenger"), weak=Endpoint("u", "weak"), + strong=Endpoint("u", "strong"), judge=Endpoint("u", "judge")) + out = meta.meta_optimize(cfg, target_yield=30, max_rounds=4) + assert out["objective"] == "yield" + assert out["best_score"] == out["best_yield"] and out["best_lift"] is None + assert out["rounds"][-1]["yield_pct"] > out["rounds"][0]["yield_pct"] + finally: + core.call_model = core._http_chat + + def _fake(ep, messages, json_mode): """3 tasks: A (only strong solves -> kept), B (both solve), C (neither).""" last = messages[-1]["content"]