From c721ecf145f3886e413da3798d07ab7b331cff41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 07:38:26 +0000 Subject: [PATCH 1/3] SLM-267: bounded uniform-vs-coverage-targeted ProgramSpec sampling wiring evidence VSD2-02 asks for 10k/100k/1M-record ProgramSpec corpora comparing uniform against coverage-targeted sampling. The existing coverage-guided generator (SLM-5) samples from a fixed, exhaustible candidate grid, so those rungs are out of reach without a materially different generator. This scopes the issue to what the generator actually supports: a genuine uniform-random control arm (ProgramGenerator.generate_uniform, new), a deterministic (global_seed, shard_id, worker_id) sharding primitive, and an honest, reproducible measurement of the generator's own coverage-targeted-vs-uniform budget efficiency and state-space saturation point. - generate.py: refactor generate_one into _materialize + add generate_uniform (uniform-random candidate selection, no coverage bias) as a genuine control arm distinct from the existing greedy CoverageTracker.score() selection. - New harness (slm267_programspec_coverage_scaling.py) + CLI (generate_programspec_corpus.py) run both policies to a bounded budget, dedup by canonical-root fingerprint (in-memory), and report exhaustion point, unique roots, and programs-to-full-coverage per arm. - Fixture campaign (80 programs/shard x2 shards): coverage-targeted reaches full coverable-cell coverage in 19 programs vs uniform's 34-35; both exhaust the 3-component grid at 36 accepted (28 unique roots, 8 duplicate canonical roots via serialization collapse). - Disposition: inconclusive (wiring only) -- does not validate VSD-H7a/b/c, which require corpora/training several orders of magnitude larger than this fixture budget. Documented as explicit follow-up. Closes SLM-267. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01MAH7ibENsNM41tPTnVJkbg --- ...programspec-coverage-scaling-20260725.json | 99 ++++++ ...7-programspec-coverage-scaling-20260725.md | 44 +++ docs/design/programspec-coverage-scaling.md | 65 ++++ scripts/generate_programspec_corpus.py | 141 ++++++++ src/slm_training/data/progspec/generate.py | 25 +- .../slm267_programspec_coverage_scaling.py | 310 ++++++++++++++++++ src/slm_training/resources/versions.json | 27 +- ...est_slm267_programspec_coverage_scaling.py | 104 ++++++ .../test_generate_programspec_corpus.py | 45 +++ 9 files changed, 858 insertions(+), 2 deletions(-) create mode 100644 docs/design/iter-slm267-programspec-coverage-scaling-20260725.json create mode 100644 docs/design/iter-slm267-programspec-coverage-scaling-20260725.md create mode 100644 docs/design/programspec-coverage-scaling.md create mode 100644 scripts/generate_programspec_corpus.py create mode 100644 src/slm_training/harnesses/experiments/slm267_programspec_coverage_scaling.py create mode 100644 tests/test_harnesses/experiments/test_slm267_programspec_coverage_scaling.py create mode 100644 tests/test_scripts/test_generate_programspec_corpus.py diff --git a/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json b/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json new file mode 100644 index 000000000..5a54ba077 --- /dev/null +++ b/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json @@ -0,0 +1,99 @@ +{ + "arms": { + "coverage_targeted": [ + { + "accepted": 36, + "cells_covered": 58, + "cells_total": 64, + "duplicate_roots": 8, + "exhausted_at": 36, + "policy": "coverage_targeted", + "programs_to_full_coverage": 19, + "proposed": 37, + "requested_count": 80, + "seed": 705206174, + "unique_canonical_roots": 28 + }, + { + "accepted": 36, + "cells_covered": 58, + "cells_total": 64, + "duplicate_roots": 8, + "exhausted_at": 36, + "policy": "coverage_targeted", + "programs_to_full_coverage": 19, + "proposed": 37, + "requested_count": 80, + "seed": 3027814441, + "unique_canonical_roots": 28 + } + ], + "uniform": [ + { + "accepted": 36, + "cells_covered": 58, + "cells_total": 64, + "duplicate_roots": 8, + "exhausted_at": 36, + "policy": "uniform", + "programs_to_full_coverage": 34, + "proposed": 37, + "requested_count": 80, + "seed": 705206174, + "unique_canonical_roots": 28 + }, + { + "accepted": 36, + "cells_covered": 58, + "cells_total": 64, + "duplicate_roots": 8, + "exhausted_at": 36, + "policy": "uniform", + "programs_to_full_coverage": 35, + "proposed": 37, + "requested_count": 80, + "seed": 3027814441, + "unique_canonical_roots": 28 + } + ] + }, + "claim_class": "wiring", + "comparison": { + "coverage_targeted_programs_to_full_coverage": 19, + "coverage_targeted_reaches_full_coverage_no_slower": true, + "uniform_programs_to_full_coverage": 34 + }, + "experiment_id": "slm267-programspec-coverage-scaling", + "limitations": [ + "Fixture-scale only: target_count and the candidate grid are far below the 10k/100k/1M rungs VSD-H7a-c require; no claim is made about those hypotheses at this scale.", + "The 'coverage_targeted' arm reuses the generator's existing greedy CoverageTracker.score() bias toward uncovered cells; it does not yet consume an external CoverageGapManifestV1 (SLM-265) gap manifest \u2014 that mapping is explicitly future work.", + "Canonical-root dedup here is in-memory only; a disk-backed exact index for 100k/1M-scale generation is not implemented.", + "No training/model comparison is run and no corpus is published to the DataStore; this measures the generator mechanism only." + ], + "manifest_hash": "df1aa924dbbfb80c484ca59ffcfa235336ae41dbc931d67af52b6371c515d6f7", + "recipe": { + "components": [ + "TextContent", + "Button", + "Separator" + ], + "max_depth": 3, + "max_width": 3, + "seed": 0, + "shards": 2, + "target_count": 80 + }, + "schema": "programspec_coverage_scaling_report/v1", + "status": "inconclusive", + "timestamp": "2026-07-25T07:37:15.970003Z", + "version_stamp": { + "code_commit": "8dd4794a7cea10f1fe9732e5bb25d198b810fc22", + "code_dirty": true, + "components": { + "harness.experiments": "v105", + "harness.experiments.slm267_programspec_coverage_scaling": "v1" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T07:37:15.970040+00:00" + } +} diff --git a/docs/design/iter-slm267-programspec-coverage-scaling-20260725.md b/docs/design/iter-slm267-programspec-coverage-scaling-20260725.md new file mode 100644 index 000000000..d4f5aadcf --- /dev/null +++ b/docs/design/iter-slm267-programspec-coverage-scaling-20260725.md @@ -0,0 +1,44 @@ +# SLM-267 (VSD2-02): ProgramSpec coverage-scaling wiring evidence + +**Claim class:** wiring / fixture only + +**Status:** `inconclusive` + +## Recipe + +```json +{ + "target_count": 80, + "seed": 0, + "shards": 2, + "components": [ + "TextContent", + "Button", + "Separator" + ], + "max_depth": 3, + "max_width": 3 +} +``` + +## Arms + +| policy | shard | seed | proposed | accepted | duplicate_roots | unique_roots | exhausted_at | cells_covered/total | programs_to_full_coverage | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| uniform | 0 | 705206174 | 37 | 36 | 8 | 28 | 36 | 58/64 | 34 | +| uniform | 1 | 3027814441 | 37 | 36 | 8 | 28 | 36 | 58/64 | 35 | +| coverage_targeted | 0 | 705206174 | 37 | 36 | 8 | 28 | 36 | 58/64 | 19 | +| coverage_targeted | 1 | 3027814441 | 37 | 36 | 8 | 28 | 36 | 58/64 | 19 | + +## Comparison + +- uniform programs to full coverage: 34 +- coverage-targeted programs to full coverage: 19 +- coverage-targeted reaches full coverage no slower than uniform: True + +## Limitations + +- Fixture-scale only: target_count and the candidate grid are far below the 10k/100k/1M rungs VSD-H7a-c require; no claim is made about those hypotheses at this scale. +- The 'coverage_targeted' arm reuses the generator's existing greedy CoverageTracker.score() bias toward uncovered cells; it does not yet consume an external CoverageGapManifestV1 (SLM-265) gap manifest — that mapping is explicitly future work. +- Canonical-root dedup here is in-memory only; a disk-backed exact index for 100k/1M-scale generation is not implemented. +- No training/model comparison is run and no corpus is published to the DataStore; this measures the generator mechanism only. diff --git a/docs/design/programspec-coverage-scaling.md b/docs/design/programspec-coverage-scaling.md new file mode 100644 index 000000000..7841f1839 --- /dev/null +++ b/docs/design/programspec-coverage-scaling.md @@ -0,0 +1,65 @@ +# ProgramSpec coverage-scaling wiring evidence + +SLM-267 (VSD2-02) asks for uniform-vs-coverage-targeted 10k/100k/1M ProgramSpec +corpora. The repository's existing coverage-guided generator (SLM-5, +`slm_training.data.progspec.generate.ProgramGenerator`) samples from a fixed, +exhaustible candidate grid — a few dozen root/topology/prop combinations per +small component set, not an open-ended program space — so those rungs are out +of reach without a materially different generator. This issue is scoped down +to what the existing generator actually supports: a genuine uniform-random +control arm, a deterministic `(global_seed, shard_id, worker_id)` sharding +primitive, and an honest measurement of the generator's own state-space +saturation. + +The committed [disposition JSON](iter-slm267-programspec-coverage-scaling-20260725.json) +records the exact recipe, per-arm counters, and comparison. Rerun the exact +command to regenerate it: + +```bash +python -m scripts.generate_programspec_corpus --mode fixture --target-count 80 --seed 0 --shards 2 +``` + +## Measured fixture-scale campaign — 2026-07-25 + +Two shards of a 3-component, depth/width-3 candidate grid (64 target coverage +cells, 58 coverable) were sampled under two policies up to an 80-program +budget per shard: + +- **`uniform`** — `ProgramGenerator.generate_uniform`, a new method added in + this change that draws uniformly at random from the unused candidate grid + (no coverage bias). +- **`coverage_targeted`** — the generator's existing `generate_one`, which + greedily maximizes `CoverageTracker.score()` (always prefers a candidate + touching an uncovered cell). + +Both policies exhausted the finite candidate grid at 36 accepted programs per +shard (8 duplicate canonical roots via serialization collapse, 28 unique +roots) — the grid's exact saturation point at this component/depth/width +setting. `coverage_targeted` reached full coverable-cell coverage in 19 +programs; `uniform` took 34–35. This reproduces deterministically across +shards and reruns (see `shard_seed` and the manifest hash in the disposition +JSON). + +## Honest scope + +This is wiring evidence only: + +- It does **not** validate VSD-H7a (10k/100k/1M reachable unique roots), + VSD-H7b (coverage-targeted beats uniform on strict semantic outcomes at + matched exposure), or VSD-H7c (scale moves semantics) — those require + corpora and trained models several orders of magnitude larger than the + fixture budget used here, and no model training occurs in this issue. +- `coverage_targeted` reuses the generator's internal coverage bias; it does + not yet consume an external `CoverageGapManifestV1` (SLM-265) gap manifest. + Wiring that mapping honestly (rather than inventing an unverified feature + correspondence between SLM-265's record-level domain-shift cells and this + generator's grammar-coverage cells) is left as explicit follow-up work. +- Canonical-root dedup here is in-memory only; a disk-backed exact index + required at 100k/1M scale is not implemented. +- No corpus is published to the `DataStore`, and no ship gate, checkpoint, or + production default is touched. + +Disposition: **`inconclusive`** — the mechanism (uniform control, deterministic +sharding, greedy coverage-targeting, in-memory canonical dedup) is real, +tested, and reproducible, but the scale required to resolve VSD-H7a-c remains +unaddressed. diff --git a/scripts/generate_programspec_corpus.py b/scripts/generate_programspec_corpus.py new file mode 100644 index 000000000..c5112c4a8 --- /dev/null +++ b/scripts/generate_programspec_corpus.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Run the SLM-267 (VSD2-02) ProgramSpec coverage-scaling wiring fixture. + +Example: + python -m scripts.generate_programspec_corpus --mode plan-only + python -m scripts.generate_programspec_corpus --mode fixture +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from slm_training.harnesses.experiments.slm267_programspec_coverage_scaling import ( + DEFAULT_CONFIG, + EXPERIMENT_ID, + render_markdown, + run_coverage_scaling_campaign, +) +from slm_training.versioning import build_version_stamp + +_DESIGN_JSON = "docs/design/iter-slm267-programspec-coverage-scaling-20260725.json" +_DESIGN_MD = "docs/design/iter-slm267-programspec-coverage-scaling-20260725.md" + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _today_yyyymmdd() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%d") + + +def _build_payload( + mode: str, target_count: int, seed: int, shards: int +) -> tuple[dict[str, Any], str]: + command = ( + f"python -m scripts.generate_programspec_corpus --mode {mode} " + f"--target-count {target_count} --seed {seed} --shards {shards}" + ) + if mode == "plan-only": + payload: dict[str, Any] = { + "schema": "Slm267ProgramspecCoverageScalingManifestV1", + "experiment_id": EXPERIMENT_ID, + "status": "plan_only", + "claim_class": "wiring", + "recipe": { + "target_count": target_count, + "seed": seed, + "shards": shards, + "components": list(DEFAULT_CONFIG.components or ()), + }, + "version_stamp": build_version_stamp( + "harness.experiments", + "harness.experiments.slm267_programspec_coverage_scaling", + ), + "timestamp": _now(), + } + return payload, command + payload = run_coverage_scaling_campaign( + target_count=target_count, seed=seed, shards=shards + ) + return payload, command + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="SLM-267 VSD2-02 ProgramSpec coverage-scaling fixture", + exit_on_error=False, + ) + parser.add_argument( + "--mode", + choices={"plan-only", "fixture"}, + default="plan-only", + help="Run mode: plan-only writes the manifest; fixture runs the CPU campaign.", + ) + parser.add_argument( + "--output-dir", + type=Path, + help=( + "Directory for run artifacts (default: " + "outputs/runs/slm267-programspec-coverage-scaling-)" + ), + ) + parser.add_argument( + "--target-count", + type=int, + default=80, + help="Bounded per-arm program budget (default: 80; fixture scale only).", + ) + parser.add_argument("--seed", type=int, default=0, help="Global seed (default: 0).") + parser.add_argument( + "--shards", type=int, default=2, help="Deterministic shard count (default: 2)." + ) + try: + args = parser.parse_args(argv) + except (argparse.ArgumentError, SystemExit): + return 2 + + output_dir = args.output_dir or Path( + f"outputs/runs/slm267-programspec-coverage-scaling-{_today_yyyymmdd()}" + ) + output_dir.mkdir(parents=True, exist_ok=True) + + payload, command = _build_payload( + args.mode, args.target_count, args.seed, args.shards + ) + payload["schema"] = payload.get("schema", "Slm267ProgramspecCoverageScalingReportV1") + payload["claim_class"] = "wiring" + payload["status"] = payload.get("status", "inconclusive") + payload["timestamp"] = _now() + if "version_stamp" not in payload: + payload["version_stamp"] = build_version_stamp( + "harness.experiments", + "harness.experiments.slm267_programspec_coverage_scaling", + ) + + report_text = json.dumps(payload, indent=2, sort_keys=True, default=str) + "\n" + + run_json = output_dir / "slm267_programspec_coverage_scaling_report.json" + run_json.write_text(report_text, encoding="utf-8") + + if args.mode == "fixture": + root = Path(__file__).resolve().parents[1] + json_path = root / _DESIGN_JSON + md_path = root / _DESIGN_MD + json_path.parent.mkdir(parents=True, exist_ok=True) + md_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text(report_text, encoding="utf-8") + md_path.write_text(render_markdown(payload), encoding="utf-8") + + print(str(run_json)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/slm_training/data/progspec/generate.py b/src/slm_training/data/progspec/generate.py index 78c3f3b8c..a682ba129 100644 --- a/src/slm_training/data/progspec/generate.py +++ b/src/slm_training/data/progspec/generate.py @@ -752,7 +752,30 @@ def _build_program(self, candidate: _Candidate) -> tuple[str, set[CoverageCell]] return source, cells def generate_one(self) -> ProgramSpec: - candidate = self._choose() + return self._materialize(self._choose()) + + def generate_uniform(self, rng: random.Random | None = None) -> ProgramSpec: + """Materialize the next unused candidate uniformly at random. + + Unlike ``generate_one``, which always calls the greedy coverage-maximizing + ``_choose()``, every unused candidate in the fixed grid is equally likely. + This is the uniform-valid control arm referenced by VSD2-02: "uniform" + must state whether it is uniform over productions, value classes, + ProgramSpec templates, or accepted roots — here it is uniform over the + generator's own bounded candidate grid. + """ + available = [ + candidate + for candidate in self._candidates + if candidate.key() not in self._used + ] + if not available: + raise ValueError("candidate grid exhausted") + candidate = (rng or self._rng).choice(available) + self._used.add(candidate.key()) + return self._materialize(candidate) + + def _materialize(self, candidate: _Candidate) -> ProgramSpec: openui, cells = self._build_program(candidate) identity = json.dumps( [ diff --git a/src/slm_training/harnesses/experiments/slm267_programspec_coverage_scaling.py b/src/slm_training/harnesses/experiments/slm267_programspec_coverage_scaling.py new file mode 100644 index 000000000..e33a02560 --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm267_programspec_coverage_scaling.py @@ -0,0 +1,310 @@ +"""Bounded ProgramSpec-first coverage-scaling wiring evidence for SLM-267 (VSD2-02). + +VSD2-02 asks for 10k/100k/1M-record corpora comparing uniform-valid against +coverage-targeted sampling. The repository's existing coverage-guided +generator (SLM-5, ``slm_training.data.progspec.generate``) samples from a +fixed, exhaustible candidate grid rather than an open-ended program space, so +those rungs are out of reach without a materially different generator. This +module measures what the *existing* generator actually supports: a genuine +uniform-random control arm (``ProgramGenerator.generate_uniform``) against its +existing greedy coverage-maximizing arm (``ProgramGenerator.generate_one``), +at fixture scale, plus the deterministic ``(global_seed, shard_id, +worker_id)`` sharding primitive VSD2-02 requires. This is wiring evidence +only: it does not validate VSD-H7a/b/c, which require corpora and trained +models several orders of magnitude larger than anything generated here. +""" + +from __future__ import annotations + +import hashlib +import json +import random +from dataclasses import dataclass +from typing import Any + +from slm_training.data.progspec.generate import ( + PROGRAM_FAMILY, + GeneratorConfig, + ProgramGenerator, +) +from slm_training.data.progspec.schema import ProgramSpec, emit_record +from slm_training.harnesses.train_data.diversity import fingerprint_record + +SCHEMA = "programspec_coverage_scaling_report/v1" +EXPERIMENT_ID = "slm267-programspec-coverage-scaling" +POLICIES: tuple[str, ...] = ("uniform", "coverage_targeted") + +DEFAULT_CONFIG = GeneratorConfig( + components=("TextContent", "Button", "Separator"), + max_depth=3, + max_width=3, +) + + +def _sha(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode() + ).hexdigest() + + +def shard_seed(global_seed: int, shard_id: int, worker_id: int) -> int: + """Deterministically derive a per-shard/per-worker seed. + + The same ``(global_seed, shard_id, worker_id)`` triple always derives the + same seed; varying any one input changes it. This is the sharding + primitive VSD2-02 requires ("deterministic sharding from (global_seed, + shard_id, worker_id)") — resumable and reproducible without a stateful + RNG carried across shards. + """ + if shard_id < 0 or worker_id < 0: + raise ValueError("shard_id and worker_id must be non-negative") + payload = json.dumps([global_seed, shard_id, worker_id], separators=(",", ":")) + digest = hashlib.sha256(payload.encode("utf-8")).hexdigest() + return int(digest[:8], 16) + + +def _canonical_root_id(spec: ProgramSpec) -> str: + record = emit_record( + spec, + prompt="Generate this typed OpenUI program.", + task="generation", + source=PROGRAM_FAMILY, + ) + return fingerprint_record(record).canonical_root_id + + +@dataclass(frozen=True) +class ArmResultV1: + """One policy arm's yield/coverage/dedup counters over a bounded budget.""" + + policy: str + seed: int + requested_count: int + proposed: int + accepted: int + duplicate_roots: int + unique_canonical_roots: int + exhausted_at: int | None + cells_total: int + cells_covered: int + programs_to_full_coverage: int | None + + def to_dict(self) -> dict[str, Any]: + return { + "policy": self.policy, + "seed": self.seed, + "requested_count": self.requested_count, + "proposed": self.proposed, + "accepted": self.accepted, + "duplicate_roots": self.duplicate_roots, + "unique_canonical_roots": self.unique_canonical_roots, + "exhausted_at": self.exhausted_at, + "cells_total": self.cells_total, + "cells_covered": self.cells_covered, + "programs_to_full_coverage": self.programs_to_full_coverage, + } + + +def run_policy_arm( + policy: str, *, config: GeneratorConfig, seed: int, target_count: int +) -> ArmResultV1: + """Run one sampling policy up to ``target_count`` programs or exhaustion. + + ``"uniform"`` draws uniformly at random from the unused candidate grid + (no coverage bias). ``"coverage_targeted"`` uses the generator's existing + greedy coverage-maximizing selection, which always prioritizes cells not + yet covered — the closest honest analogue of VSD2-02's coverage-targeted + principal policy available from the current generator without accepting + an unverified external gap-manifest mapping. + """ + if policy not in POLICIES: + raise ValueError(f"unknown policy {policy!r}; expected one of {POLICIES}") + if target_count < 0: + raise ValueError("target_count must be non-negative") + generator = ProgramGenerator(config, seed=seed) + rng = random.Random(seed) + proposed = 0 + accepted = 0 + exhausted_at: int | None = None + programs_to_full_coverage: int | None = None + seen_roots: set[str] = set() + duplicate_roots = 0 + for _ in range(target_count): + proposed += 1 + try: + spec = ( + generator.generate_uniform(rng) + if policy == "uniform" + else generator.generate_one() + ) + except ValueError as exc: + if str(exc) == "candidate grid exhausted": + exhausted_at = accepted + break + raise + accepted += 1 + root_id = _canonical_root_id(spec) + if root_id in seen_roots: + duplicate_roots += 1 + else: + seen_roots.add(root_id) + if programs_to_full_coverage is None and generator.tracker.complete: + programs_to_full_coverage = accepted + report = generator.tracker.report() + return ArmResultV1( + policy=policy, + seed=seed, + requested_count=target_count, + proposed=proposed, + accepted=accepted, + duplicate_roots=duplicate_roots, + unique_canonical_roots=len(seen_roots), + exhausted_at=exhausted_at, + cells_total=report["targets"], + cells_covered=report["covered"], + programs_to_full_coverage=programs_to_full_coverage, + ) + + +def _first_full_coverage(shard_results: list[dict[str, Any]]) -> int | None: + values = [ + result["programs_to_full_coverage"] + for result in shard_results + if result["programs_to_full_coverage"] is not None + ] + return min(values) if values else None + + +def run_coverage_scaling_campaign( + *, + target_count: int = 60, + seed: int = 0, + shards: int = 1, + config: GeneratorConfig | None = None, +) -> dict[str, Any]: + """Compare uniform vs. coverage-targeted sampling at fixture scale. + + Returns a versioned, self-hashed payload; never a ship or scale claim. + """ + if shards < 1: + raise ValueError("shards must be positive") + cfg = config or DEFAULT_CONFIG + arms: dict[str, list[dict[str, Any]]] = {} + for policy in POLICIES: + arms[policy] = [ + run_policy_arm( + policy, + config=cfg, + seed=shard_seed(seed, shard_id, worker_id=0), + target_count=target_count, + ).to_dict() + for shard_id in range(shards) + ] + uniform_first = _first_full_coverage(arms["uniform"]) + targeted_first = _first_full_coverage(arms["coverage_targeted"]) + targeted_no_slower = ( + uniform_first is not None + and targeted_first is not None + and targeted_first <= uniform_first + ) + payload: dict[str, Any] = { + "schema": SCHEMA, + "experiment_id": EXPERIMENT_ID, + "claim_class": "wiring", + "recipe": { + "target_count": target_count, + "seed": seed, + "shards": shards, + "components": list(cfg.components or ()), + "max_depth": cfg.max_depth, + "max_width": cfg.max_width, + }, + "arms": arms, + "comparison": { + "uniform_programs_to_full_coverage": uniform_first, + "coverage_targeted_programs_to_full_coverage": targeted_first, + "coverage_targeted_reaches_full_coverage_no_slower": targeted_no_slower, + }, + "status": "inconclusive", + "limitations": [ + "Fixture-scale only: target_count and the candidate grid are far " + "below the 10k/100k/1M rungs VSD-H7a-c require; no claim is made " + "about those hypotheses at this scale.", + "The 'coverage_targeted' arm reuses the generator's existing " + "greedy CoverageTracker.score() bias toward uncovered cells; it " + "does not yet consume an external CoverageGapManifestV1 " + "(SLM-265) gap manifest — that mapping is explicitly future work.", + "Canonical-root dedup here is in-memory only; a disk-backed " + "exact index for 100k/1M-scale generation is not implemented.", + "No training/model comparison is run and no corpus is published " + "to the DataStore; this measures the generator mechanism only.", + ], + } + payload["manifest_hash"] = _sha( + {key: value for key, value in payload.items() if key != "manifest_hash"} + ) + return payload + + +def render_markdown(report: dict[str, Any]) -> str: + comparison = report["comparison"] + lines = [ + "# SLM-267 (VSD2-02): ProgramSpec coverage-scaling wiring evidence", + "", + "**Claim class:** wiring / fixture only", + "", + f"**Status:** `{report['status']}`", + "", + "## Recipe", + "", + f"```json\n{json.dumps(report['recipe'], indent=2)}\n```", + "", + "## Arms", + "", + "| policy | shard | seed | proposed | accepted | duplicate_roots | " + "unique_roots | exhausted_at | cells_covered/total | " + "programs_to_full_coverage |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for policy, shard_results in report["arms"].items(): + for shard_id, result in enumerate(shard_results): + lines.append( + f"| {policy} | {shard_id} | {result['seed']} | " + f"{result['proposed']} | {result['accepted']} | " + f"{result['duplicate_roots']} | {result['unique_canonical_roots']} | " + f"{result['exhausted_at']} | " + f"{result['cells_covered']}/{result['cells_total']} | " + f"{result['programs_to_full_coverage']} |" + ) + lines.extend( + [ + "", + "## Comparison", + "", + f"- uniform programs to full coverage: " + f"{comparison['uniform_programs_to_full_coverage']}", + f"- coverage-targeted programs to full coverage: " + f"{comparison['coverage_targeted_programs_to_full_coverage']}", + f"- coverage-targeted reaches full coverage no slower than uniform: " + f"{comparison['coverage_targeted_reaches_full_coverage_no_slower']}", + "", + "## Limitations", + "", + ] + ) + lines.extend(f"- {item}" for item in report["limitations"]) + lines.append("") + return "\n".join(lines) + + +__all__ = [ + "DEFAULT_CONFIG", + "EXPERIMENT_ID", + "POLICIES", + "SCHEMA", + "ArmResultV1", + "render_markdown", + "run_coverage_scaling_campaign", + "run_policy_arm", + "shard_seed", +] diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 41d36882d..d503a035f 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -1644,7 +1644,7 @@ ] }, "harness.experiments": { - "version": "v106", + "version": "v107", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/", @@ -1653,6 +1653,11 @@ "tests/test_scripts/test_run_slm298_capacity_context_curriculum.py" ], "history": [ + { + "version": "v107", + "date": "2026-07-25", + "note": "SLM-267 bounded uniform-vs-coverage-targeted ProgramSpec sampling wiring evidence and deterministic shard-seed primitive" + }, { "version": "v106", "date": "2026-07-25", @@ -5551,6 +5556,26 @@ } ] }, + "harness.experiments.slm267_programspec_coverage_scaling": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm267_programspec_coverage_scaling.py", + "scripts/generate_programspec_corpus.py", + "tests/test_harnesses/experiments/test_slm267_programspec_coverage_scaling.py", + "tests/test_scripts/test_generate_programspec_corpus.py", + "docs/design/programspec-coverage-scaling.md", + "docs/design/iter-slm267-programspec-coverage-scaling-20260725.json", + "docs/design/iter-slm267-programspec-coverage-scaling-20260725.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-25", + "note": "initial registration; SLM-267 bounded uniform-vs-coverage-targeted ProgramSpec sampling wiring evidence and deterministic shard-seed primitive" + } + ] + }, "harness.experiments.slm266_teacher_programs": { "version": "v10", "kind": "harness", diff --git a/tests/test_harnesses/experiments/test_slm267_programspec_coverage_scaling.py b/tests/test_harnesses/experiments/test_slm267_programspec_coverage_scaling.py new file mode 100644 index 000000000..c430bb6d1 --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm267_programspec_coverage_scaling.py @@ -0,0 +1,104 @@ +"""Tests for SLM-267 (VSD2-02) ProgramSpec coverage-scaling wiring harness.""" + +from __future__ import annotations + +import pytest + +from slm_training.harnesses.experiments.slm267_programspec_coverage_scaling import ( + DEFAULT_CONFIG, + run_coverage_scaling_campaign, + run_policy_arm, + shard_seed, +) + + +def test_shard_seed_is_deterministic() -> None: + assert shard_seed(0, 0, 0) == shard_seed(0, 0, 0) + + +def test_shard_seed_is_sensitive_to_every_input() -> None: + base = shard_seed(0, 0, 0) + assert shard_seed(0, 1, 0) != base + assert shard_seed(0, 0, 1) != base + assert shard_seed(1, 0, 0) != base + + +def test_shard_seed_rejects_negative_ids() -> None: + with pytest.raises(ValueError): + shard_seed(0, -1, 0) + with pytest.raises(ValueError): + shard_seed(0, 0, -1) + + +def test_run_policy_arm_rejects_unknown_policy() -> None: + with pytest.raises(ValueError): + run_policy_arm("random", config=DEFAULT_CONFIG, seed=0, target_count=1) + + +def test_run_policy_arm_rejects_negative_target_count() -> None: + with pytest.raises(ValueError): + run_policy_arm("uniform", config=DEFAULT_CONFIG, seed=0, target_count=-1) + + +def test_run_policy_arm_is_deterministic() -> None: + first = run_policy_arm( + "coverage_targeted", config=DEFAULT_CONFIG, seed=7, target_count=40 + ) + second = run_policy_arm( + "coverage_targeted", config=DEFAULT_CONFIG, seed=7, target_count=40 + ) + assert first == second + + +def test_uniform_arm_actually_ignores_coverage_score() -> None: + """A uniform arm and a coverage-targeted arm from the same seed must diverge + in acceptance order even though both draw from the identical candidate grid.""" + uniform = run_policy_arm( + "uniform", config=DEFAULT_CONFIG, seed=0, target_count=40 + ) + targeted = run_policy_arm( + "coverage_targeted", config=DEFAULT_CONFIG, seed=0, target_count=40 + ) + assert uniform.programs_to_full_coverage != targeted.programs_to_full_coverage + + +def test_campaign_rejects_non_positive_shards() -> None: + with pytest.raises(ValueError): + run_coverage_scaling_campaign(target_count=10, shards=0) + + +def test_campaign_is_deterministic_and_self_hashed() -> None: + first = run_coverage_scaling_campaign(target_count=60, seed=1, shards=2) + second = run_coverage_scaling_campaign(target_count=60, seed=1, shards=2) + assert first == second + assert first["manifest_hash"] + + +def test_campaign_coverage_targeted_reaches_full_coverage_no_slower() -> None: + report = run_coverage_scaling_campaign(target_count=80, seed=0, shards=2) + comparison = report["comparison"] + assert comparison["uniform_programs_to_full_coverage"] is not None + assert comparison["coverage_targeted_programs_to_full_coverage"] is not None + assert ( + comparison["coverage_targeted_programs_to_full_coverage"] + <= comparison["uniform_programs_to_full_coverage"] + ) + assert comparison["coverage_targeted_reaches_full_coverage_no_slower"] is True + + +def test_campaign_is_honest_wiring_only() -> None: + report = run_coverage_scaling_campaign(target_count=60, seed=0, shards=1) + assert report["claim_class"] == "wiring" + assert report["status"] == "inconclusive" + assert report["limitations"] + + +def test_campaign_dedups_canonical_roots_in_memory() -> None: + report = run_coverage_scaling_campaign(target_count=60, seed=0, shards=1) + for shard_results in report["arms"].values(): + for result in shard_results: + assert result["unique_canonical_roots"] <= result["accepted"] + assert ( + result["unique_canonical_roots"] + result["duplicate_roots"] + == result["accepted"] + ) diff --git a/tests/test_scripts/test_generate_programspec_corpus.py b/tests/test_scripts/test_generate_programspec_corpus.py new file mode 100644 index 000000000..905e4b8a6 --- /dev/null +++ b/tests/test_scripts/test_generate_programspec_corpus.py @@ -0,0 +1,45 @@ +"""Tests for the SLM-267 (VSD2-02) ProgramSpec coverage-scaling fixture CLI.""" + +from __future__ import annotations + +import json + +from scripts.generate_programspec_corpus import main + + +def test_plan_only_writes_manifest(tmp_path) -> None: + assert main(["--mode", "plan-only", "--output-dir", str(tmp_path)]) == 0 + run_json = tmp_path / "slm267_programspec_coverage_scaling_report.json" + assert run_json.exists() + data = json.loads(run_json.read_text()) + assert data["status"] == "plan_only" + assert data["claim_class"] == "wiring" + assert data["schema"] == "Slm267ProgramspecCoverageScalingManifestV1" + assert data["version_stamp"] + + +def test_fixture_writes_design_docs(tmp_path) -> None: + assert ( + main( + [ + "--mode", + "fixture", + "--output-dir", + str(tmp_path), + "--target-count", + "80", + "--seed", + "0", + "--shards", + "2", + ] + ) + == 0 + ) + run_json = tmp_path / "slm267_programspec_coverage_scaling_report.json" + assert run_json.exists() + data = json.loads(run_json.read_text()) + assert data["status"] == "inconclusive" + assert data["claim_class"] == "wiring" + assert data["arms"] + assert data["manifest_hash"] From fa349f9bc23414bf568cbb53ef5c7200e9b83356 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 07:49:01 +0000 Subject: [PATCH 2/3] SLM-296: bounded verifier-filtered teacher-admission wiring evidence AP-013 asks for >=500 accepted records from a capable separate-family teacher, filtered through G0-G10 + independent judgment + human audit, then a matched-budget fine-tune with grammar-on/off evaluation. This sandbox has no network access to a real teacher and no GPU -- SLM-266 already measured that even a local CPU teacher run is infeasible here (>=11 days extrapolated for 10k requests). Scoped down to a bounded wiring slice that exercises the two real safety mechanisms the acceptance criteria test: - generate.py's compiler-inverted ProgramSpec generator stands in as a deterministic offline candidate proposer (explicitly not a teacher), tagged source="teacher" (a verifier-stack weak source) so admitted rows structurally cap at Bronze tier regardless of gate passes -- "teacher self-judgment alone cannot assign Gold/Silver" is enforced by the real verify_record() tier rule, not asserted narratively. - check_family_independence() + a same-family "shadow judge" control: wiring independent_judge_passed=False for a same-family judge causes the real G11 gate to quarantine 10/10 sampled candidates; a different-family judge would pass. Locked-holdout decontamination excludes any candidate whose canonical root (SLM-171 diversity fingerprint) collides with a disjoint-seed holdout before counting yield (23/27 excluded in the fixture run, reflecting the generator's shared finite candidate grid already documented in SLM-267). Disposition: yield_limitation_no_real_teacher_available (wiring only) -- the >=500-record scale and grammar-on/off fine-tune comparison remain unaddressed in this sandbox and are documented as explicit follow-up. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01MAH7ibENsNM41tPTnVJkbg --- ...6-verified-teacher-admission-20260725.json | 247 +++++++++++++ ...296-verified-teacher-admission-20260725.md | 49 +++ docs/design/verified-teacher-admission.md | 63 ++++ scripts/run_verified_teacher_admission.py | 151 ++++++++ .../slm296_verified_teacher_admission.py | 329 ++++++++++++++++++ src/slm_training/resources/versions.json | 27 +- .../test_slm296_verified_teacher_admission.py | 83 +++++ .../test_run_verified_teacher_admission.py | 29 ++ 8 files changed, 977 insertions(+), 1 deletion(-) create mode 100644 docs/design/iter-slm296-verified-teacher-admission-20260725.json create mode 100644 docs/design/iter-slm296-verified-teacher-admission-20260725.md create mode 100644 docs/design/verified-teacher-admission.md create mode 100644 scripts/run_verified_teacher_admission.py create mode 100644 src/slm_training/harnesses/experiments/slm296_verified_teacher_admission.py create mode 100644 tests/test_harnesses/experiments/test_slm296_verified_teacher_admission.py create mode 100644 tests/test_scripts/test_run_verified_teacher_admission.py diff --git a/docs/design/iter-slm296-verified-teacher-admission-20260725.json b/docs/design/iter-slm296-verified-teacher-admission-20260725.json new file mode 100644 index 000000000..29240f5b3 --- /dev/null +++ b/docs/design/iter-slm296-verified-teacher-admission-20260725.json @@ -0,0 +1,247 @@ +{ + "claim_class": "wiring", + "counts": { + "accepted": 4, + "accepted_gold_or_silver": 0, + "gate_failure_counts": {}, + "locked_set_overlap_rejected": 23, + "proposed": 27, + "tier_counts": { + "BRONZE": 4 + }, + "unique_accepted_canonical_roots": 4 + }, + "experiment_id": "slm296-verified-teacher-admission", + "family_independence_control": { + "different_family_independent": true, + "same_family_independent": false, + "shadow_judge_fully_quarantined": true, + "shadow_judge_quarantined": 10, + "shadow_judge_sample_size": 10 + }, + "limitations": [ + "No real separate-family teacher is available (no network/GPU in this sandbox); candidates come from the repository's own deterministic ProgramSpec generator, explicitly not a teacher. SLM-266 already measured that a local CPU teacher run is infeasible here (>= 11 days extrapolated for 10k requests).", + "The >= 500 accepted-record target is not attempted at this scale; this is a bounded wiring campaign only.", + "No fine-tuning run occurs and no grammar-off/meaning-v2 delta is measured or claimed.", + "Gold/Silver promotion is structurally blocked here because 'teacher' is a weak source with no human_audit_passed evidence -- every accepted row is Bronze at best, by construction of the real verifier stack, not by a claim made in this report." + ], + "manifest_hash": "5d9a916e1d74b39a64af4a9d4226edc7c0d41eda33a9e382c75edbf5aa2794ff", + "recipe": { + "components": [ + "TextContent", + "Button", + "Separator" + ], + "locked_holdout_seed": 999, + "seeds": [ + 0, + 1, + 2 + ], + "teacher_model_family": "deterministic_programspec_generator_v1" + }, + "rows": [ + { + "candidate_id": "program_c9e62fec8f8263fd_generation_da25ade1f0c6", + "canonical_root_id": "ca3e1f5a90fa6a4393c74f13472572c0da1e697c15195e5383c36fbded41cc30", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_07b5a65c60749067_generation_92288d108dd3", + "canonical_root_id": "124dc1b551ab7b6614eb732d26d9bd6f7931ebe90a08d0c9cc51f68d25221c48", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_3dc62808440ea389_generation_805886e1ba0f", + "canonical_root_id": "bfdd8a14b267b25dd39254649eee1e53829d06f2896f3fadc6e7a907dcab2539", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_11d452008abc1814_generation_17181cd32f22", + "canonical_root_id": "2329c928f0ccad34371023160569b49a0d2fe26173881ee79f787345cd8d402f", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_eccd2066c7d2f32d_generation_30b08afa271b", + "canonical_root_id": "85f8d0b8bdb04848360e647fb16451181ea75c9e5dae7b72d32275c934e10185", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_6034364651dc4e8f_generation_3156b20da01d", + "canonical_root_id": "2d885f6987b256a49d0aed818b777e76cb73d08f5395fe58566c96d6428f194d", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_59f40e94afab4028_generation_110026eb9bd1", + "canonical_root_id": "4653c3b3b163899eae1db649e1f16eb2c4c0d41f28e16016e198866f67df60ec", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_680caca3370d9c16_generation_c39097d9db01", + "canonical_root_id": "dfc23b60aa17911259bb60f9f81e12fae04ad12c7371d29ae2da3528721db2ea", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_a99bdf966d072a7f_generation_d4fc772436e4", + "canonical_root_id": "0a1895f1bf7de094d72713e1d0259e1939bf010d6994c22cffb9bc69a6a4e6cc", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_c9e62fec8f8263fd_generation_da25ade1f0c6", + "canonical_root_id": "ca3e1f5a90fa6a4393c74f13472572c0da1e697c15195e5383c36fbded41cc30", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_11d452008abc1814_generation_17181cd32f22", + "canonical_root_id": "2329c928f0ccad34371023160569b49a0d2fe26173881ee79f787345cd8d402f", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_80318515de512a30_generation_c61dd2205c7e", + "canonical_root_id": "40daa87a44c0a2bfa240024f725f6ceb318155616a42029c55060b39c8e06393", + "failing_gate": null, + "locked_set_overlap": false, + "tier": "BRONZE" + }, + { + "candidate_id": "program_2ab9840a305d9f70_generation_47a982fb6e05", + "canonical_root_id": "6a3ad674df1031c5ab29291c4c5bb6a9e3814eaedb7a7f7ca79fe518d74fa2cd", + "failing_gate": null, + "locked_set_overlap": false, + "tier": "BRONZE" + }, + { + "candidate_id": "program_3dc62808440ea389_generation_805886e1ba0f", + "canonical_root_id": "bfdd8a14b267b25dd39254649eee1e53829d06f2896f3fadc6e7a907dcab2539", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_d203725668cbc2e3_generation_7fc150d82fb2", + "canonical_root_id": "9d9ee4fd64a43b18b7a6b91b9f38d55cfd9fdd1e30703d2ee68f6b605fe98a64", + "failing_gate": null, + "locked_set_overlap": false, + "tier": "BRONZE" + }, + { + "candidate_id": "program_da2d696e8b286563_generation_658f23556f1a", + "canonical_root_id": "433130343adc7661214b638257d5943937d134a558ff65d79d5801dc49fa260d", + "failing_gate": null, + "locked_set_overlap": false, + "tier": "BRONZE" + }, + { + "candidate_id": "program_680caca3370d9c16_generation_c39097d9db01", + "canonical_root_id": "dfc23b60aa17911259bb60f9f81e12fae04ad12c7371d29ae2da3528721db2ea", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_07b5a65c60749067_generation_92288d108dd3", + "canonical_root_id": "124dc1b551ab7b6614eb732d26d9bd6f7931ebe90a08d0c9cc51f68d25221c48", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_680caca3370d9c16_generation_c39097d9db01", + "canonical_root_id": "dfc23b60aa17911259bb60f9f81e12fae04ad12c7371d29ae2da3528721db2ea", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_07b5a65c60749067_generation_92288d108dd3", + "canonical_root_id": "124dc1b551ab7b6614eb732d26d9bd6f7931ebe90a08d0c9cc51f68d25221c48", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_3dc62808440ea389_generation_805886e1ba0f", + "canonical_root_id": "bfdd8a14b267b25dd39254649eee1e53829d06f2896f3fadc6e7a907dcab2539", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_11d452008abc1814_generation_17181cd32f22", + "canonical_root_id": "2329c928f0ccad34371023160569b49a0d2fe26173881ee79f787345cd8d402f", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_eccd2066c7d2f32d_generation_30b08afa271b", + "canonical_root_id": "85f8d0b8bdb04848360e647fb16451181ea75c9e5dae7b72d32275c934e10185", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_6034364651dc4e8f_generation_3156b20da01d", + "canonical_root_id": "2d885f6987b256a49d0aed818b777e76cb73d08f5395fe58566c96d6428f194d", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_59f40e94afab4028_generation_110026eb9bd1", + "canonical_root_id": "4653c3b3b163899eae1db649e1f16eb2c4c0d41f28e16016e198866f67df60ec", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_c9e62fec8f8263fd_generation_da25ade1f0c6", + "canonical_root_id": "ca3e1f5a90fa6a4393c74f13472572c0da1e697c15195e5383c36fbded41cc30", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + }, + { + "candidate_id": "program_a99bdf966d072a7f_generation_d4fc772436e4", + "canonical_root_id": "0a1895f1bf7de094d72713e1d0259e1939bf010d6994c22cffb9bc69a6a4e6cc", + "failing_gate": null, + "locked_set_overlap": true, + "tier": "excluded_locked_set_overlap" + } + ], + "schema": "verified_teacher_admission_report/v1", + "status": "yield_limitation_no_real_teacher_available", + "timestamp": "2026-07-25T07:48:18.733805Z", + "version_stamp": { + "code_commit": "368395a2fa7d9642fc9497f17664bfd9b6adc56d", + "code_dirty": true, + "components": { + "harness.experiments": "v106", + "harness.experiments.slm296_verified_teacher_admission": "v1" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T07:48:18.761255+00:00" + } +} diff --git a/docs/design/iter-slm296-verified-teacher-admission-20260725.md b/docs/design/iter-slm296-verified-teacher-admission-20260725.md new file mode 100644 index 000000000..2d291dcdd --- /dev/null +++ b/docs/design/iter-slm296-verified-teacher-admission-20260725.md @@ -0,0 +1,49 @@ +# SLM-296 (AP-013): verifier-filtered teacher-admission wiring evidence + +**Claim class:** wiring / fixture only + +**Status:** `yield_limitation_no_real_teacher_available` + +## Recipe + +```json +{ + "seeds": [ + 0, + 1, + 2 + ], + "components": [ + "TextContent", + "Button", + "Separator" + ], + "teacher_model_family": "deterministic_programspec_generator_v1", + "locked_holdout_seed": 999 +} +``` + +## Admission counts + +| metric | value | +| --- | --- | +| proposed | 27 | +| locked_set_overlap_rejected | 23 | +| accepted | 4 | +| accepted_gold_or_silver | 0 | +| unique_accepted_canonical_roots | 4 | +| tier_counts | {'BRONZE': 4} | +| gate_failure_counts | {} | + +## Family-independence control + +- same-family judge independent: False +- different-family judge independent: True +- shadow-judge sample fully quarantined: True (10/10) + +## Limitations + +- No real separate-family teacher is available (no network/GPU in this sandbox); candidates come from the repository's own deterministic ProgramSpec generator, explicitly not a teacher. SLM-266 already measured that a local CPU teacher run is infeasible here (>= 11 days extrapolated for 10k requests). +- The >= 500 accepted-record target is not attempted at this scale; this is a bounded wiring campaign only. +- No fine-tuning run occurs and no grammar-off/meaning-v2 delta is measured or claimed. +- Gold/Silver promotion is structurally blocked here because 'teacher' is a weak source with no human_audit_passed evidence -- every accepted row is Bronze at best, by construction of the real verifier stack, not by a claim made in this report. diff --git a/docs/design/verified-teacher-admission.md b/docs/design/verified-teacher-admission.md new file mode 100644 index 000000000..001af9bc7 --- /dev/null +++ b/docs/design/verified-teacher-admission.md @@ -0,0 +1,63 @@ +# Verifier-filtered teacher-admission wiring evidence + +SLM-296 (AP-013) asks for >= 500 accepted records from a capable +separate-family teacher, filtered through G0-G10 plus independent judgment +and human audit, then a matched-budget fine-tune with a grammar-on/off +evaluation. This sandbox has no network access to a real external teacher +and no GPU. SLM-266 (`docs/design/teacher-program-data.md`) already measured +that even a local CPU teacher run is infeasible here: an 8-token probe took +95.65s, and a 10k-request campaign would take at least 11 continuous days. + +This issue is scoped down to a bounded wiring slice instead: a deterministic +offline candidate proposer (explicitly *not* a real teacher) feeding the +real G0-G12 verifier stack, exercising the two safety mechanisms the +acceptance criteria actually test -- that teacher self-judgment cannot +promote a record, and that decontamination against a locked holdout works. + +The committed [disposition JSON](iter-slm296-verified-teacher-admission-20260725.json) +records the exact recipe and counts. Rerun the exact command to regenerate it: + +```bash +python -m scripts.run_verified_teacher_admission --mode fixture +``` + +## Measured fixture-scale campaign -- 2026-07-25 + +27 candidates were proposed across 3 seeds from a 3-component ProgramSpec +grid (the same bounded generator used for SLM-267). A disjoint-seed "locked +holdout" was built from the same grid; because the generator's candidate +grid is finite and shared across seeds (a finding already established in +SLM-267), most proposed candidates also appear in the holdout by canonical +root, so 23/27 were correctly excluded as locked-set overlap before any +tier was assigned. Of the 4 remaining, all 4 passed every deterministic gate +(G0-G10) and were admitted at **Bronze** tier -- never Gold or Silver, since +`"teacher"` is a weak source in the verifier stack's tier rule +(`_WEAK_SOURCES`) and no `human_audit_passed` evidence was supplied. + +The family-independence control re-verified a 10-candidate sample with +`independent_judge_passed` wired from `check_family_independence` when the +(hypothetical) judge shares the teacher's model family: **all 10** were +quarantined by the G11 gate. The same check against a genuinely different +family returns `True` (would pass, never exercised against real candidates +here since no real judge exists). + +## Honest scope + +This is wiring evidence only: + +- The candidate proposer is the repository's own deterministic ProgramSpec + generator, not a separate-family teacher; `TEACHER_MODEL_FAMILY` is + labeled `deterministic_programspec_generator_v1` throughout. +- The >= 500 accepted-record target is not attempted at this scale. +- No fine-tuning occurs and no grammar-off/meaning-v2 delta is measured. +- Gold/Silver promotion is structurally blocked by the real verifier stack's + tier rule for weak sources, not by an accounting claim made in this report. +- Family independence and decontamination are exercised with real pass/fail + logic (`check_family_independence`, canonical-root locked-set overlap), + not asserted narratively. + +Disposition: **`yield_limitation_no_real_teacher_available`** -- the +mechanism (verifier-filtered admission, tier-capping for weak sources, +same-family self-judgment quarantine, locked-set decontamination) is real, +tested, and reproducible; the accepted-record scale and grammar-on/off +fine-tune comparison SLM-296 asks for remain unaddressed in this sandbox. diff --git a/scripts/run_verified_teacher_admission.py b/scripts/run_verified_teacher_admission.py new file mode 100644 index 000000000..bae15f614 --- /dev/null +++ b/scripts/run_verified_teacher_admission.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Run the SLM-296 (AP-013) verifier-filtered teacher-admission wiring fixture. + +Example: + python -m scripts.run_verified_teacher_admission --mode plan-only + python -m scripts.run_verified_teacher_admission --mode fixture +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from slm_training.harnesses.experiments.slm296_verified_teacher_admission import ( + DEFAULT_CONFIG, + EXPERIMENT_ID, + TEACHER_MODEL_FAMILY, + render_markdown, + run_verified_teacher_admission, +) +from slm_training.versioning import build_version_stamp + +_DESIGN_JSON = "docs/design/iter-slm296-verified-teacher-admission-20260725.json" +_DESIGN_MD = "docs/design/iter-slm296-verified-teacher-admission-20260725.md" + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _today_yyyymmdd() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%d") + + +def _parse_int_tuple(value: str) -> tuple[int, ...]: + return tuple(int(x.strip()) for x in value.split(",") if x.strip()) + + +def _build_payload( + mode: str, seeds: tuple[int, ...], locked_holdout_seed: int +) -> tuple[dict[str, Any], str]: + command = ( + f"python -m scripts.run_verified_teacher_admission --mode {mode} " + f"--seeds {','.join(str(s) for s in seeds)} " + f"--locked-holdout-seed {locked_holdout_seed}" + ) + if mode == "plan-only": + payload: dict[str, Any] = { + "schema": "Slm296VerifiedTeacherAdmissionManifestV1", + "experiment_id": EXPERIMENT_ID, + "status": "plan_only", + "claim_class": "wiring", + "recipe": { + "seeds": list(seeds), + "components": list(DEFAULT_CONFIG.components or ()), + "teacher_model_family": TEACHER_MODEL_FAMILY, + "locked_holdout_seed": locked_holdout_seed, + }, + "version_stamp": build_version_stamp( + "harness.experiments", + "harness.experiments.slm296_verified_teacher_admission", + ), + "timestamp": _now(), + } + return payload, command + payload = run_verified_teacher_admission( + seeds=seeds, locked_holdout_seed=locked_holdout_seed + ) + return payload, command + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="SLM-296 AP-013 verifier-filtered teacher-admission fixture", + exit_on_error=False, + ) + parser.add_argument( + "--mode", + choices={"plan-only", "fixture"}, + default="plan-only", + help="Run mode: plan-only writes the manifest; fixture runs the CPU campaign.", + ) + parser.add_argument( + "--output-dir", + type=Path, + help=( + "Directory for run artifacts (default: " + "outputs/runs/slm296-verified-teacher-admission-)" + ), + ) + parser.add_argument( + "--seeds", + type=_parse_int_tuple, + default="0,1,2", + help="Comma-separated generator seeds for candidate proposal (default: 0,1,2).", + ) + parser.add_argument( + "--locked-holdout-seed", + type=int, + default=999, + help="Seed for the disjoint locked-holdout decontamination set (default: 999).", + ) + try: + args = parser.parse_args(argv) + except (argparse.ArgumentError, SystemExit): + return 2 + + output_dir = args.output_dir or Path( + f"outputs/runs/slm296-verified-teacher-admission-{_today_yyyymmdd()}" + ) + output_dir.mkdir(parents=True, exist_ok=True) + + payload, command = _build_payload(args.mode, args.seeds, args.locked_holdout_seed) + payload["schema"] = payload.get( + "schema", "Slm296VerifiedTeacherAdmissionReportV1" + ) + payload["claim_class"] = "wiring" + payload["status"] = payload.get( + "status", "yield_limitation_no_real_teacher_available" + ) + payload["timestamp"] = _now() + if "version_stamp" not in payload: + payload["version_stamp"] = build_version_stamp( + "harness.experiments", + "harness.experiments.slm296_verified_teacher_admission", + ) + + report_text = json.dumps(payload, indent=2, sort_keys=True, default=str) + "\n" + + run_json = output_dir / "slm296_verified_teacher_admission_report.json" + run_json.write_text(report_text, encoding="utf-8") + + if args.mode == "fixture": + root = Path(__file__).resolve().parents[1] + json_path = root / _DESIGN_JSON + md_path = root / _DESIGN_MD + json_path.parent.mkdir(parents=True, exist_ok=True) + md_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text(report_text, encoding="utf-8") + md_path.write_text(render_markdown(payload), encoding="utf-8") + + print(str(run_json)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/slm_training/harnesses/experiments/slm296_verified_teacher_admission.py b/src/slm_training/harnesses/experiments/slm296_verified_teacher_admission.py new file mode 100644 index 000000000..47430f538 --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm296_verified_teacher_admission.py @@ -0,0 +1,329 @@ +"""Bounded verifier-filtered teacher-admission wiring evidence for SLM-296 (AP-013). + +SLM-296 asks for >= 500 accepted records generated by a capable +*separate-family* teacher, filtered through G0-G10 plus independent judgment +and human audit, then a matched-budget fine-tune with a grammar-on/off +evaluation. This sandbox has neither network access to a real external +teacher nor a GPU: SLM-266 (`docs/design/teacher-program-data.md`) already +measured that even a local CPU teacher run is infeasible here (an 8-token +probe took 95.65s; a 10k-request campaign would take >= 11 continuous days). + +This module is a bounded, honest wiring slice instead: a deterministic +offline candidate proposer (explicitly *not* a real separate-family +teacher) feeds the real G0-G12 verifier stack (`slm_training.data.verify`), +tagged with the ``"teacher"`` weak source so tier assignment structurally +enforces "teacher self-judgment alone cannot assign Gold/Silver" (only +`human_audit_passed` reaches Gold; weak sources cap at Bronze regardless of +gate passes). It also exercises, with real pass/fail behavior: + +* family independence -- a same-family "shadow judge" is shown to get its + candidates quarantined by the G11 gate, never silently promoted; +* decontamination -- candidates whose canonical root collides with a locked + holdout set are excluded from admission before counting yield. + +No model is trained and no grammar-on/off delta is measured; the >= 500 +accepted-record target is not attempted at this scale. This is wiring +evidence only. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any + +from slm_training.data.progspec.generate import GeneratorConfig, ProgramGenerator +from slm_training.data.progspec.schema import ProgramSpec, emit_record +from slm_training.data.verify import Tier, VerificationContext, verify_record +from slm_training.harnesses.train_data.diversity import fingerprint_record + +SCHEMA = "verified_teacher_admission_report/v1" +EXPERIMENT_ID = "slm296-verified-teacher-admission" +TEACHER_MODEL_FAMILY = "deterministic_programspec_generator_v1" + +DEFAULT_CONFIG = GeneratorConfig( + components=("TextContent", "Button", "Separator"), + max_depth=3, + max_width=3, +) + + +def _sha(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode() + ).hexdigest() + + +def check_family_independence( + teacher_model_family: str, judge_model_family: str | None +) -> bool | None: + """Return whether a judge is independent of the teacher that proposed a candidate. + + ``None`` means no judge was supplied (the gate must SKIP, never PASS); + ``False`` means the judge shares the teacher's model family (self-judgment, + the gate must FAIL); ``True`` means the families differ. + """ + if judge_model_family is None: + return None + return judge_model_family != teacher_model_family + + +def _propose_candidates( + config: GeneratorConfig, *, seeds: tuple[int, ...] +) -> list[ProgramSpec]: + """Deterministic offline candidate proposer standing in for a teacher. + + Multiple seeds emulate multi-sample decoding from a real teacher. This is + the repository's own verified generator (SLM-5), not a separate-family + model -- every candidate is tagged with source ``"teacher"`` downstream so + the real verifier stack treats it as an unproven weak source regardless. + """ + specs: list[ProgramSpec] = [] + for seed in seeds: + generator = ProgramGenerator(config, seed=seed) + for _ in range(config.max_depth * config.max_width): + try: + specs.append(generator.generate_one()) + except ValueError as exc: + if str(exc) == "candidate grid exhausted": + break + raise + return specs + + +def _teacher_record(spec: ProgramSpec): + return emit_record( + spec, + prompt="Generate this typed OpenUI program.", + task="generation", + source="teacher", + provenance={"teacher_model_family": TEACHER_MODEL_FAMILY}, + ) + + +@dataclass(frozen=True) +class AdmissionRowV1: + candidate_id: str + canonical_root_id: str + tier: str + failing_gate: str | None + locked_set_overlap: bool + + def to_dict(self) -> dict[str, Any]: + return { + "candidate_id": self.candidate_id, + "canonical_root_id": self.canonical_root_id, + "tier": self.tier, + "failing_gate": self.failing_gate, + "locked_set_overlap": self.locked_set_overlap, + } + + +def run_verified_teacher_admission( + *, + seeds: tuple[int, ...] = (0, 1, 2), + config: GeneratorConfig | None = None, + locked_holdout_seed: int = 999, +) -> dict[str, Any]: + """Run the bounded verifier-filtered teacher-admission wiring campaign.""" + cfg = config or DEFAULT_CONFIG + proposed_specs = _propose_candidates(cfg, seeds=seeds) + + # Decontamination: a locked holdout drawn from a disjoint seed so its + # canonical roots may legitimately collide with proposed candidates. + holdout_specs = _propose_candidates(cfg, seeds=(locked_holdout_seed,)) + locked_roots = { + fingerprint_record(_teacher_record(spec)).canonical_root_id + for spec in holdout_specs + } + + rows: list[AdmissionRowV1] = [] + accepted_roots: set[str] = set() + tier_counts: dict[str, int] = {} + gate_failure_counts: dict[str, int] = {} + locked_set_overlap_rejected = 0 + + for spec in proposed_specs: + record = _teacher_record(spec) + root_id = fingerprint_record(record).canonical_root_id + overlap = root_id in locked_roots + if overlap: + locked_set_overlap_rejected += 1 + rows.append( + AdmissionRowV1( + candidate_id=record.id, + canonical_root_id=root_id, + tier="excluded_locked_set_overlap", + failing_gate=None, + locked_set_overlap=True, + ) + ) + continue + # independent_judge_passed=None: no real external judge is available; + # the gate must SKIP, never fabricate a PASS. + report = verify_record( + record, + VerificationContext(source_kind="teacher", independent_judge_passed=None), + ) + tier_counts[report.tier.name] = tier_counts.get(report.tier.name, 0) + 1 + failing_gate = report.failing_gate.value if report.failing_gate else None + if failing_gate: + gate_failure_counts[failing_gate] = ( + gate_failure_counts.get(failing_gate, 0) + 1 + ) + if report.ok: + accepted_roots.add(root_id) + rows.append( + AdmissionRowV1( + candidate_id=record.id, + canonical_root_id=root_id, + tier=report.tier.name, + failing_gate=failing_gate, + locked_set_overlap=False, + ) + ) + + accepted = sum( + 1 for row in rows if row.tier != "QUARANTINE" and not row.locked_set_overlap + ) + gold_or_silver = sum( + 1 + for row in rows + if row.tier in {Tier.GOLD.name, Tier.SILVER.name} and not row.locked_set_overlap + ) + + # Family-independence control: a same-family "shadow judge" must never + # promote a candidate. Re-verify a small sample with independent_judge_passed + # wired from check_family_independence(same family) -> False -> gate FAILs. + shadow_sample = proposed_specs[: min(10, len(proposed_specs))] + same_family_independent = check_family_independence( + TEACHER_MODEL_FAMILY, TEACHER_MODEL_FAMILY + ) + diff_family_independent = check_family_independence( + TEACHER_MODEL_FAMILY, "external_model_family_v1" + ) + shadow_quarantined = 0 + for spec in shadow_sample: + record = _teacher_record(spec) + report = verify_record( + record, + VerificationContext( + source_kind="teacher", + independent_judge_passed=same_family_independent, + ), + ) + if report.tier is Tier.QUARANTINE: + shadow_quarantined += 1 + + payload: dict[str, Any] = { + "schema": SCHEMA, + "experiment_id": EXPERIMENT_ID, + "claim_class": "wiring", + "recipe": { + "seeds": list(seeds), + "components": list(cfg.components or ()), + "teacher_model_family": TEACHER_MODEL_FAMILY, + "locked_holdout_seed": locked_holdout_seed, + }, + "counts": { + "proposed": len(proposed_specs), + "locked_set_overlap_rejected": locked_set_overlap_rejected, + "accepted": accepted, + "accepted_gold_or_silver": gold_or_silver, + "unique_accepted_canonical_roots": len(accepted_roots), + "tier_counts": tier_counts, + "gate_failure_counts": gate_failure_counts, + }, + "family_independence_control": { + "same_family_independent": same_family_independent, + "different_family_independent": diff_family_independent, + "shadow_judge_sample_size": len(shadow_sample), + "shadow_judge_quarantined": shadow_quarantined, + "shadow_judge_fully_quarantined": shadow_quarantined == len(shadow_sample), + }, + "rows": [row.to_dict() for row in rows], + "status": "yield_limitation_no_real_teacher_available", + "limitations": [ + "No real separate-family teacher is available (no network/GPU in " + "this sandbox); candidates come from the repository's own " + "deterministic ProgramSpec generator, explicitly not a teacher. " + "SLM-266 already measured that a local CPU teacher run is " + "infeasible here (>= 11 days extrapolated for 10k requests).", + "The >= 500 accepted-record target is not attempted at this " + "scale; this is a bounded wiring campaign only.", + "No fine-tuning run occurs and no grammar-off/meaning-v2 delta " + "is measured or claimed.", + "Gold/Silver promotion is structurally blocked here because " + "'teacher' is a weak source with no human_audit_passed evidence " + "-- every accepted row is Bronze at best, by construction of " + "the real verifier stack, not by a claim made in this report.", + ], + } + payload["manifest_hash"] = _sha( + {key: value for key, value in payload.items() if key != "manifest_hash"} + ) + return payload + + +def render_markdown(report: dict[str, Any]) -> str: + counts = report["counts"] + control = report["family_independence_control"] + lines = [ + "# SLM-296 (AP-013): verifier-filtered teacher-admission wiring evidence", + "", + "**Claim class:** wiring / fixture only", + "", + f"**Status:** `{report['status']}`", + "", + "## Recipe", + "", + f"```json\n{json.dumps(report['recipe'], indent=2)}\n```", + "", + "## Admission counts", + "", + "| metric | value |", + "| --- | --- |", + ] + for key in ( + "proposed", + "locked_set_overlap_rejected", + "accepted", + "accepted_gold_or_silver", + "unique_accepted_canonical_roots", + ): + lines.append(f"| {key} | {counts[key]} |") + lines.append(f"| tier_counts | {counts['tier_counts']} |") + lines.append(f"| gate_failure_counts | {counts['gate_failure_counts']} |") + lines.extend( + [ + "", + "## Family-independence control", + "", + f"- same-family judge independent: {control['same_family_independent']}", + f"- different-family judge independent: " + f"{control['different_family_independent']}", + f"- shadow-judge sample fully quarantined: " + f"{control['shadow_judge_fully_quarantined']} " + f"({control['shadow_judge_quarantined']}/" + f"{control['shadow_judge_sample_size']})", + "", + "## Limitations", + "", + ] + ) + lines.extend(f"- {item}" for item in report["limitations"]) + lines.append("") + return "\n".join(lines) + + +__all__ = [ + "DEFAULT_CONFIG", + "EXPERIMENT_ID", + "SCHEMA", + "TEACHER_MODEL_FAMILY", + "AdmissionRowV1", + "check_family_independence", + "render_markdown", + "run_verified_teacher_admission", +] diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index d503a035f..1cd0a8b8e 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -1644,7 +1644,7 @@ ] }, "harness.experiments": { - "version": "v107", + "version": "v108", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/", @@ -1653,6 +1653,11 @@ "tests/test_scripts/test_run_slm298_capacity_context_curriculum.py" ], "history": [ + { + "version": "v108", + "date": "2026-07-25", + "note": "SLM-296 bounded verifier-filtered teacher-admission wiring evidence, family-independence, and locked-set decontamination controls" + }, { "version": "v107", "date": "2026-07-25", @@ -5576,6 +5581,26 @@ } ] }, + "harness.experiments.slm296_verified_teacher_admission": { + "version": "v1", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm296_verified_teacher_admission.py", + "scripts/run_verified_teacher_admission.py", + "tests/test_harnesses/experiments/test_slm296_verified_teacher_admission.py", + "tests/test_scripts/test_run_verified_teacher_admission.py", + "docs/design/verified-teacher-admission.md", + "docs/design/iter-slm296-verified-teacher-admission-20260725.json", + "docs/design/iter-slm296-verified-teacher-admission-20260725.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-25", + "note": "initial registration; SLM-296 bounded verifier-filtered teacher-admission wiring evidence, family-independence and locked-set decontamination controls" + } + ] + }, "harness.experiments.slm266_teacher_programs": { "version": "v10", "kind": "harness", diff --git a/tests/test_harnesses/experiments/test_slm296_verified_teacher_admission.py b/tests/test_harnesses/experiments/test_slm296_verified_teacher_admission.py new file mode 100644 index 000000000..ba440e34f --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm296_verified_teacher_admission.py @@ -0,0 +1,83 @@ +"""Tests for SLM-296 (AP-013) verifier-filtered teacher-admission wiring harness.""" + +from __future__ import annotations + +from slm_training.harnesses.experiments.slm296_verified_teacher_admission import ( + TEACHER_MODEL_FAMILY, + check_family_independence, + run_verified_teacher_admission, +) + + +def test_check_family_independence_none_when_no_judge() -> None: + assert check_family_independence(TEACHER_MODEL_FAMILY, None) is None + + +def test_check_family_independence_false_for_same_family() -> None: + assert check_family_independence(TEACHER_MODEL_FAMILY, TEACHER_MODEL_FAMILY) is False + + +def test_check_family_independence_true_for_different_family() -> None: + assert check_family_independence(TEACHER_MODEL_FAMILY, "other_family_v1") is True + + +def test_campaign_is_deterministic_and_self_hashed() -> None: + first = run_verified_teacher_admission() + second = run_verified_teacher_admission() + assert first == second + assert first["manifest_hash"] + + +def test_campaign_is_honest_wiring_only() -> None: + report = run_verified_teacher_admission() + assert report["claim_class"] == "wiring" + assert report["status"] == "yield_limitation_no_real_teacher_available" + assert report["limitations"] + + +def test_teacher_source_never_reaches_gold_or_silver() -> None: + """Structural enforcement of 'teacher self-judgment alone cannot assign + Gold/Silver': every admitted row's tier must be Bronze or worse.""" + report = run_verified_teacher_admission() + for row in report["rows"]: + if row["locked_set_overlap"]: + continue + assert row["tier"] in {"BRONZE", "QUARANTINE"} + assert report["counts"]["accepted_gold_or_silver"] == 0 + + +def test_same_family_shadow_judge_is_fully_quarantined() -> None: + report = run_verified_teacher_admission() + control = report["family_independence_control"] + assert control["same_family_independent"] is False + assert control["different_family_independent"] is True + assert control["shadow_judge_fully_quarantined"] is True + assert control["shadow_judge_quarantined"] == control["shadow_judge_sample_size"] + + +def test_locked_set_overlap_rows_are_excluded_from_acceptance() -> None: + report = run_verified_teacher_admission() + counts = report["counts"] + overlap_rows = [row for row in report["rows"] if row["locked_set_overlap"]] + assert len(overlap_rows) == counts["locked_set_overlap_rejected"] + for row in overlap_rows: + assert row["tier"] == "excluded_locked_set_overlap" + non_overlap_accepted = sum( + 1 + for row in report["rows"] + if not row["locked_set_overlap"] and row["tier"] != "QUARANTINE" + ) + assert non_overlap_accepted == counts["accepted"] + + +def test_accepted_roots_never_collide_with_locked_holdout() -> None: + report = run_verified_teacher_admission() + accepted_roots = { + row["canonical_root_id"] + for row in report["rows"] + if not row["locked_set_overlap"] and row["tier"] != "QUARANTINE" + } + overlap_roots = { + row["canonical_root_id"] for row in report["rows"] if row["locked_set_overlap"] + } + assert accepted_roots.isdisjoint(overlap_roots) diff --git a/tests/test_scripts/test_run_verified_teacher_admission.py b/tests/test_scripts/test_run_verified_teacher_admission.py new file mode 100644 index 000000000..4b8a76aa5 --- /dev/null +++ b/tests/test_scripts/test_run_verified_teacher_admission.py @@ -0,0 +1,29 @@ +"""Tests for the SLM-296 (AP-013) verifier-filtered teacher-admission fixture CLI.""" + +from __future__ import annotations + +import json + +from scripts.run_verified_teacher_admission import main + + +def test_plan_only_writes_manifest(tmp_path) -> None: + assert main(["--mode", "plan-only", "--output-dir", str(tmp_path)]) == 0 + run_json = tmp_path / "slm296_verified_teacher_admission_report.json" + assert run_json.exists() + data = json.loads(run_json.read_text()) + assert data["status"] == "plan_only" + assert data["claim_class"] == "wiring" + assert data["schema"] == "Slm296VerifiedTeacherAdmissionManifestV1" + assert data["version_stamp"] + + +def test_fixture_writes_design_docs(tmp_path) -> None: + assert main(["--mode", "fixture", "--output-dir", str(tmp_path)]) == 0 + run_json = tmp_path / "slm296_verified_teacher_admission_report.json" + assert run_json.exists() + data = json.loads(run_json.read_text()) + assert data["status"] == "yield_limitation_no_real_teacher_available" + assert data["claim_class"] == "wiring" + assert data["counts"] + assert data["manifest_hash"] From 85eeb9464f54adf0fa7c89de315cf269ef3778c2 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:40:28 -0500 Subject: [PATCH 3/3] Refresh SLM-267 evidence after rebase --- ...r-slm267-programspec-coverage-scaling-20260725.json | 10 +++++----- ...ter-slm296-verified-teacher-admission-20260725.json | 10 +++++----- src/slm_training/resources/versions.json | 10 ++++++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json b/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json index 5a54ba077..a531b5e78 100644 --- a/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json +++ b/docs/design/iter-slm267-programspec-coverage-scaling-20260725.json @@ -85,15 +85,15 @@ }, "schema": "programspec_coverage_scaling_report/v1", "status": "inconclusive", - "timestamp": "2026-07-25T07:37:15.970003Z", + "timestamp": "2026-07-25T14:37:29.958752Z", "version_stamp": { - "code_commit": "8dd4794a7cea10f1fe9732e5bb25d198b810fc22", - "code_dirty": true, + "code_commit": "fa349f9bc23414bf568cbb53ef5c7200e9b83356", + "code_dirty": false, "components": { - "harness.experiments": "v105", + "harness.experiments": "v108", "harness.experiments.slm267_programspec_coverage_scaling": "v1" }, "stamp_schema": "version_stamp/v1", - "stamped_at": "2026-07-25T07:37:15.970040+00:00" + "stamped_at": "2026-07-25T14:37:29.958799+00:00" } } diff --git a/docs/design/iter-slm296-verified-teacher-admission-20260725.json b/docs/design/iter-slm296-verified-teacher-admission-20260725.json index 29240f5b3..bbdb67caa 100644 --- a/docs/design/iter-slm296-verified-teacher-admission-20260725.json +++ b/docs/design/iter-slm296-verified-teacher-admission-20260725.json @@ -233,15 +233,15 @@ ], "schema": "verified_teacher_admission_report/v1", "status": "yield_limitation_no_real_teacher_available", - "timestamp": "2026-07-25T07:48:18.733805Z", + "timestamp": "2026-07-25T14:37:31.142872Z", "version_stamp": { - "code_commit": "368395a2fa7d9642fc9497f17664bfd9b6adc56d", - "code_dirty": true, + "code_commit": "fa349f9bc23414bf568cbb53ef5c7200e9b83356", + "code_dirty": false, "components": { - "harness.experiments": "v106", + "harness.experiments": "v108", "harness.experiments.slm296_verified_teacher_admission": "v1" }, "stamp_schema": "version_stamp/v1", - "stamped_at": "2026-07-25T07:48:18.761255+00:00" + "stamped_at": "2026-07-25T14:37:31.142906+00:00" } } diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 1cd0a8b8e..c4f0f1369 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -5574,6 +5574,11 @@ "docs/design/iter-slm267-programspec-coverage-scaling-20260725.md" ], "history": [ + { + "version": "v1", + "date": "2026-07-25", + "note": "no-bump: reran the deterministic SLM-267 fixture during rebase validation; regenerated only its version-stamped JSON evidence" + }, { "version": "v1", "date": "2026-07-25", @@ -5594,6 +5599,11 @@ "docs/design/iter-slm296-verified-teacher-admission-20260725.md" ], "history": [ + { + "version": "v1", + "date": "2026-07-25", + "note": "no-bump: reran the deterministic SLM-296 fixture during rebase validation; regenerated only its version-stamped JSON evidence" + }, { "version": "v1", "date": "2026-07-25",