From 9420a7b2b86b51c2dbb03a8474d3dcf191b8f88c Mon Sep 17 00:00:00 2001 From: yashb98 Date: Mon, 20 Jul 2026 04:39:24 +0000 Subject: [PATCH 01/35] Harden the research loop: loop_state schema fork, thermal kill-switch, crash-survivable rungs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three long-standing blockers on the unattended loop, all execution-verified (411 passed / 1 skipped, green *while* a trainer is live — the case that matters). 1. loop_state schema fork (the last cron blocker). default_state() emitted a 7-key shape that diverged from the §C5 pinned 12-key one, so a fail-open recovery silently DROPPED the recovery-critical train_pid/ckpt_path/ resume_cmd instead of returning them as None — the recovery chain would KeyError exactly when it was needed. Now emits the full live schema; the dead fork keys (last_marker, iteration) and the unused advance(marker=) arg are gone. Three new tests pin the shape, the fail-open keys, and their survival across record_resume(). 2. sentinel thermal kill-switch. The box was hard-locking mid-run under sustained load (a ~16-18h 420M rung could never finish). watch now samples the GPU die temp, the hottest ACPI SoC zone, and — the definitive, self-calibrating signal — the GPU's own hw/sw thermal-slowdown flag, and SIGTERMs at >=90C or an active throttle after 3 consecutive samples (debounced against nvidia-smi blips). Fail-open throughout: an unreadable sensor never fabricates heat. The kill JSON gains trigger/gpu_temp_c/ soc_temp_c/gpu_throttling; heartbeats carry the temps. 3. Crash-survivable rungs. train_ablation.py checkpoints full training state (model + both optimizers + step + python/numpy/torch/cuda RNG, atomic write + fsync before rename) every --resume_every steps and auto-resumes, failing closed to a fresh start on an unreadable checkpoint; a run with no resume file is numerically identical to before. run_ladder.sh loops passes until every cell has its .done marker (a thermally-killed cell is re-attempted, resuming from its checkpoint, rather than the driver exiting after one pass), behind a bounded cool-down gate and an escalating hot-spell backoff, and smoke is now sentinel-watched like the cells. Also fixes a test-isolation bug found while running the gate beside the live HybridSSM trainer: test_boot_resume's _run() helper never set the BOOT_TRAINER_RE hook that boot_resume.sh documents "for hermetic tests only", so pgrep matched the REAL host trainer and three recovery-guard decisions degraded to already-running. The suite was therefore green-by-accident on an idle box and red during training — i.e. the recovery guards went unverified precisely when recovery matters. The later tests in that file already scoped their own pattern; the early helper was missed. Added a regression test that spawns a trainer-shaped process and asserts the decision is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../train_ablation.py | 71 +++++++++++- .../run_ladder.sh | 104 ++++++++++++++++-- README.md | 25 +++++ research/loop_state.py | 32 ++++-- research/tests/test_orchestration_chaos.py | 44 ++++++++ sentinel.py | 86 +++++++++++++-- 6 files changed, 330 insertions(+), 32 deletions(-) diff --git a/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/train_ablation.py b/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/train_ablation.py index 72439f8..a9c8956 100644 --- a/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/train_ablation.py +++ b/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/train_ablation.py @@ -22,7 +22,9 @@ import argparse import math +import os import pathlib +import random import sys import time @@ -33,6 +35,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) # local normuon.py import safe_cuda # noqa: E402 +import numpy as np # noqa: E402 import torch # noqa: E402 from torch.optim import AdamW # noqa: E402 from torch.utils.data import DataLoader # noqa: E402 @@ -85,6 +88,9 @@ def main(): ap.add_argument("--grad_clip", type=float, default=1.0) ap.add_argument("--mem_fraction", type=float, default=0.85) ap.add_argument("--log_every", type=int, default=20) + ap.add_argument("--resume_every", type=int, default=200, + help="save a full-state resume checkpoint every N optimizer steps " + "(crash-survival: the box hard-locks under load every ~5-10h)") ap.add_argument("--no_compile", action="store_true") ap.add_argument("--tag", default=None, help="override the output tag (default: _seed)") a = ap.parse_args() @@ -93,6 +99,7 @@ def main(): tag = a.tag if a.tag else f"{arm}_seed{a.seed}" log_path = RESULTS / f"{tag}.log" ckpt_path = RESULTS / f"checkpoint_{tag}.pt" + resume_path = RESULTS / f"resume_{tag}.pt" token_budget = a.steps * TOK_PER_STEP if not torch.cuda.is_available(): @@ -121,14 +128,64 @@ def main(): optims, n_2d, n_rest = build_optims(model, arm, a.peak_lr, a.normuon_lr, a.weight_decay) tq.log(f"[{tag}] param split: {n_2d} 2D->{arm} | {n_rest} rest->AdamW", log_path) - train_model = model if a.no_compile else torch.compile(model) - base_ppl, _ = tq.evaluate(model, val_tokens, device, SEQ_LEN) - tq.log(f"[{tag}] baseline (random-init) val PPL={base_ppl:.2f}", log_path) + # ---- crash-survival resume (model + optim state + step + RNG). The box hard-locks + # under sustained load every ~5-10h but a 420M rung needs ~16-18h, so progress MUST + # survive a reboot. Saved every --resume_every steps (atomic + fsync). A fresh run + # with no resume_.pt behaves EXACTLY as before (identical numerics). ---- + def save_resume(cur_step, cur_base_ppl, cur_t0): + payload = { + "step": cur_step, + "model": model.state_dict(), + "optims": [o.state_dict() for o, _ in optims], + "baseline_ppl": cur_base_ppl, + "elapsed": time.time() - cur_t0, + "rng": {"python": random.getstate(), "numpy": np.random.get_state(), + "torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state_all()}, + "meta": {"tag": tag, "arm": arm, "seed": a.seed, "total_steps": a.steps}, + } + tmp = resume_path.with_suffix(".pt.tmp") + with open(tmp, "wb") as f: + torch.save(payload, f) + f.flush(); os.fsync(f.fileno()) # durable BEFORE rename (box hard-locks) + os.replace(tmp, resume_path) # atomic swap: old ckpt intact until this + + resume_step, resume_elapsed, resumed = 0, 0.0, False + if resume_path.exists(): + try: + # load to CPU: RNG-state must stay a CPU ByteTensor for set_rng_state; model + # load_state_dict / optim load_state_dict move their tensors to the param + # device themselves. weights_only=False: our own ckpt holds RNG/optim objects. + ck = torch.load(resume_path, map_location="cpu", weights_only=False) + model.load_state_dict(ck["model"]) + for o, s in zip((o for o, _ in optims), ck["optims"]): + o.load_state_dict(s) + random.setstate(ck["rng"]["python"]); np.random.set_state(ck["rng"]["numpy"]) + torch.set_rng_state(ck["rng"]["torch"]); torch.cuda.set_rng_state_all(ck["rng"]["cuda"]) + resume_step = int(ck["step"]); base_ppl = ck["baseline_ppl"] + resume_elapsed = float(ck.get("elapsed", 0.0)); resumed = True + tq.log(f"[{tag}] RESUMED from step {resume_step}/{a.steps} " + f"(baseline PPL={base_ppl:.2f}, {resume_elapsed/3600:.2f}h prior compute)", log_path) + except Exception as e: # corrupt/incompatible -> fail closed to fresh + tq.log(f"[{tag}] resume ckpt unreadable ({e!r}); starting fresh", log_path) + resume_step, resume_elapsed, resumed = 0, 0.0, False - step, accum, t0 = 0, 0, time.time() + train_model = model if a.no_compile else torch.compile(model) + if not resumed: + base_ppl, _ = tq.evaluate(model, val_tokens, device, SEQ_LEN) + tq.log(f"[{tag}] baseline (random-init) val PPL={base_ppl:.2f}", log_path) + + # Return the baseline-eval's reserved allocator blocks to the unified pool BEFORE the + # first training step. Fresh runs otherwise stack ~40GB of step-1 logits on top of the + # ~48GB still-reserved eval pool (different shapes → not reused), a transient that hit + # ~88% of the shared pool and tripped sentinel's 0.80 kill (and neared the box's crash + # cliff). Resumed runs skip the eval, so this only helps fresh runs / the smoke; it is + # numerically inert (frees cached-but-unused memory only). (2026-07-11) + torch.cuda.empty_cache() + + step, accum, t0 = resume_step, 0, time.time() - resume_elapsed for o, _ in optims: o.zero_grad(set_to_none=True) - done = False + done = step >= a.steps while not done: for inp, lbl in loader: inp, lbl = inp.to(device, non_blocking=True), lbl.to(device, non_blocking=True) @@ -155,6 +212,8 @@ def main(): tq.log(f"[{tag}] step {step}/{a.steps} loss {loss.item():.4f} " f"lr {a.peak_lr*factor:.2e} |grad| {float(grad_norm):.2f} " f"tok/s {tps:,.0f} mem {torch.cuda.max_memory_allocated()/1e9:.1f}GB", log_path) + if step % a.resume_every == 0: + save_resume(step, base_ppl, t0) # crash-survival; a kill just dies fast (frees pool) if step >= a.steps: done = True break @@ -173,6 +232,8 @@ def main(): "total_steps": a.steps}, }, ckpt_path) tq.log(f"[{tag}] saved {ckpt_path}", log_path) + if resume_path.exists(): + os.remove(resume_path) # rung complete: resume ckpt no longer needed return 0 diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder.sh index ef5e12d..f7204b1 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder.sh @@ -16,8 +16,36 @@ TRAIN=$IMU1/train_ablation.py RESULTS=$IMU1/results LOG=$LDIR/run_ladder.log PY=python3 +COOL_C=70 # 2026-07-08: user heat-crash hypothesis — cool the box below this (C) before each cell mkdir -p "$LDIR" +# Hottest of the GPU die + all ACPI SoC zones, whole deg C (empty if unreadable). No sudo. +hottest_c () { + local g z zc max="" + g=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -dc '0-9') + [ -n "$g" ] && max=$g + for z in /sys/class/thermal/thermal_zone*/temp; do + [ -r "$z" ] || continue + zc=$(( $(cat "$z" 2>/dev/null || echo 0) / 1000 )) + { [ -z "$max" ] || [ "$zc" -gt "$max" ]; } && max=$zc + done + echo "$max" +} + +# Bounded, fail-open cool-down: wait up to ~30 min for the box to drop below COOL_C before +# launching a cell (implements the heat-crash mitigation). Unreadable temp => proceed. +cool_down () { + local tag=$1 h + for _ in $(seq 1 60); do + h=$(hottest_c) + [ -z "$h" ] && { echo "[$(date '+%T')] [cooldown] $tag: temp unreadable, proceeding"; return 0; } + [ "$h" -lt "$COOL_C" ] && { echo "[$(date '+%T')] [cooldown] $tag: ${h}C < ${COOL_C}C, launching"; return 0; } + echo "[$(date '+%T')] [cooldown] $tag: ${h}C >= ${COOL_C}C, waiting 30s for cool-down" + sleep 30 + done + echo "[$(date '+%T')] [cooldown] $tag: still >= ${COOL_C}C after 30min — launching anyway (bounded)" +} + # CORE cells: " ". 168M=2564 steps (3 seeds/arm), 420M=6409 steps (2 seeds/arm). CELLS=( "2564 adamw 0" "2564 normuon 0" "2564 adamw 1" "2564 normuon 1" "2564 adamw 2" "2564 normuon 2" @@ -29,10 +57,11 @@ run_cell () { local budgetM=$(( steps * 65536 / 1000000 )) local tag=persist_${budgetM}M_${arm}_s${seed} [ -f "$LDIR/${tag}.done" ] && { echo "[$(date '+%T')] [skip] $tag"; return 0; } + cool_down "$tag" # 2026-07-08: don't launch onto a hot box (heat-crash mitigation) # pool headroom (unified memory shared; wait for >=60 GB free) for _ in $(seq 1 90); do a=$(free -g | awk '/Mem:/{print $7}'); [ "${a:-0}" -ge 60 ] && break; sleep 10; done echo "[$(date '+%F %T')] START $tag steps=$steps (~$(( steps * 65536 / 1000000 ))M tok)" - $PY "$TRAIN" --optimizer "$arm" --seed "$seed" --steps "$steps" --tag "$tag" \ + $PY "$TRAIN" --optimizer "$arm" --seed "$seed" --steps "$steps" --tag "$tag" --resume_every 100 \ >> "$RESULTS/${tag}.out" 2>&1 & local tpid=$! $PY "$ROOT/sentinel.py" watch --pid "$tpid" --kill-at 0.80 --log "$LDIR/sentinel_${tag}.log" \ @@ -41,9 +70,9 @@ run_cell () { wait "$tpid"; local rc=$? kill "$spid" 2>/dev/null if [ $rc -eq 0 ] && [ -f "$RESULTS/checkpoint_${tag}.pt" ]; then - touch "$LDIR/${tag}.done"; echo "[$(date '+%T')] [done] $tag" + touch "$LDIR/${tag}.done"; echo "[$(date '+%T')] [done] $tag"; return 0 else - echo "[$(date '+%T')] [FAIL rc=$rc] $tag — will retry on next driver pass" + echo "[$(date '+%T')] [FAIL rc=$rc] $tag — will retry on next driver pass"; return 1 fi } @@ -57,17 +86,68 @@ for _ in $(seq 1 360); do pgrep -f "train_grpo.py|run_phase1_passk.py|train_abla sleep 5 # 3) preflight (§C6) $PY "$ROOT/sentinel.py" preflight || { echo "preflight FAIL — abort"; exit 1; } -# 4) SMOKE (§C5.0): 1 step, no compile, confirms model build + data + step + checkpoint save +# 4) SMOKE (§C5.0): 1 step, no compile, confirms model build + data + step + checkpoint save. +# Watched by sentinel (§C6) like the cells — it is an unattended GPU launch. echo "[$(date '+%T')] smoke: train_ablation.py --steps 1" -$PY "$TRAIN" --optimizer adamw --seed 0 --steps 1 --no_compile --tag smoke_ladder >> "$LDIR/smoke.log" 2>&1 -if [ $? -ne 0 ] || [ ! -f "$RESULTS/checkpoint_smoke_ladder.pt" ]; then +$PY "$TRAIN" --optimizer adamw --seed 0 --steps 1 --no_compile --tag smoke_ladder >> "$LDIR/smoke.log" 2>&1 & +smpid=$! +$PY "$ROOT/sentinel.py" watch --pid "$smpid" --kill-at 0.80 --log "$LDIR/sentinel_smoke_ladder.log" >/dev/null 2>&1 & +smwatch=$! +wait "$smpid"; smrc=$? +kill "$smwatch" 2>/dev/null +if [ $smrc -ne 0 ] || [ ! -f "$RESULTS/checkpoint_smoke_ladder.pt" ]; then echo "SMOKE FAILED — abort (see $LDIR/smoke.log)"; exit 1 fi rm -f "$RESULTS/checkpoint_smoke_ladder.pt"; echo "[$(date '+%T')] smoke OK" -# 5) train the core cells (one at a time; .done markers make this idempotent/resumable) -for c in "${CELLS[@]}"; do run_cell $c; done -touch "$LDIR/ladder.done" -echo "===== $(date '+%F %T') CORE LADDER COMPLETE — $LDIR/ladder.done =====" -# 6) score if a scorer is present (armed separately, like the RLVR autoscore) -[ -f "$LDIR/score_ladder.py" ] && $PY "$LDIR/score_ladder.py" >> "$LOG" 2>&1 +# 5) train the core cells (one at a time; .done markers make this idempotent/resumable). +# 2026-07-09 fix: LOOP over passes so a thermally-killed cell is RE-ATTEMPTED (resuming from its +# checkpoint via --resume_every) instead of the driver quitting after ONE pass (the 07-09 stall: +# midday heat SIGTERM'd all 4 cells in a row, then the driver exited and nothing ran for ~17h). +# Each pass skips .done cells instantly and cools down before every (re)attempt. Bounded by MAXPASS. +pass=0; MAXPASS=100; hot_backoff=0; prev_missing=999 +while : ; do + fails=0 + for c in "${CELLS[@]}"; do run_cell $c || fails=$((fails+1)); done + missing=0 + for c in "${CELLS[@]}"; do + set -- $c; s=$1; arm=$2; seed=$3; bM=$(( s * 65536 / 1000000 )) + [ -f "$LDIR/persist_${bM}M_${arm}_s${seed}.done" ] || missing=$((missing+1)) + done + [ "$missing" -eq 0 ] && break + pass=$((pass+1)) + [ "$pass" -ge "$MAXPASS" ] && { echo "[$(date '+%T')] MAXPASS=$MAXPASS reached, $missing cells incomplete — stopping (resume ckpts preserved; re-run to continue)"; break; } + # HOT-SPELL BACKOFF (2026-07-10): if a whole pass FAILED with NO cell completing (missing didn't + # drop) and there were failures, the box is too hot to make progress — during peak-heat spikes a + # cell reheats to 90C and thermal-dies in <20 steps, thrashing the GPU at ~0 net progress. Wait an + # escalating (5→30 min cap) cool-off for a cooler window instead of re-attempting every ~3 min. + # Any completion (missing drops) resets the backoff so cool hours run at full speed. + if [ "$fails" -gt 0 ] && [ "$missing" -ge "$prev_missing" ]; then + hot_backoff=$(( hot_backoff + 1 )); wait_s=$(( hot_backoff * 300 )); [ "$wait_s" -gt 1800 ] && wait_s=1800 + echo "[$(date '+%F %T')] pass $pass: $missing/${#CELLS[@]} incomplete, no progress ($fails failed — box likely too hot) — backing off ${wait_s}s for a cooler window" + sleep "$wait_s" + else + hot_backoff=0 + echo "[$(date '+%F %T')] pass $pass: $missing/${#CELLS[@]} cells incomplete ($fails failed) — re-attempting (cells resume from checkpoint)" + fi + prev_missing=$missing +done +# 5b) COMPLETION GATE (2026-07-07 fix): only mark the ladder complete + score when EVERY cell +# has its .done marker (run_cell writes .done only on rc==0 AND a saved checkpoint). Stops a +# failed cell from producing a FALSE 'CORE LADDER COMPLETE' + a scorer/ledger run over an +# incomplete ladder (the 09:39 bug: all 4x420M failed on a DNS blip yet ladder.done was set). +missing=0 +for c in "${CELLS[@]}"; do + set -- $c; s=$1; arm=$2; seed=$3; bM=$(( s * 65536 / 1000000 )) + [ -f "$LDIR/persist_${bM}M_${arm}_s${seed}.done" ] || missing=$((missing+1)) +done +if [ "$missing" -eq 0 ]; then + touch "$LDIR/ladder.done" + echo "===== $(date '+%F %T') CORE LADDER COMPLETE — $LDIR/ladder.done =====" + # 6) score only when the ladder is genuinely complete (all cells .done) + [ -f "$LDIR/score_ladder.py" ] && $PY "$LDIR/score_ladder.py" >> "$LOG" 2>&1 +else + rm -f "$LDIR/ladder.done" + echo "===== $(date '+%F %T') LADDER INCOMPLETE — $missing/${#CELLS[@]} cells missing .done ($fails failed this pass); NOT scoring, ladder.done cleared =====" + exit 1 +fi } >> "$LOG" 2>&1 diff --git a/README.md b/README.md index a945719..9e9f236 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,30 @@ with the Chen-2021 estimator, on decontaminated GSM8K + MATH-500) and tested it. The reasoning capability lives in SFT/distillation, not RL at this scale — the gate saved a multi-seed cohort before it was spent. +**7 · Scaling persistence of the NorMuon win (in progress).** Study #2 attributed +the IMU-1 win largely to **NorMuon**; this ladder asks whether its **+0.474 +wikitext BPB** edge over AdamW **persists or converges with budget**. At fixed +N=596M it sweeps the token budget — 42M (reused) + **168M ×{NorMuon,AdamW}×3 seeds** ++ **420M ×2 seeds** — varying only `--steps`. The six 168M rungs are **done**; the +four 420M rungs (~16–18 h each) are **running**. Honest ceiling: **directional** — +the 420M top rung is n=2 (< 3 seeds); a headline needs a 3rd seed and/or an 840M rung. + +> **GB10 thermal-survival note (2026-07-08→10).** The 420M rungs surfaced a hardware +> reality: under sustained load in warm ambient the unified Grace-Blackwell package +> **overheats to 92–94 °C with the GPU thermal-throttling**, which had been silently +> hard-locking the whole box mid-run (a ~16–18 h rung could never finish, restarting +> from step 0 each crash). The run is now **crash-survivable**: `train_ablation.py` +> checkpoints full training state (model + both optimizers + step + RNG, atomic+fsync) +> every 100 steps and **auto-resumes** from it; `sentinel.py watch` gained a **thermal +> kill-switch** (SIGTERM at ≥90 °C or a hardware throttle flag — which also *prevents* +> the hard-locks by shedding load before the box wedges); a dense `thermal_log.py` +> records the full temperature envelope; and `run_ladder.sh` loops-until-done behind a +> cool-down gate, backed by an `@reboot` auto-resume (`boot_resume.sh`). Net: the +> multi-day ladder now survives each thermal event by losing ≤~15 min (one checkpoint +> interval) instead of a whole rung. Cooling the box's ambient is the high-leverage +> throughput fix — cooling *time* after a kill is only ~10–30 s; it's the ~3 min reheat +> to 90 °C under warm ambient that throttles daytime throughput. + Each study lives under `Qwen3-0.6B/experiments/__/` with its methodology (`c5_evidence.json`), results (`verdict.json` / `reasoning_verdict.json`), and a per-run record in `research/ledger/runs/`. @@ -165,6 +189,7 @@ BuildFromScratch/ │ ├── experiments/ # single-variable studies (arch / optimizer / data / post-training) │ └── README.md # the long-form study writeup ├── safe_cuda.py # GB10 unified-memory guard (caps the CUDA process) +├── sentinel.py # resource watchdog — preflight / watch (memory + thermal kill-switch) / liveness ├── jax_safe_env.py # JAX preallocation guard for the shared-memory box ├── mfu_meter.py # MFU / HFU + achieved-TFLOPS (honest, GB10 peak flagged estimated) └── flop_accounting.py # FLOP-per-token (6N + 12·L·H·Q·T) — feeds the iso-FLOP gate diff --git a/research/loop_state.py b/research/loop_state.py index bb9b207..05f26f7 100644 --- a/research/loop_state.py +++ b/research/loop_state.py @@ -36,10 +36,26 @@ MAX_AUTO_RESUMES = 2 +# Recovery-critical flat in-flight fields (§C5) — the ones /ablation-runner writes +# and the recovery chain reads to re-adopt a killed trainer. A fresh/recovered +# default carries them as None (= "no in-flight run"), so a fail-open read is +# schema-shaped like a live file instead of silently dropping these keys. +IN_FLIGHT_FIELDS = ("in_flight_run", "train_pid", "ckpt_path", "resume_cmd") + + def default_state() -> dict: - return {"schema_version": SCHEMA_VERSION, "stage": "S0", - "in_flight_run": None, "auto_resumes": 0, "iteration": 0, - "updated": None, "last_marker": None} + """The §C5 pinned bootstrap schema, verbatim from research-loop/SKILL.md + (+ `updated`, which the writer sets on the first advance). Emitting the FULL + live 12-key shape here is what closes the schema fork: a fail-open recovery + (missing/corrupt file) now yields the same keys a live file has — in + particular the recovery-critical `train_pid`/`ckpt_path`/`resume_cmd` are + present as None rather than absent, so downstream recovery reads a clean + "nothing to resume" instead of KeyError-ing on a divergent 7-key default.""" + return {"schema_version": SCHEMA_VERSION, "iteration_date": None, + "stage": "S0", "in_flight_run": None, "train_pid": None, + "ckpt_path": None, "resume_cmd": None, "auto_resumes": 0, + "last_radar": None, "objective": "any", "notes": "", + "updated": None} def load(path) -> dict: @@ -83,9 +99,11 @@ def save(path, state: dict) -> None: raise -def advance(path, stage: str, ts: str | None = None, in_flight=..., marker=...) -> dict: - """Move to `stage` and persist. `in_flight`/`marker` updated only if passed - (sentinel `...` means leave unchanged).""" +def advance(path, stage: str, ts: str | None = None, in_flight=...) -> dict: + """Move to `stage` and persist. `in_flight` updated only if passed + (sentinel `...` means leave unchanged). All other live keys — including the + recovery-critical `train_pid`/`ckpt_path`/`resume_cmd` — are preserved + verbatim across the transition (load returns a valid file as-is).""" if stage not in STAGES: raise ValueError(f"unknown stage {stage!r}") st = load(path) @@ -94,8 +112,6 @@ def advance(path, stage: str, ts: str | None = None, in_flight=..., marker=...) st["updated"] = ts if in_flight is not ...: st["in_flight_run"] = in_flight - if marker is not ...: - st["last_marker"] = marker save(path, st) return st diff --git a/research/tests/test_orchestration_chaos.py b/research/tests/test_orchestration_chaos.py index 546b9ae..7cb9e1f 100644 --- a/research/tests/test_orchestration_chaos.py +++ b/research/tests/test_orchestration_chaos.py @@ -21,6 +21,50 @@ def test_corrupt_state_fails_open(tmp_path): assert st["_recovered"] is True and st["stage"] == "S0" +def test_default_state_carries_full_live_schema(tmp_path): + """Schema-fork guard: default_state() must emit the §C5 pinned 12-key shape, + NOT a divergent 7-key one. In particular the recovery-critical flat in-flight + fields must be present (as None) so a fail-open recovery is schema-shaped.""" + d = ls.default_state() + for k in ("schema_version", "iteration_date", "stage", "in_flight_run", + "train_pid", "ckpt_path", "resume_cmd", "auto_resumes", + "last_radar", "objective", "notes", "updated"): + assert k in d, f"default_state missing canonical key {k!r}" + for k in ls.IN_FLIGHT_FIELDS: + assert d[k] is None # no in-flight run on a fresh/recovered default + assert d["objective"] == "any" # §C13 selection filter must not be defeated + assert "last_marker" not in d and "iteration" not in d # dead fork keys gone + + +def test_fail_open_recovery_has_recovery_keys(tmp_path): + """A corrupt state recovered mid-flight must still expose the recovery keys + as None (a clean 'nothing to resume'), never KeyError on a 7-key default.""" + p = tmp_path / "loop_state.json" + p.write_text("}{ truncated garbage") + st = ls.load(p) + assert st["_recovered"] is True + assert st["train_pid"] is None and st["ckpt_path"] is None and st["resume_cmd"] is None + + +def test_in_flight_fields_survive_resume_accounting(tmp_path): + """The dead-trainer case: a VALID state carrying a live trainer's pid/ckpt/ + resume_cmd must preserve all three across record_resume — the recovery chain + re-adopts the trainer from exactly these fields.""" + p = tmp_path / "loop_state.json" + st = ls.default_state() + st.update(stage="S7", in_flight_run="2026-07-19_qwen3_x", + train_pid=424242, ckpt_path="/x/exp/ckpt_step_900.pt", + resume_cmd="python train_ablation.py --resume /x/exp/ckpt_step_900.pt") + ls.save(p, st) + ls.record_resume(p, cap=2) # one dead-run recovery attempt + back = ls.load(p) + assert back["train_pid"] == 424242 + assert back["ckpt_path"] == "/x/exp/ckpt_step_900.pt" + assert back["resume_cmd"].endswith("ckpt_step_900.pt") + assert back["in_flight_run"] == "2026-07-19_qwen3_x" + assert back["auto_resumes"] == 1 # accounting still advanced + + def test_missing_state_fails_open(tmp_path): st = ls.load(tmp_path / "absent.json") assert st["_recovered"] is True and st["stage"] == "S0" diff --git a/sentinel.py b/sentinel.py index 27760b3..0d31193 100644 --- a/sentinel.py +++ b/sentinel.py @@ -86,6 +86,15 @@ DEFAULT_INTERVAL = 30.0 # watch: sample period, seconds DEFAULT_GRACE = 60.0 # watch: SIGTERM -> SIGKILL grace, seconds TRAINER_PATTERN = "train" # §C5.1: pgrep -af train +# Thermal guard (added 2026-07-08: box may be hard-locking under sustained load from +# heat). Primary signal is the GPU's OWN hw/sw thermal-slowdown flag (self-calibrating: +# it fires at the real ~95C hardware ceiling), backstopped by a conservative absolute +# ceiling. Set HIGH on purpose — load temps were never instrumented, so a low ceiling +# would false-kill healthy runs. Tune down only after the dense thermal log shows the +# actual load envelope. Both are debounced to avoid killing on a single nvidia-smi blip. +TEMP_KILL_C = 90.0 # watch: kill if GPU die or hottest SoC zone >= this +TEMP_WARN_C = 82.0 # watch: log a WARN above this (still training) +TEMP_KILL_CONSECUTIVE = 3 # debounce: N consecutive over-limit samples before a kill def utcnow() -> str: @@ -333,6 +342,44 @@ def pid_rss_gib(pid: int): return None # gone, or a zombie (zombies have no VmRSS line) +def gpu_thermal(): + """(gpu_die_temp_C, throttling_bool) from nvidia-smi, or (None, False) if unreadable. + throttling = GPU firmware reports hw or sw thermal slowdown ACTIVE (the definitive, + self-calibrating 'too hot' signal). Stdlib-only; fail-open (never fabricates heat).""" + try: + out = subprocess.run( + ["nvidia-smi", + "--query-gpu=temperature.gpu,clocks_throttle_reasons.hw_thermal_slowdown," + "clocks_throttle_reasons.sw_thermal_slowdown", + "--format=csv,noheader"], + capture_output=True, text=True, timeout=10, + ).stdout + except (OSError, subprocess.TimeoutExpired): + return None, False + for line in out.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 3 and parts[0].isdigit(): + throttling = (parts[1] == "Active") or (parts[2] == "Active") # "Not Active" != "Active" + return float(parts[0]), throttling + return None, False + + +def hottest_soc_c(): + """Hottest SoC/board ACPI thermal-zone temp (C), or None. On the GB10 superchip the + Grace CPU + Blackwell GPU share one package, so these track SoC heat next to the + GPU-die sensor. Fail-open (unreadable -> None, never a fake high reading).""" + import glob + hottest = None + for p in glob.glob("/sys/class/thermal/thermal_zone*/temp"): + try: + with open(p) as f: + v = int(f.read().strip()) / 1000.0 + except (OSError, ValueError): + continue + hottest = v if hottest is None else max(hottest, v) + return hottest + + def watch(pid, kill_at, log_path, interval, grace, marker_path=None) -> int: marker = Path(marker_path) if marker_path else MARKER logf = open(log_path, "a", buffering=1) if log_path else None @@ -349,6 +396,7 @@ def log(msg): f"grace={grace:g}s start_ticks={start_ticks} marker={marker}" ) samples = 0 + hot_streak = 0 while True: if not watched_alive(pid, start_ticks): log(f"watched pid {pid} exited on its own; disarming (no kill)") @@ -362,12 +410,32 @@ def log(msg): usage = (total_kib - avail_kib) / total_kib rss_gib = pid_rss_gib(pid) # §C6: free + process RSS every sample rss_s = f"{rss_gib:.1f} GiB" if rss_gib is not None else "n/a" - if usage >= kill_at: - reason = ( - f"pool usage {usage:.1%} >= kill-at {kill_at:.0%} " - f"(MemAvailable {avail_kib / 2**20:.1f} GiB of " - f"{total_kib / 2**20:.1f} GiB; trainer rss {rss_s})" - ) + gpu_t, throttling = gpu_thermal() # thermal guard (fail-open: None => no kill) + soc_t = hottest_soc_c() + hot = max([t for t in (gpu_t, soc_t) if t is not None], default=None) + temp_s = ((f"gpu {gpu_t:.0f}C" if gpu_t is not None else "gpu n/a") + + (f" soc {soc_t:.0f}C" if soc_t is not None else " soc n/a") + + (" THROTTLING" if throttling else "")) + danger = throttling or (hot is not None and hot >= TEMP_KILL_C) + hot_streak = hot_streak + 1 if danger else 0 + thermal_kill = hot_streak >= TEMP_KILL_CONSECUTIVE + if hot is not None and hot >= TEMP_WARN_C and not thermal_kill: + log(f"WARN: {temp_s} (hot={hot:.0f}C >= warn {TEMP_WARN_C:.0f}C) " + f"[danger streak {hot_streak}/{TEMP_KILL_CONSECUTIVE}]") + if usage >= kill_at or thermal_kill: + if thermal_kill: + trigger = "thermal" + reason = ( + f"{temp_s} at/over thermal limit for {hot_streak} consecutive samples " + f"(>= {TEMP_KILL_C:.0f}C or hw/sw throttle; pool {usage:.1%}, rss {rss_s})" + ) + else: + trigger = "memory" + reason = ( + f"pool usage {usage:.1%} >= kill-at {kill_at:.0%} " + f"(MemAvailable {avail_kib / 2**20:.1f} GiB of " + f"{total_kib / 2**20:.1f} GiB; trainer rss {rss_s}; {temp_s})" + ) if not watched_alive(pid, start_ticks): # re-verify identity log(f"watched pid {pid} exited on its own; disarming (no kill)") return 0 @@ -397,7 +465,11 @@ def log(msg): payload = { "time": utcnow(), "killed_pid": pid, + "trigger": trigger, "reason": reason, + "gpu_temp_c": gpu_t, + "soc_temp_c": soc_t, + "gpu_throttling": throttling, "pool_usage": round(usage, 4), "pool_total_gb": round(total_kib / 2**20, 1), "trainer_rss_gb": ( @@ -415,7 +487,7 @@ def log(msg): return 3 samples += 1 if samples % 20 == 0: # heartbeat every ~10 min at default interval - log(f"heartbeat: pool usage {usage:.1%}, pid {pid} alive, rss {rss_s}") + log(f"heartbeat: pool usage {usage:.1%}, pid {pid} alive, rss {rss_s}, {temp_s}") time.sleep(interval) From 1803b5bb25803d43580218cbc3d2d9985e2efee9 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Mon, 20 Jul 2026 04:40:30 +0000 Subject: [PATCH 02/35] HybridSSM-0.2B: the repo's first novel from-scratch model (JAX/Flax), first pretrain arm in flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hybrid attention-SSM LM written from a blank file in JAX/Flax — the standing "next model in JAX" request, and the first build here with no bit-exact oracle (novel design → the gate is a numerical cross-check at ~1e-2, §C14). Architecture (ARCHITECTURE.md): d=768, 24 layers in a 1:1 [full-attn, efficient] interleave, GQA 12/4 head_dim 64 + RoPE, Mamba-2-style selective SSM via associative_scan on the efficient layers, SwiGLU, RMSNorm, tied embedding over the Qwen3 151,936 vocab, chunked CE. 189.1M non-embed / 305.8M total. The mixer type / attention fraction / NoPE toggles are what make it a study rather than a model: the headline object is an emergence-speed curve, not a single number. Guards (§C1) verified in-code, not asserted: jax_safe_env imports before jax (train_hybrid.py:10 vs :15), and chunked_cross_entropy (model.py:136) does a streaming max + sumexp over vocab chunks so the (N, 151936) logit matrix is never materialized — the exact shape that hard-crashed the box in June. Arm ssm_base_s0 is training on real data (FineWeb-Edu sample-10BT via the Qwen3 tokcache, 170,034,304 train + 300,000 val tokens, seed 0 — the ~168M rung of the brief's token-budget ladder, and BPB-comparable to the 596M study). It survived a real incident: the sentinel killed it at step 580 when the SSM scan + chunked CE under autodiff held ~61 GB (pool 81.3% >= the 0.80 line, no thermal component); nn.remat on the decoder block (model.py:129) plus batch 8->4 took allocation to 16.6 GB, and it resumed from the step-400 checkpoint. That was a manual recovery behind a config change — the "not safe to auto-resume at the same config" path — so no auto-resume budget was consumed. c5_evidence.json is included and is honest about its own provenance: it was RECONSTRUCTED after the launch, because the 2026-07-19 launch created the ledger run entry without ever writing it, so §C5's "evidence recorded BEFORE launch" was not met for this run. Every item carries a src tag — log / derived / attestation / not-captured — rather than a uniform claim of compliance. Two items are weak and say so: the §C5.0 smoke numbers survive only as prose in BUILD_STATUS.md with no captured log, and the pre-launch concurrency check left no evidence at all. Recorded, not buried: verify.py last ran 2026-07-19 12:52 and model.py changed at 22:27 to add nn.remat, so the verify gate has NOT been re-run against the model that is actually training. nn.remat is semantically identity and the loss curve is continuous across the resume, but that is corroboration, not verification — re-run verify.py and capture verify.log before this arm is scored (GPU work, so it waits for the arm to finish per the one-job rule). The arm as launched also deviates from ARCHITECTURE.md: seq 2048 not 4096, and plain AdamW not Muon, since the JAX Muon port is unwritten — which makes AdamW @ seq 2048 the ladder's baseline recipe. No result is claimed. n=1 seed, no comparand, no iso-FLOP match: this is a baseline datum, capped at directional at absolute best. Co-Authored-By: Claude Opus 4.8 (1M context) --- HybridSSM-0.2B/.gitignore | 10 + .../ARCHITECTURE.md | 55 ++++++ .../BUILD_STATUS.md | 78 ++++++++ .../c5_evidence.json | 117 ++++++++++++ .../2026-07-19_hybrid-ssm-0.2b_build/model.py | 168 +++++++++++++++++ .../2026-07-19_hybrid-ssm-0.2b_build/ssm.py | 104 +++++++++++ .../2026-07-19_hybrid-ssm-0.2b_build/train.py | 175 ++++++++++++++++++ .../train_hybrid.py | 175 ++++++++++++++++++ .../verify.py | 79 ++++++++ 9 files changed, 961 insertions(+) create mode 100644 HybridSSM-0.2B/.gitignore create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ARCHITECTURE.md create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ssm.py create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py create mode 100644 HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/verify.py diff --git a/HybridSSM-0.2B/.gitignore b/HybridSSM-0.2B/.gitignore new file mode 100644 index 0000000..54ed0e2 --- /dev/null +++ b/HybridSSM-0.2B/.gitignore @@ -0,0 +1,10 @@ +# HybridSSM-0.2B: commit code + docs only; exclude training artifacts +*.pkl +*.pkl.tmp +*.log +*.stdout +__pycache__/ +*.pyc +*.thermal.log +probe*.log +sentinel_kill_*.json diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ARCHITECTURE.md b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ARCHITECTURE.md new file mode 100644 index 0000000..75bb596 --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ARCHITECTURE.md @@ -0,0 +1,55 @@ +# HybridSSM-0.2B — architecture design (novel from-scratch build, JAX/Flax) + +**Purpose:** study the attention-vs-efficient-mixer composition (brief `hybrid-attention-rethink`, +arXiv 2606.15378) on a single GB10, on this repo's evidence standard. Novel design → **no bit-exact +oracle**; the verify gate is numerical cross-check vs an independent reference at ~1e-2 (§C14/JAX). +Framework **JAX/Flax** (user-confirmed 2026-07-19; installed + verified, `associative_scan` runs). + +## Config (base arm) + +| field | value | why | +|---|---|---| +| d_model | 768 | ~146M non-embed at L=24 (paper S4 104M < this < S5 477M) | +| n_layers | 24 | 1:1 interleave → 12 full-attention + 12 efficient-mixer | +| layer pattern | `[full, eff, full, eff, …]` | paper's 1:1 main setting (1:3 ≈ same val loss is a later arm) | +| full-attn | GQA n_heads=12, n_kv=4, head_dim=64, RoPE (θ=1e4) | Qwen3-family attention, shrunk | +| efficient mixer | **Mamba-2-style selective SSM** via `associative_scan` (diagonal linear recurrence + input/gate proj) | JAX-native scan; the SWA-128 and GatedDeltaNet variants are ablation arms | +| MLP | SwiGLU, intermediate 2048 | Qwen3 recipe | +| norm | RMSNorm (eps 1e-6), pre-norm | Qwen3 recipe | +| vocab / tokenizer | **151,936 (Qwen3-0.6B-Base)** | reuses the validated text-lm-v2 data + eval pipeline; BPB-comparable to the 596M study. Report NON-EMBED params (paper convention). | +| tied embedding | yes | 117M embed counted once | +| CE | **chunked** over vocab (152k > 64k, §C1) | never materialize (N,152k) logits — the box-crash vector | +| seq_len (pretrain) | **4096** (probe decides; 16K is the paper's, memory-tight here) | emergence-speed + relative-hybrid comparison is visible at 4K; long-context/NoPE finding needs a later ctx-extension arm | +| optimizer | Muon (2D weights) + AdamW (1D/embed) — JAX port of `normuon.py` | paper uses Muon; repo has the PyTorch impl to port + cross-check | +| precision | bf16 compute, fp32 master/optimizer state | Qwen3 recipe | + +## The toggles that make it a STUDY (single-variable ablation matrix) + +Each is one flag on the base arm; iso-FLOP where the flag changes params (≤5%, §C18); ≥3 seeds; BPB +CIs on wikitext-2 + code (text-lm-v2) + a long-context retrieval probe (RULER-NIAH-style): + +1. **mixer type** on the efficient layers: `ssm` (Mamba-2) vs `swa128` (sliding-window attn, w=128) vs + `full` (all-attention control = the dense baseline, comparable to the 596M study) vs `none` (all-SSM). +2. **attention fraction / placement**: 1:1 vs 1:3 (one full-attn per three efficient) — the "how much + attention does a hybrid need" curve. +3. **NoPE-on-full-attention** (the paper's headline design knob): RoPE vs NoPE on the full-attn layers of + the SWA-128 hybrid — predicted long-context gain, ~zero short-context cost. +4. **token-budget ladder** (the emergence-speed instrument, mirroring the scaling-persistence study): score + each hybrid at increasing budgets → does the efficient-mixer choice affect emergence SPEED but converge? + +Headline object = the **emergence-speed curve** (quality vs tokens per hybrid) + a box-scale validation of +NoPE-on-full-attn. This is the same *shape* as "the disappearing win" — a coherent next chapter. + +## Files (novel design → its own folder, not canonical) + +- `model.py` — the JAX/Flax hybrid model (implemented from blank). +- `ssm.py` — the selective-SSM mixer (associative_scan) + the SWA mixer. +- `muon_jax.py` — JAX Muon (ported + cross-checked vs `normuon.py`). +- `verify.py` — numerical cross-check vs an independent reference (~1e-2) + shape/dtype grid. +- `train.py` — training loop (chunked CE, safe guards, ckpt/resume for the recovery chain). + +## Guards (§C1) + +`import jax_safe_env` BEFORE `import jax` (PREALLOCATE=false, MEM_FRACTION=0.5). Chunked CE for the 152k +vocab. sentinel preflight before any GPU work; sentinel watch + the hardened `boot_resume.sh` recovery +chain + thermal kill beside any unattended trainer. diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md new file mode 100644 index 0000000..6a1b4d9 --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md @@ -0,0 +1,78 @@ +# HybridSSM-0.2B — build status (updated 2026-07-20 04:15 UTC) + +**Novel from-scratch hybrid attention-SSM LM in JAX/Flax is BUILT, VERIFIED, SMOKE-PASSED — and the +first pretrain arm (`ssm_base_s0`) is IN FLIGHT on real data.** + +## Build phase — done ✓ (2026-07-19, all correctness-gated) + +- **Architecture** (`ARCHITECTURE.md`): d=768, 24 layers (1:1 full-attn:efficient interleave), GQA 12/4, + SwiGLU, RMSNorm, RoPE↔NoPE toggle, Qwen3 152k tokenizer, chunked CE. Design doc estimated ~146M + non-embed; the built model reports **189.1M non-embed / 305.8M total** (`[build]` line of every run + log) — tied embedding = 151,936 × 768 = 116.7M counted once. +- **Implementation** (`ssm.py`, `model.py`): SelectiveSSM (Mamba-2-style diagonal scan via + `associative_scan`) + SlidingWindowAttention + GQA attention + the full hybrid decoder. Written from blank. +- **Verify gate** (`verify.py` → **PASS**): SSM parallel-scan == sequential reference (max|Δ|=2.4e-7); + chunked CE == naive CE (|Δ|=4.8e-5, never materializes the 152k logits); param count sane; all 8 + ablation toggles forward-finite; forward deterministic. ⚠️ **See "Open gate gap" below — this PASS + predates the `nn.remat` memory fix and has not been re-run since.** +- **Smoke** (`train.py --smoke` → **PASS**, all variants): SSM / SWA-128+NoPE / 1:3-attention each overfit + a fixed batch 8.8 → ~0.003 loss (forward+backward+AdamW+chunked-CE all learn), grad norms healthy + (33 → 0.03), checkpoint save→reload exact (max|Δ|=0.0 — recovery-chain ready). Smoke used synthetic data. +- **Fit probes on real data** (`probe.log` / `probe2.log` / `probe3.log`, 15 / 12 / 30 steps): + step-0 loss 12.4317 / 12.4312 / 12.4312 ≈ ln(151936)=11.93 + init noise, and 30 steps moves 12.43 → 8.42. + +## Pretrain arm `ssm_base_s0` — IN FLIGHT + +Ledger run `2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0` (type=ablation, status=running, +lifecycle_stage=architecture, framework=jax, technique `hybrid-attention-rethink`). + +| field | value | source | +|---|---|---| +| data | FineWeb-Edu sample-10BT, Qwen3-0.6B-Base tokenizer, **170,034,304 train + 300,000 val** tokens, seed 0 | `tokcache_170034304_300000_seed0_Qwen3-0.6B-Base.pt`, built by `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:151` | +| config as launched | seq **2048**, batch **4**, 20,756 steps × 8,192 tok/step, AdamW lr 3e-3, warmup 200 | live cmdline of PID 3164922 | +| trainer / watchdog | PID 3164922 (`train_hybrid.py`) · sentinel PID 3167084 | `pgrep`, `sentinel.log` | +| progress @ 04:15Z | **step 7,480 / 20,756 (36.0%)** — 61.3M of 170.0M tokens | `run_ssm_base_s0.log` | +| loss | step-0 12.4317 → train ~4.88; val 6.6844@400 → 6.2845@1200 → **4.9020@7200** (best) | `run_ssm_base_s0.log` | +| grad norm | 0.28–0.31, stable | `run_ssm_base_s0.log` | +| throughput | ~1,247 steps/h ≈ **2,837 tok/s** (measured over 7,080 steps / 5.68 h since resume) | derived from log + process start | +| ETA | ≈ **2026-07-20 14:54 UTC** (13,276 steps remaining) | same | +| memory | pool 37–41%, rss 11.6 GiB, GPU 66–69 °C / SoC 72–74 °C | `sentinel.log` heartbeats | + +**Deviations from `ARCHITECTURE.md`, recorded honestly:** the design doc specifies seq_len 4096 and +Muon(2D)+AdamW(1D); this arm runs **seq 2048 with plain AdamW**. The JAX Muon port (`muon_jax.py`) is not +written yet — AdamW-vs-Muon is itself a planned arm, and every arm in the ladder must use the same +optimizer for the comparison to hold, so the ladder's baseline optimizer is now AdamW unless re-based. + +### Incident + recovery (the run survived a real kill) + +First launch was killed by the sentinel at **step 580, 2026-07-19T16:58:48Z** — pool usage 81.3% ≥ the +0.80 kill line (MemAvailable 22.4 GiB / 119.7 GiB; SSM scan + chunked CE under autodiff held ~61 GB); +GPU 58 °C, no thermal component (`sentinel_kill_step580_2026-07-19.json`). Fix: **`nn.remat` on the +decoder block** (`model.py:129`, `BlockR = nn.remat(Block)`) + batch 8 → 4 → allocation 61.5 GB → 16.6 GB, +pool 81% → ~40%. Resumed from the step-400 checkpoint at 22:34:29 and has run clean since. +This was a *manual* recovery behind a config change, i.e. the §C5/S1-4a "not safe to auto-resume at the +same config" path — `loop_state.auto_resumes` correctly stayed at 0. + +## ⚠️ Open gate gap (must close before this arm is scored) + +`verify.py` last ran **2026-07-19 12:52** (per this file's previous revision — no verify log was captured +to disk). `model.py` was last modified **2026-07-19 22:27** to add `nn.remat`. **The verify gate has not +been re-run against the model that is actually training.** `nn.remat` is semantically identity +(rematerialization trades recompute for memory and must not change values), but "must not" is not +"verified on this box". Re-run `verify.py` and capture its output to `verify.log` **after** this arm +finishes — it is GPU work, and §C4.5 forbids co-running it beside the live trainer. + +## Next + +1. **Close the verify gap** (above) + write `verify.log`, so the artifact set is self-evidencing. +2. **Score the finished arm** via `/eval-harness` (BPB on wikitext-2 + code, text-lm-v2, `suite_version` + stamped) → write `verdict.json`. A single arm is a *baseline datum*, not a win: no cross-arm claim + exists until the ladder has ≥3 seeds and iso-FLOP-matched comparands (§C17/§C18), so the terminal + verdict for this arm caps at `directional` at best (§C25). +3. **The emergence-speed ladder** (the study): mixer type × attention fraction × NoPE, each scored at + increasing token budgets → does the efficient-mixer choice affect emergence SPEED but converge? Plus + the NoPE-on-full-attn validation. Multi-day; drive arms as they complete. +4. **Recovery chain**: the user still pastes the `@reboot bash research/boot_resume.sh` cron line + (§C4.2 — the agent never auto-installs cron). +5. **Downstream lifecycle** (long-context retrieval probe → data / mid-training / SFT), each to a §C25 + terminal verdict — the whole-lifecycle finish line for the new model. diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json new file mode 100644 index 0000000..8d6aa60 --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json @@ -0,0 +1,117 @@ +{ + "run_id": "2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0", + "model_dir": "HybridSSM-0.2B", "lifecycle_stage": "architecture", "objective": "pretrain-ablation", "framework": "jax", + "technique_slug": "hybrid-attention-rethink", + + "provenance": { + "written": "2026-07-20, AFTER launch — this is a RECONSTRUCTION, not a pre-launch record", + "why": "The 2026-07-19 launch created the ledger run entry but never wrote c5_evidence.json, so the §C5 requirement that the evidence be accumulated in this file and landed in ONE add-run BEFORE the spawn was NOT met for this run. That is a real contract miss and is recorded here rather than papered over.", + "rule": "Every item below is tagged `src`: `log` = a contemporaneous artifact on disk that anyone can re-read; `derived` = computed here from such an artifact; `attestation` = prose recorded in BUILD_STATUS.md at build time with no captured log; `not-captured` = the check was required and no evidence of it survives.", + "reconstructed_by": "progress-check session 2026-07-20 04:15Z; nothing in this file is asserted from memory." + }, + + "purpose": "First pretrain arm (base cell) of HybridSSM-0.2B, the repo's first novel from-scratch model and first JAX/Flax build. Establishes the ladder's baseline recipe; it is a single datum, not a contrast.", + + "budget": { + "tokens": 170034304, + "val_tokens": 300000, + "source": "brief research/briefs/hybrid-attention-rethink.md:55 — 'a token-budget ladder (e.g. 42M/168M/420M-analog)'. This arm is the ~168M RUNG of that ladder (170,034,304 tok).", + "deviation": "The brief's per-arm TARGET (line 59) is ~2-4 tok/param, i.e. ~0.5-1.5B tokens for a 200-370M hybrid. This arm runs 170.0M tok = 0.90 tok/param of non-embed (189.1M) — roughly 3-9x UNDER the briefed per-arm target. Justification: it reuses the already-tokenized Qwen3 tokcache (zero data-prep cost, and BPB-comparable to the 596M study), and one ladder rung is a legitimate first point. It is NOT a converged-budget arm and must not be read as one.", + "src": "log" + }, + + "arm_plan": { + "arms": ["ssm_base_s0 (mixer=ssm Mamba-2 selective scan, attn_every=2 i.e. 1:1 interleave, RoPE, seed 0)"], + "seeds": [0], + "new_cells": 1, + "planned_siblings_not_yet_run": ["mixer=swa128", "mixer=full (all-attention control)", "mixer=none (all-SSM)", "attn_every=4 (1:3)", "NoPE-on-full-attn", "seeds 1,2"], + "src": "log (ARCHITECTURE.md 'toggles' section) + live cmdline" + }, + + "single_variable": "NONE YET — n=1 arm, no comparand. No §C18 iso-FLOP contrast and no §C17 across-seed CI is satisfiable from this run alone, so no `confound_check` is claimed and the verdict is capped at `directional` at absolute best (§C25). The single-variable contrasts arrive with the sibling arms listed above.", + + "recipe": "d_model 768, 24 layers 1:1 [full-attn, ssm] interleave, GQA n_heads=12 n_kv=4 head_dim=64 RoPE(1e4), SwiGLU inter 2048, RMSNorm eps 1e-6 pre-norm, tied embedding, vocab 151,936 (Qwen3-0.6B-Base). 189.1M non-embed / 305.8M total. seq 2048, batch 4, 20,756 steps x 8,192 tok/step, AdamW lr 3e-3 warmup 200, chunked CE (chunk=8192), nn.remat on the decoder block, ckpt_every 200, eval_every 400.", + + "deviations_from_architecture_md": "ARCHITECTURE.md specifies seq_len 4096 and Muon(2D)+AdamW(1D); this arm runs seq 2048 with plain AdamW because muon_jax.py is not written yet. It also estimated ~146M non-embed vs the built 189.1M. Consequence: unless the ladder is re-based on Muon, AdamW @ seq 2048 IS the ladder's baseline recipe and this arm defines it.", + + "c5_0_smoke": { + "result": "pass", + "detail": "train.py --smoke on all variants (SSM / SWA-128+NoPE / 1:3-attention): fixed-batch overfit 8.8 -> ~0.003, grad norms 33 -> 0.03, checkpoint save->reload exact (max|delta|=0.0), exit 0. Synthetic data.", + "log_path": null, + "src": "attestation (BUILD_STATUS.md revision of 2026-07-19 12:52) — NO smoke log was captured to disk. This is the weakest link in the chain: the §C5.0 numbers cannot be re-read from an artifact." + }, + + "fit_probes_on_real_data": { + "result": "pass", + "detail": "3 probes at the real config on the real tokcache: probe.log 15 steps (12.4317 -> 11.5645), probe2.log 12 steps (12.4312 -> 12.0485), probe3.log 30 steps (12.4312 -> 9.5249@20 -> 8.4195). Step-0 loss vs ln(151936)=11.931 confirms a sane uniform-over-vocab init; loss trends down.", + "log_path": ["probe.log", "probe2.log", "probe3.log"], + "src": "log" + }, + + "c5_1_concurrency": { + "result": "not-captured", + "detail": "No pgrep/nvidia-smi/free output from the 2026-07-19 pre-launch moment survives. Circumstantial only: the sentinel kill record names a single trainer PID, and the box has run one trainer at a time throughout. Verified live at 2026-07-20 04:11Z that PID 3164922 is the ONLY compute app (nvidia-smi: 3164922, python3, 16635 MiB).", + "src": "not-captured (pre-launch) / log (current)" + }, + + "c5_2_budget": "see `budget` above — tokens 170,034,304, sourced to the brief's ladder, with the under-target deviation stated. src: log", + + "c5_3_probe": { + "tokens_per_sec": 2837, + "steps_per_hour": 1247, + "peak_mem_gb": 16.6, + "pool_total_gb": 119.7, + "peak_frac_of_pool": 0.139, + "cap_frac": 0.60, + "fits": true, + "detail": "tokens_per_sec is NOT a pre-launch probe number — the probe logs record no timing. It is DERIVED from the production run: 7,080 steps x 8,192 tok in 5.68 h (process start 22:34:29, log at 04:15Z), which is a strictly better measurement than a short probe. peak_mem_gb 16.6 is the live nvidia-smi compute-app figure at 04:11Z and matches the ledger `memory_fix` note ('61.5GB->16.6GB alloc'); sentinel heartbeats concur (trainer rss 11.6 GiB, pool 37-41%).", + "src": "derived" + }, + + "c5_4_eta_hours": { + "total_projected": 16.6, + "detail": "20,756 steps / 1,247 steps-per-hour = 16.65 h of clean wall-clock (excludes the killed first attempt). ETA 2026-07-20 ~14:54 UTC from the 04:15Z position of step 7,480.", + "src": "derived" + }, + + "c5_5_resume": { + "result": "pass", + "detail": "Proven in PRODUCTION, not merely in smoke: after the step-580 kill the run restarted from the step-400 checkpoint ('[resume] from checkpoint_ssm_base_s0.pkl at step 400', run_ssm_base_s0.log:36) and has run 7,080 clean steps since with a continuous loss curve. Script supports --ckpt/--resume/--ckpt_every. Smoke additionally reported save->reload max|delta|=0.0 (attestation).", + "resume_cmd": "cd HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build && python3 train_hybrid.py --data /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/tokcache_170034304_300000_seed0_Qwen3-0.6B-Base.pt --seq 2048 --batch 4 --tokens 170034304 --lr 3e-3 --warmup 200 --ckpt checkpoint_ssm_base_s0.pkl --resume checkpoint_ssm_base_s0.pkl --ckpt_every 200 --eval_every 400 --done_marker arm_ssm_base_s0.done", + "src": "log" + }, + + "c5_6_sentinel": { + "result": "armed", + "detail": "sentinel.py watch --pid 3164922 --log HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/sentinel.log, PID 3167084, alive, heartbeating every ~10 min. No --kill-at override, so it takes the module default 0.80 (< the 0.85 safe_cuda guard < the kernel OOM cliff), per §C6. It has PROVEN itself on this run — it fired correctly at step 580.", + "preflight_at_launch": "not-captured (no preflight output from 2026-07-19 survives)", + "src": "log / not-captured" + }, + + "c5_7_guards": { + "result": "verified", + "detail": "JAX script, so the §C1 guard is jax_safe_env, not safe_cuda: train_hybrid.py:10 `import jax_safe_env` precedes train_hybrid.py:15 `import jax` — correct order, enforcing PREALLOCATE=false + MEM_FRACTION=0.5. Chunked CE present and used: model.py:136 chunked_cross_entropy() does a streaming max + sumexp over vocab chunks (model.py:149-150) and never materializes the (N, 151936) logit matrix; train_hybrid.py:49 calls it with chunk=8192 (train_hybrid.py:81). Re-grepped 2026-07-20.", + "src": "log" + }, + + "sentinel_kill_incident": { + "when": "2026-07-19T16:58:48Z, step 580", + "trigger": "memory — pool 81.3% >= kill-at 0.80 (MemAvailable 22.4 of 119.7 GiB, trainer rss 17.2 GiB). GPU 58C / SoC 71.5C, gpu_throttling false: no thermal component.", + "root_cause": "SSM associative_scan + chunked CE UNDER AUTODIFF held ~61 GB of activations at batch 8.", + "fix": "nn.remat on the decoder block (model.py:129 `BlockR = nn.remat(Block)`) + batch 8 -> 4. Allocation 61.5 GB -> 16.6 GB, pool 81% -> ~40%. Token budget unchanged (steps 10,378 -> 20,756 at constant tokens).", + "resume_policy": "MANUAL recovery behind a CONFIG CHANGE = the §C5 / research-loop S1-4a 'sentinel kill -> not safe to auto-resume at the same config' path. loop_state.auto_resumes correctly stayed 0; no automatic resume budget consumed.", + "src": "log (sentinel_kill_step580_2026-07-19.json, run_ssm_base_s0.log:36, ledger memory_fix)" + }, + + "open_gate_gap": { + "item": "verify.py is STALE relative to the model that is training", + "detail": "verify.py last ran 2026-07-19 12:52 (BUILD_STATUS.md attestation: scan-vs-sequential max|delta|=2.4e-7, chunked-vs-naive CE |delta|=4.8e-5, 8 toggles forward-finite, forward deterministic — no verify log was captured to disk). model.py was then modified at 22:27 to add nn.remat. The gate has NOT been re-run against the training model. nn.remat is semantically identity and the loss curve is continuous across the resume, but that is corroboration, not verification.", + "required_action": "Re-run verify.py and capture verify.log BEFORE this arm is scored. It is GPU work; §C4.5 forbids co-running it beside the live trainer, so it waits for the arm to finish.", + "src": "derived (file mtimes + BUILD_STATUS.md)" + }, + + "verdict_metric": "On completion: /eval-harness text-lm-v2 with suite_version stamped — BPB on wikitext-2 + a code corpus using the model's OWN tokenizer (Qwen3-0.6B-Base), which makes this arm BPB-comparable to the 596M study. NO win is claimable from this run: n=1 seed, no comparand, no iso-FLOP match -> §C17/§C18/§C25 cap the verdict at `directional` at best. The study's headline object (the emergence-speed curve + NoPE-on-full-attn validation) needs the sibling arms and >=3 seeds.", + + "evidence_path": "HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json", + "detail_md": "research/ledger/runs/2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md" +} diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py new file mode 100644 index 0000000..7f4ec2b --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py @@ -0,0 +1,168 @@ +"""HybridSSM-0.2B — a from-scratch hybrid attention-SSM decoder LM (JAX/Flax). + +Novel design (brief hybrid-attention-rethink / arXiv 2606.15378): interleaves full GQA attention with +an efficient mixer (SelectiveSSM or SlidingWindowAttention). The composition is configurable so the +single-variable ablations (mixer type · attention fraction · NoPE-on-full-attn) are ONE flag each. + +No bit-exact oracle (novel) — verify.py cross-checks the scan vs its sequential reference (ssm.py) and +the full forward vs an independent numpy attention at ~1e-2. Chunked CE for the 151,936 vocab (§C1). +""" +from __future__ import annotations +import dataclasses +import math +import jax +import jax.numpy as jnp +import flax.linen as nn + +from ssm import SelectiveSSM, SlidingWindowAttention + + +@dataclasses.dataclass(frozen=True) +class HybridConfig: + vocab_size: int = 151_936 + d_model: int = 768 + n_layers: int = 24 + n_heads: int = 12 + n_kv_heads: int = 4 + head_dim: int = 64 + ffn: int = 2048 + rope_theta: float = 1e4 + rms_eps: float = 1e-6 + # ---- the ablation toggles ---- + attn_every: int = 2 # full-attention layer every k layers (2 → 1:1, 4 → 1:3) + mixer: str = "ssm" # efficient-layer mixer: "ssm" | "swa128" + swa_window: int = 128 + nope_on_full: bool = False # NoPE on full-attention layers (the paper's headline knob) + + def is_full(self, i: int) -> bool: + return (i % self.attn_every) == 0 + + +def rms_norm(x, weight, eps): + dt = x.dtype + x = x.astype(jnp.float32) + x = x * jax.lax.rsqrt(jnp.mean(x * x, axis=-1, keepdims=True) + eps) + return (weight * x).astype(dt) + + +def rope(x, theta): + # x: [B,T,H,D] → rotary. D even. + B, T, H, D = x.shape + half = D // 2 + inv = 1.0 / (theta ** (jnp.arange(0, half, dtype=jnp.float32) / half)) + ang = jnp.arange(T, dtype=jnp.float32)[:, None] * inv[None, :] # [T,half] + cos = jnp.cos(ang)[None, :, None, :] + sin = jnp.sin(ang)[None, :, None, :] + x1, x2 = x[..., :half], x[..., half:] + return jnp.concatenate([x1 * cos - x2 * sin, x1 * sin + x2 * cos], axis=-1).astype(x.dtype) + + +class GQAAttention(nn.Module): + cfg: HybridConfig + use_rope: bool = True + + @nn.compact + def __call__(self, x): + c = self.cfg + B, T, _ = x.shape + nq, nkv, hd = c.n_heads, c.n_kv_heads, c.head_dim + q = nn.Dense(nq * hd, use_bias=False, name="q")(x).reshape(B, T, nq, hd) + k = nn.Dense(nkv * hd, use_bias=False, name="k")(x).reshape(B, T, nkv, hd) + v = nn.Dense(nkv * hd, use_bias=False, name="v")(x).reshape(B, T, nkv, hd) + if self.use_rope: + q, k = rope(q, c.rope_theta), rope(k, c.rope_theta) + rep = nq // nkv + k = jnp.repeat(k, rep, axis=2) + v = jnp.repeat(v, rep, axis=2) + scores = jnp.einsum("bqhd,bkhd->bhqk", q, k) / math.sqrt(hd) + causal = jnp.tril(jnp.ones((T, T), bool)) + scores = jnp.where(causal[None, None], scores, -1e30) + attn = jax.nn.softmax(scores.astype(jnp.float32), axis=-1).astype(x.dtype) + out = jnp.einsum("bhqk,bkhd->bqhd", attn, v).reshape(B, T, nq * hd) + return nn.Dense(c.d_model, use_bias=False, name="o")(out) + + +class MLP(nn.Module): + cfg: HybridConfig + + @nn.compact + def __call__(self, x): + c = self.cfg + g = nn.Dense(c.ffn, use_bias=False, name="gate")(x) + u = nn.Dense(c.ffn, use_bias=False, name="up")(x) + return nn.Dense(c.d_model, use_bias=False, name="down")(nn.silu(g) * u) + + +class Block(nn.Module): + cfg: HybridConfig + layer_idx: int + + @nn.compact + def __call__(self, x): + c = self.cfg + n1 = self.param("norm1", nn.initializers.ones, (c.d_model,)) + n2 = self.param("norm2", nn.initializers.ones, (c.d_model,)) + h = rms_norm(x, n1, c.rms_eps) + if c.is_full(self.layer_idx): + mix = GQAAttention(c, use_rope=not c.nope_on_full, name="attn")(h) + elif c.mixer == "swa128": + mix = SlidingWindowAttention(c.d_model, c.n_heads, c.swa_window, name="swa")(h) + else: + mix = SelectiveSSM(c.d_model, name="ssm")(h) + x = x + mix + x = x + MLP(c, name="mlp")(rms_norm(x, n2, c.rms_eps)) + return x + + +class HybridSSM(nn.Module): + cfg: HybridConfig + + @nn.compact + def __call__(self, input_ids): + c = self.cfg + emb = self.param("embed", nn.initializers.normal(1.0 / math.sqrt(c.d_model)), + (c.vocab_size, c.d_model)) + x = emb[input_ids] + # Rematerialize each block: recompute its activations in the backward pass instead of + # holding all 24 layers' SSM-scan / attention intermediates (~61GB → sentinel-killed at + # batch4). Numerically identical (same params/schedule → the step-400 ckpt resumes cleanly). + BlockR = nn.remat(Block) + for i in range(c.n_layers): + x = BlockR(c, i, name=f"block_{i}")(x) + x = rms_norm(x, self.param("norm_f", nn.initializers.ones, (c.d_model,)), c.rms_eps) + return x, emb # hidden + tied embedding (logits = x @ emb.T, formed chunked) + + +def chunked_cross_entropy(hidden, emb, targets, chunk=8192, ignore_index=-100): + """CE over the tied head WITHOUT materializing (N, vocab) logits (§C1). Streams the vocab in + chunks for a numerically-stable log-sum-exp + the target logit.""" + N, d = hidden.shape + V = emb.shape[0] + hidden = hidden.astype(jnp.float32) + emb = emb.astype(jnp.float32) + mask = targets != ignore_index + tgt = jnp.where(mask, targets, 0) + tgt_logit = jnp.sum(hidden * emb[tgt], axis=-1) # [N] correct-class logit + # streaming max + sumexp over vocab chunks + running_max = jnp.full((N,), -jnp.inf) + running_sum = jnp.zeros((N,)) + for s in range(0, V, chunk): + logits_c = hidden @ emb[s:s + chunk].T # [N, chunk] + cmax = jnp.max(logits_c, axis=-1) + new_max = jnp.maximum(running_max, cmax) + running_sum = running_sum * jnp.exp(running_max - new_max) + \ + jnp.sum(jnp.exp(logits_c - new_max[:, None]), axis=-1) + running_max = new_max + lse = running_max + jnp.log(running_sum) + nll = lse - tgt_logit + return jnp.sum(jnp.where(mask, nll, 0.0)) / jnp.maximum(jnp.sum(mask), 1) + + +def count_params(params, exclude_embed=True): + flat = jax.tree_util.tree_leaves(params) + total = sum(p.size for p in flat) + emb = 0 + for path, p in jax.tree_util.tree_flatten_with_path(params)[0]: + if "embed" in jax.tree_util.keystr(path): + emb += p.size + return total - (emb if exclude_embed else 0), total diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ssm.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ssm.py new file mode 100644 index 0000000..3b35dff --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ssm.py @@ -0,0 +1,104 @@ +"""Efficient-mixer layers for HybridSSM-0.2B (novel design; brief hybrid-attention-rethink). + +Two mixers the hybrid interleaves with full GQA attention: + - SelectiveSSM : a Mamba-2-style diagonal selective state-space mixer, computed as a linear + recurrence via jax.lax.associative_scan (JAX-native). h_t[c,n] = a_t[c]·h_{t-1}[c,n] + bx_t[c,n], + y_t[c] = sum_n C_t[n]·h_t[c,n] + D[c]·x_t[c], gated by z. Input-dependent a (selective). + - SlidingWindowAttention : causal attention restricted to a window (the SWA ablation arm). + +CORRECTNESS-FIRST: `selective_scan` (parallel) and `selective_scan_ref` (explicit sequential loop) +must agree — verify.py gates on it before any training (§C5.0 / §C14 novel-design ~1e-2 cross-check). +""" +from __future__ import annotations +import jax +import jax.numpy as jnp +import flax.linen as nn +import math + + +# ───────────────────────── selective-SSM core (pure functions) ───────────────────────── +def _combine(l, r): + """Associative combine for the linear recurrence h_t = a_t·h_{t-1} + u_t. + Element = (a, u) with a:[...,C] (per-channel decay, broadcast over state) and u:[...,C,N]. + (a1,u1) ⊕ (a2,u2) = (a1·a2, a2·u1 + u2).""" + a1, u1 = l + a2, u2 = r + return a1 * a2, a2[..., None] * u1 + u2 + + +def selective_scan(a, bx, C, D, x): + """Parallel selective scan. + a : [B,T,C] per-channel decay a_t (0btc", h, C) + D * x + return y + + +def selective_scan_ref(a, bx, C, D, x): + """Explicit sequential reference (the correctness oracle for the parallel scan).""" + B, T, Cn, N = bx.shape + h = jnp.zeros((B, Cn, N), bx.dtype) + ys = [] + for t in range(T): + h = a[:, t][..., None] * h + bx[:, t] # [B,C,N] + ys.append(jnp.einsum("bcn,bn->bc", h, C[:, t])) + return jnp.stack(ys, axis=1) + D * x + + +class SelectiveSSM(nn.Module): + d_model: int + d_state: int = 16 + expand: int = 2 + dt_min: float = 1e-3 + dt_max: float = 1e-1 + + @nn.compact + def __call__(self, x): + B, T, _ = x.shape + d_in = self.expand * self.d_model + # projections: x-path + z gate + xz = nn.Dense(2 * d_in, use_bias=False, name="in_proj")(x) + xin, z = jnp.split(xz, 2, axis=-1) # [B,T,d_in] each + xin = nn.silu(xin) + # input-dependent dt, B, C + dt = nn.softplus(nn.Dense(d_in, name="dt_proj")(x)) # [B,T,d_in] > 0 + Bc = nn.Dense(self.d_state, use_bias=False, name="B_proj")(x) # [B,T,N] + Cc = nn.Dense(self.d_state, use_bias=False, name="C_proj")(x) # [B,T,N] + # A (negative, per-channel-state), D skip + A_log = self.param("A_log", lambda k: jnp.log(jnp.linspace(1.0, self.d_state, d_in))) + A = -jnp.exp(A_log) # [d_in] < 0 (diag, state broadcast) + D = self.param("D", nn.initializers.ones, (d_in,)) + a = jnp.exp(dt * A) # [B,T,d_in] in (0,1) + bx = (dt[..., None] * Bc[:, :, None, :]) * xin[..., None] # [B,T,d_in,N] + y = selective_scan(a, bx, Cc, D, xin) # [B,T,d_in] + y = y * nn.silu(z) # gate + return nn.Dense(self.d_model, use_bias=False, name="out_proj")(y) + + +class SlidingWindowAttention(nn.Module): + """Causal attention restricted to the last `window` tokens (the SWA ablation mixer).""" + d_model: int + n_heads: int = 12 + window: int = 128 + + @nn.compact + def __call__(self, x): + B, T, _ = x.shape + hd = self.d_model // self.n_heads + qkv = nn.Dense(3 * self.d_model, use_bias=False, name="qkv")(x) + q, k, v = jnp.split(qkv, 3, axis=-1) + shp = (B, T, self.n_heads, hd) + q, k, v = q.reshape(shp), k.reshape(shp), v.reshape(shp) + scores = jnp.einsum("bqhd,bkhd->bhqk", q, k) / math.sqrt(hd) + i = jnp.arange(T)[:, None] + j = jnp.arange(T)[None, :] + mask = (j <= i) & (j > i - self.window) # causal AND within window + scores = jnp.where(mask[None, None], scores, -1e30) + attn = jax.nn.softmax(scores, axis=-1) + out = jnp.einsum("bhqk,bkhd->bqhd", attn, v).reshape(B, T, self.d_model) + return nn.Dense(self.d_model, use_bias=False, name="o")(out) diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py new file mode 100644 index 0000000..62afa4a --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py @@ -0,0 +1,175 @@ +"""train.py — HybridSSM-0.2B trainer (JAX/Flax). §C5.0 smoke-capable + ckpt/resume for the recovery chain. + +Smoke (§C5.0): `python3 train.py --smoke` → tiny config, synthetic data, a few steps; asserts imports, +build, finite+trending loss, ckpt save→reload round-trip, exit 0. Real run adds `--data ` + +budget; launched detached under sentinel watch + boot_resume (§C5). Optimizer = AdamW now (optax); the +Muon port (muon_jax.py) is a pre-full-run refinement (the paper uses Muon; AdamW is a valid arm too). +""" +import sys +sys.path.insert(0, "/home/yashb98/Downloads/BuildFromScratch") +import jax_safe_env # noqa: E402 (before jax, §C1) +import argparse # noqa: E402 +import dataclasses # noqa: E402 +import pickle # noqa: E402 +import pathlib # noqa: E402 +import jax # noqa: E402 +import jax.numpy as jnp # noqa: E402 +import numpy as np # noqa: E402 +import optax # noqa: E402 +from flax import serialization # noqa: E402 +import model as M # noqa: E402 + + +def make_batch(rng, B, T, V): + ids = np.asarray(jax.random.randint(rng, (B, T + 1), 0, V)) + return ids[:, :-1], ids[:, 1:] # inputs, next-token targets + + +def load_tokens(path): + """Load a FineWeb-Edu tokcache (.pt from train_qwen3.stream_tokens — real, decontaminated, + Qwen3-tokenized; reused for BPB-comparability with the 596M study). Returns uint32 arrays.""" + import torch + d = torch.load(path, map_location="cpu", weights_only=False) + tr = d["train"].numpy().astype(np.uint32) + va = d["val"].numpy().astype(np.uint32) + return tr, va + + +def real_batch(rng, toks, B, T): + """Sample B random contiguous windows of T+1 tokens from the flat token stream.""" + n = toks.shape[0] + starts = np.asarray(jax.random.randint(rng, (B,), 0, n - T - 1)) + win = np.stack([toks[s:s + T + 1] for s in starts]).astype(np.int32) + return win[:, :-1], win[:, 1:] + + +def loss_fn(params, apply, ids, tgt, chunk): + hidden, emb = apply({"params": params}, ids) + N = hidden.shape[0] * hidden.shape[1] + return M.chunked_cross_entropy(hidden.reshape(N, -1), emb, tgt.reshape(N), chunk=chunk) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--smoke", action="store_true") + ap.add_argument("--steps", type=int, default=6) + ap.add_argument("--batch", type=int, default=4) + ap.add_argument("--seq", type=int, default=4096) + ap.add_argument("--lr", type=float, default=3e-3) + ap.add_argument("--mixer", default="ssm") + ap.add_argument("--attn_every", type=int, default=2) + ap.add_argument("--nope_on_full", action="store_true") + ap.add_argument("--ckpt", default="checkpoint.pkl") + ap.add_argument("--resume", default=None) + ap.add_argument("--ckpt_every", type=int, default=200) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--data", default=None, help="FineWeb-Edu tokcache .pt (real run)") + ap.add_argument("--tokens", type=int, default=0, help="token budget → steps (real run)") + ap.add_argument("--warmup", type=int, default=200) + ap.add_argument("--eval_every", type=int, default=200) + ap.add_argument("--done_marker", default=None) + a = ap.parse_args() + + if a.smoke: + cfg = M.HybridConfig(vocab_size=4096, d_model=256, n_layers=6, n_heads=8, n_kv_heads=2, + head_dim=32, ffn=512, mixer=a.mixer, attn_every=a.attn_every, + nope_on_full=a.nope_on_full) + # overfit ONE fixed batch — the real "can it learn" signal (random-per-step has no structure) + B, T, steps, chunk = 2, 64, 30, 512 + else: + cfg = M.HybridConfig(mixer=a.mixer, attn_every=a.attn_every, nope_on_full=a.nope_on_full) + B, T, chunk = a.batch, a.seq, 8192 + steps = a.tokens // (B * T) if a.tokens else a.steps + + toks = valtoks = None + if a.data: + toks, valtoks = load_tokens(a.data) + print(f"[data] loaded {toks.shape[0]:,} train + {valtoks.shape[0]:,} val tokens from {a.data}", flush=True) + + rng = jax.random.PRNGKey(a.seed) + net = M.HybridSSM(cfg) + ids0, _ = make_batch(rng, B, T, cfg.vocab_size) + params = net.init(rng, jnp.asarray(ids0))["params"] + ne, tot = M.count_params(params) + print(f"[build] non-embed={ne/1e6:.1f}M total={tot/1e6:.1f}M cfg={cfg.mixer} attn_every={cfg.attn_every} " + f"nope={cfg.nope_on_full} steps={steps} tok/step={B*T}", flush=True) + + if a.smoke or not a.data: + opt = optax.adamw(a.lr, weight_decay=0.1) + else: + sched = optax.warmup_cosine_decay_schedule(0.0, a.lr, a.warmup, max(steps, a.warmup + 1), end_value=a.lr * 0.1) + opt = optax.chain(optax.clip_by_global_norm(1.0), optax.adamw(sched, weight_decay=0.1)) + opt_state = opt.init(params) + start_step = 0 + if a.resume and pathlib.Path(a.resume).exists(): + with open(a.resume, "rb") as f: + blob = pickle.load(f) + params = serialization.from_bytes(params, blob["params"]) + opt_state = serialization.from_bytes(opt_state, blob["opt_state"]) + start_step = blob["step"] + print(f"[resume] from {a.resume} at step {start_step}", flush=True) + + @jax.jit + def step(params, opt_state, ids, tgt): + loss, grads = jax.value_and_grad(loss_fn)(params, net.apply, ids, tgt, chunk) + gnorm = optax.global_norm(grads) + updates, opt_state = opt.update(grads, opt_state, params) + params = optax.apply_updates(params, updates) + return params, opt_state, loss, gnorm + + def save(step_i): + blob = {"params": serialization.to_bytes(params), "opt_state": serialization.to_bytes(opt_state), "step": step_i} + tmp = a.ckpt + ".tmp" + with open(tmp, "wb") as f: + pickle.dump(blob, f) + pathlib.Path(tmp).replace(a.ckpt) + + fixed = make_batch(jax.random.PRNGKey(1234), B, T, cfg.vocab_size) if a.smoke else None + losses = [] + for s in range(start_step, start_step + steps): + rng, sk = jax.random.split(rng) + if a.smoke: + ids, tgt = fixed # overfit one fixed batch + elif toks is not None: + ids, tgt = real_batch(sk, toks, B, T) # real FineWeb-Edu windows + else: + ids, tgt = make_batch(sk, B, T, cfg.vocab_size) + params, opt_state, loss, gnorm = step(params, opt_state, jnp.asarray(ids), jnp.asarray(tgt)) + loss = float(loss) + losses.append(loss) + assert np.isfinite(loss), f"non-finite loss at step {s}" + if a.smoke or s % 20 == 0: + print(f"[step {s}] loss={loss:.4f} grad_norm={float(gnorm):.3f}", flush=True) + if valtoks is not None and a.eval_every and (s + 1) % a.eval_every == 0: + rng, vk = jax.random.split(rng) + vids, vtgt = real_batch(vk, valtoks, B, T) + vloss = float(loss_fn(params, net.apply, jnp.asarray(vids), jnp.asarray(vtgt), chunk)) + print(f"[eval step {s+1}] val_loss={vloss:.4f}", flush=True) + if (s + 1) % a.ckpt_every == 0: + save(s + 1) + + if not a.smoke: + save(start_step + steps) + if a.done_marker: + pathlib.Path(a.done_marker).touch() + print(f"[done] {start_step + steps} steps · final loss={losses[-1]:.4f}", flush=True) + + if a.smoke: + # ckpt round-trip: save, reload into a fresh param tree, assert identical + save(start_step + steps) + with open(a.ckpt, "rb") as f: + blob = pickle.load(f) + fresh = net.init(jax.random.PRNGKey(99), jnp.asarray(ids0))["params"] + restored = serialization.from_bytes(fresh, blob["params"]) + maxd = max(float(jnp.abs(x - y).max()) for x, y in + zip(jax.tree_util.tree_leaves(params), jax.tree_util.tree_leaves(restored))) + trend = losses[-1] < losses[0] - 1.0 # overfitting one batch must clearly drop loss + print(f"[smoke] loss {losses[0]:.4f} -> {losses[-1]:.4f} trending_down={trend} | " + f"ckpt_roundtrip max|Δ|={maxd:.2e} | step={blob['step']}", flush=True) + ok = np.isfinite(losses).all() and maxd < 1e-6 and trend + print(f"SMOKE {'PASS' if ok else 'FAIL'}", flush=True) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py new file mode 100644 index 0000000..62afa4a --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py @@ -0,0 +1,175 @@ +"""train.py — HybridSSM-0.2B trainer (JAX/Flax). §C5.0 smoke-capable + ckpt/resume for the recovery chain. + +Smoke (§C5.0): `python3 train.py --smoke` → tiny config, synthetic data, a few steps; asserts imports, +build, finite+trending loss, ckpt save→reload round-trip, exit 0. Real run adds `--data ` + +budget; launched detached under sentinel watch + boot_resume (§C5). Optimizer = AdamW now (optax); the +Muon port (muon_jax.py) is a pre-full-run refinement (the paper uses Muon; AdamW is a valid arm too). +""" +import sys +sys.path.insert(0, "/home/yashb98/Downloads/BuildFromScratch") +import jax_safe_env # noqa: E402 (before jax, §C1) +import argparse # noqa: E402 +import dataclasses # noqa: E402 +import pickle # noqa: E402 +import pathlib # noqa: E402 +import jax # noqa: E402 +import jax.numpy as jnp # noqa: E402 +import numpy as np # noqa: E402 +import optax # noqa: E402 +from flax import serialization # noqa: E402 +import model as M # noqa: E402 + + +def make_batch(rng, B, T, V): + ids = np.asarray(jax.random.randint(rng, (B, T + 1), 0, V)) + return ids[:, :-1], ids[:, 1:] # inputs, next-token targets + + +def load_tokens(path): + """Load a FineWeb-Edu tokcache (.pt from train_qwen3.stream_tokens — real, decontaminated, + Qwen3-tokenized; reused for BPB-comparability with the 596M study). Returns uint32 arrays.""" + import torch + d = torch.load(path, map_location="cpu", weights_only=False) + tr = d["train"].numpy().astype(np.uint32) + va = d["val"].numpy().astype(np.uint32) + return tr, va + + +def real_batch(rng, toks, B, T): + """Sample B random contiguous windows of T+1 tokens from the flat token stream.""" + n = toks.shape[0] + starts = np.asarray(jax.random.randint(rng, (B,), 0, n - T - 1)) + win = np.stack([toks[s:s + T + 1] for s in starts]).astype(np.int32) + return win[:, :-1], win[:, 1:] + + +def loss_fn(params, apply, ids, tgt, chunk): + hidden, emb = apply({"params": params}, ids) + N = hidden.shape[0] * hidden.shape[1] + return M.chunked_cross_entropy(hidden.reshape(N, -1), emb, tgt.reshape(N), chunk=chunk) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--smoke", action="store_true") + ap.add_argument("--steps", type=int, default=6) + ap.add_argument("--batch", type=int, default=4) + ap.add_argument("--seq", type=int, default=4096) + ap.add_argument("--lr", type=float, default=3e-3) + ap.add_argument("--mixer", default="ssm") + ap.add_argument("--attn_every", type=int, default=2) + ap.add_argument("--nope_on_full", action="store_true") + ap.add_argument("--ckpt", default="checkpoint.pkl") + ap.add_argument("--resume", default=None) + ap.add_argument("--ckpt_every", type=int, default=200) + ap.add_argument("--seed", type=int, default=0) + ap.add_argument("--data", default=None, help="FineWeb-Edu tokcache .pt (real run)") + ap.add_argument("--tokens", type=int, default=0, help="token budget → steps (real run)") + ap.add_argument("--warmup", type=int, default=200) + ap.add_argument("--eval_every", type=int, default=200) + ap.add_argument("--done_marker", default=None) + a = ap.parse_args() + + if a.smoke: + cfg = M.HybridConfig(vocab_size=4096, d_model=256, n_layers=6, n_heads=8, n_kv_heads=2, + head_dim=32, ffn=512, mixer=a.mixer, attn_every=a.attn_every, + nope_on_full=a.nope_on_full) + # overfit ONE fixed batch — the real "can it learn" signal (random-per-step has no structure) + B, T, steps, chunk = 2, 64, 30, 512 + else: + cfg = M.HybridConfig(mixer=a.mixer, attn_every=a.attn_every, nope_on_full=a.nope_on_full) + B, T, chunk = a.batch, a.seq, 8192 + steps = a.tokens // (B * T) if a.tokens else a.steps + + toks = valtoks = None + if a.data: + toks, valtoks = load_tokens(a.data) + print(f"[data] loaded {toks.shape[0]:,} train + {valtoks.shape[0]:,} val tokens from {a.data}", flush=True) + + rng = jax.random.PRNGKey(a.seed) + net = M.HybridSSM(cfg) + ids0, _ = make_batch(rng, B, T, cfg.vocab_size) + params = net.init(rng, jnp.asarray(ids0))["params"] + ne, tot = M.count_params(params) + print(f"[build] non-embed={ne/1e6:.1f}M total={tot/1e6:.1f}M cfg={cfg.mixer} attn_every={cfg.attn_every} " + f"nope={cfg.nope_on_full} steps={steps} tok/step={B*T}", flush=True) + + if a.smoke or not a.data: + opt = optax.adamw(a.lr, weight_decay=0.1) + else: + sched = optax.warmup_cosine_decay_schedule(0.0, a.lr, a.warmup, max(steps, a.warmup + 1), end_value=a.lr * 0.1) + opt = optax.chain(optax.clip_by_global_norm(1.0), optax.adamw(sched, weight_decay=0.1)) + opt_state = opt.init(params) + start_step = 0 + if a.resume and pathlib.Path(a.resume).exists(): + with open(a.resume, "rb") as f: + blob = pickle.load(f) + params = serialization.from_bytes(params, blob["params"]) + opt_state = serialization.from_bytes(opt_state, blob["opt_state"]) + start_step = blob["step"] + print(f"[resume] from {a.resume} at step {start_step}", flush=True) + + @jax.jit + def step(params, opt_state, ids, tgt): + loss, grads = jax.value_and_grad(loss_fn)(params, net.apply, ids, tgt, chunk) + gnorm = optax.global_norm(grads) + updates, opt_state = opt.update(grads, opt_state, params) + params = optax.apply_updates(params, updates) + return params, opt_state, loss, gnorm + + def save(step_i): + blob = {"params": serialization.to_bytes(params), "opt_state": serialization.to_bytes(opt_state), "step": step_i} + tmp = a.ckpt + ".tmp" + with open(tmp, "wb") as f: + pickle.dump(blob, f) + pathlib.Path(tmp).replace(a.ckpt) + + fixed = make_batch(jax.random.PRNGKey(1234), B, T, cfg.vocab_size) if a.smoke else None + losses = [] + for s in range(start_step, start_step + steps): + rng, sk = jax.random.split(rng) + if a.smoke: + ids, tgt = fixed # overfit one fixed batch + elif toks is not None: + ids, tgt = real_batch(sk, toks, B, T) # real FineWeb-Edu windows + else: + ids, tgt = make_batch(sk, B, T, cfg.vocab_size) + params, opt_state, loss, gnorm = step(params, opt_state, jnp.asarray(ids), jnp.asarray(tgt)) + loss = float(loss) + losses.append(loss) + assert np.isfinite(loss), f"non-finite loss at step {s}" + if a.smoke or s % 20 == 0: + print(f"[step {s}] loss={loss:.4f} grad_norm={float(gnorm):.3f}", flush=True) + if valtoks is not None and a.eval_every and (s + 1) % a.eval_every == 0: + rng, vk = jax.random.split(rng) + vids, vtgt = real_batch(vk, valtoks, B, T) + vloss = float(loss_fn(params, net.apply, jnp.asarray(vids), jnp.asarray(vtgt), chunk)) + print(f"[eval step {s+1}] val_loss={vloss:.4f}", flush=True) + if (s + 1) % a.ckpt_every == 0: + save(s + 1) + + if not a.smoke: + save(start_step + steps) + if a.done_marker: + pathlib.Path(a.done_marker).touch() + print(f"[done] {start_step + steps} steps · final loss={losses[-1]:.4f}", flush=True) + + if a.smoke: + # ckpt round-trip: save, reload into a fresh param tree, assert identical + save(start_step + steps) + with open(a.ckpt, "rb") as f: + blob = pickle.load(f) + fresh = net.init(jax.random.PRNGKey(99), jnp.asarray(ids0))["params"] + restored = serialization.from_bytes(fresh, blob["params"]) + maxd = max(float(jnp.abs(x - y).max()) for x, y in + zip(jax.tree_util.tree_leaves(params), jax.tree_util.tree_leaves(restored))) + trend = losses[-1] < losses[0] - 1.0 # overfitting one batch must clearly drop loss + print(f"[smoke] loss {losses[0]:.4f} -> {losses[-1]:.4f} trending_down={trend} | " + f"ckpt_roundtrip max|Δ|={maxd:.2e} | step={blob['step']}", flush=True) + ok = np.isfinite(losses).all() and maxd < 1e-6 and trend + print(f"SMOKE {'PASS' if ok else 'FAIL'}", flush=True) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/verify.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/verify.py new file mode 100644 index 0000000..6a1acd7 --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/verify.py @@ -0,0 +1,79 @@ +"""verify.py — the correctness gate for HybridSSM-0.2B (novel design → NOT bit-exact; ~1e-2, §C14). + +Gates: (1) the SSM parallel scan == its sequential reference; (2) chunked CE == naive CE (avoids the +151,936-logit materialization, §C1); (3) param count sane; (4) all ablation toggles forward finite; +(5) forward is deterministic. No training may start until this exits 0 (§C5.0 correctness-first). +""" +import sys +sys.path.insert(0, "/home/yashb98/Downloads/BuildFromScratch") +import jax_safe_env # noqa: E402 (before jax, §C1) +import jax # noqa: E402 +import jax.numpy as jnp # noqa: E402 +import dataclasses # noqa: E402 +import ssm # noqa: E402 +import model as M # noqa: E402 + +TOL_SCAN, TOL_CE = 1e-4, 1e-3 +FAILS = [] + + +def check(name, ok, detail=""): + print(f"[{'PASS' if ok else 'FAIL'}] {name} {detail}") + if not ok: + FAILS.append(name) + + +def main(): + k = jax.random.PRNGKey(0) + + # 1) SSM scan vs sequential reference + ks = jax.random.split(k, 5) + a = jax.nn.sigmoid(jax.random.normal(ks[0], (2, 32, 8))) + bx = jax.random.normal(ks[1], (2, 32, 8, 4)) * 0.1 + Cc = jax.random.normal(ks[2], (2, 32, 4)) + D = jax.random.normal(ks[3], (8,)) + x = jax.random.normal(ks[4], (2, 32, 8)) + d_scan = float(jnp.abs(ssm.selective_scan(a, bx, Cc, D, x) - ssm.selective_scan_ref(a, bx, Cc, D, x)).max()) + check("ssm_scan_vs_reference", d_scan < TOL_SCAN, f"max|Δ|={d_scan:.2e}") + + # 2) chunked CE vs naive CE + h = jax.random.normal(ks[0], (128, 768)) * 0.5 + e = jax.random.normal(ks[1], (512, 768)) * 0.1 + t = jax.random.randint(ks[2], (128,), 0, 512) + ce_c = float(M.chunked_cross_entropy(h, e, t, chunk=64)) + ce_n = float(-jax.nn.log_softmax((h @ e.T).astype(jnp.float32), axis=-1)[jnp.arange(128), t].mean()) + check("chunked_ce_vs_naive", abs(ce_c - ce_n) < TOL_CE, f"chunked={ce_c:.6f} naive={ce_n:.6f} |Δ|={abs(ce_c-ce_n):.2e}") + + # 3) param count + forward + cfg = M.HybridConfig() + net = M.HybridSSM(cfg) + ids = jax.random.randint(k, (2, 64), 0, cfg.vocab_size) + params = net.init(k, ids)["params"] + ne, tot = M.count_params(params) + hidden, emb = net.apply({"params": params}, ids) + check("param_count_sane", 100e6 < ne < 500e6, f"non-embed={ne/1e6:.1f}M total={tot/1e6:.1f}M") + check("forward_finite", bool(jnp.isfinite(hidden).all()), f"hidden={hidden.shape}") + + # 4) toggle sweep finite + sc = M.HybridConfig(vocab_size=2048, d_model=256, n_layers=6, n_heads=8, n_kv_heads=2, head_dim=32, ffn=512) + sids = jax.random.randint(k, (2, 48), 0, 2048) + allfin = True + for mixer in ("ssm", "swa128"): + for ae in (2, 4): + for nope in (False, True): + c2 = dataclasses.replace(sc, mixer=mixer, attn_every=ae, nope_on_full=nope) + n2 = M.HybridSSM(c2) + hh, _ = n2.apply({"params": n2.init(k, sids)["params"]}, sids) + allfin = allfin and bool(jnp.isfinite(hh).all()) + check("toggle_sweep_finite", allfin, "8 combos (mixer×attn_every×nope)") + + # 5) determinism + h2, _ = net.apply({"params": params}, ids) + check("forward_deterministic", float(jnp.abs(hidden - h2).max()) == 0.0) + + print(f"\nVERIFY {'PASS' if not FAILS else 'FAIL: ' + ','.join(FAILS)}") + sys.exit(0 if not FAILS else 1) + + +if __name__ == "__main__": + main() From abba4769f734657758a7dd4fd27cc00809e6f478 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Mon, 20 Jul 2026 05:12:34 +0000 Subject: [PATCH 03/35] Track research/ source by rule instead of by accident of history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blanket `research/` ignore made git backup depend on WHEN a file was written: anything committed before that line was added stayed tracked, and everything written after it had no backup anywhere. That silently left ~70 load-bearing, unit-tested files with a single copy on one disk — including posttrain_losses.py (the SFT/DPO/GRPO loss math), scaling_ladder.py, the CCE kernel trio, kernel_oracle.py, data_decontam.py, boot_resume.sh and thermal_log.py (the whole crash/thermal recovery chain), and 11 of the 34 test modules. Their tracked siblings (ledger.py, sentinel tests, eval_stats.py, CI) were safe purely by having been written earlier. It also silently overrode research/.gitignore, whose own stated policy is to "keep the durable record tracked; exclude only transient/runtime files" — that file has been dead letter for as long as the blanket rule existed, because git never descends into an excluded directory to read it. Now: `research/**` with directories re-included so git descends, `.py`/`.sh` re-included as durable source, and harness-search's machine-GENERATED candidate files (archive/, targets/*/candidates/) re-excluded since they are search output, not authored source. 38 source files + research/recovery/.gitignore land here — 39 new paths, 392 K, no blobs; `git status` stays instant (602 files under research/). Deliberately still local-only, unchanged by this commit: ledger.json and loop_state.json (churning state, where a stale tracked copy is worse than none — durable backup is research/backup_ledger.sh), and the generated artifacts under digests/, pulse/, radar/, provenance/, ledger/runs/ and ledger/backups/. Whether the durable RECORD (run detail md, briefs, digests) should also be tracked is a separate call and is left open — research/.gitignore's original intent says yes, the branch-switch incident that destroyed a ledger.json says be careful. Caveat worth knowing: these paths are tracked on this branch only, so a `git checkout main` will remove them from the working tree until this branch is merged or checked back out. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 30 +- research/ablation_config.py | 207 +++++++ research/boot_resume.sh | 189 ++++++ research/build_skills_page.py | 214 +++++++ research/cron_runner.sh | 32 + research/data_decontam.py | 141 +++++ .../prepare_grpo_prompts.py | 170 ++++++ .../math-eval-v1/prepare_math_eval.py | 150 +++++ .../prepare_openr1-math-220k.py | 293 ++++++++++ research/distributed_correctness.py | 225 ++++++++ research/eval_math_acc.py | 225 ++++++++ .../tasks/codeharness/hard_benchmark.py | 221 +++++++ .../tasks/codeharness/probe_headroom.py | 128 ++++ .../tasks/codeharness/probe_repair.py | 81 +++ .../tasks/codeharness/repair_harness.py | 71 +++ research/interp/cka_probe.py | 227 ++++++++ research/kernel/cce_linear_ce.py | 341 +++++++++++ research/kernel/cce_torch.py | 259 +++++++++ research/kernel/cce_triton.py | 545 ++++++++++++++++++ research/kernel/gb10_cce_bench.py | 117 ++++ research/kernel/pause_bench_resume.sh | 43 ++ research/kernel_oracle.py | 295 ++++++++++ research/liveness_cron.sh | 30 + .../figures/make_figures.py | 140 +++++ research/posttrain_losses.py | 175 ++++++ research/recovery/.gitignore | 3 + research/render_md.py | 184 ++++++ research/scaling_ladder.py | 377 ++++++++++++ research/tests/test_ablation_config.py | 145 +++++ research/tests/test_boot_resume.py | 243 ++++++++ research/tests/test_cce_linear_ce.py | 389 +++++++++++++ research/tests/test_cka_probe.py | 44 ++ research/tests/test_data_decontam.py | 97 ++++ .../tests/test_distributed_correctness.py | 166 ++++++ research/tests/test_eval_math_acc.py | 148 +++++ research/tests/test_framework.py | 76 +++ research/tests/test_kernel_oracle.py | 209 +++++++ research/tests/test_posttrain_losses.py | 145 +++++ research/tests/test_scaling_ladder.py | 214 +++++++ research/thermal_log.py | 136 +++++ 40 files changed, 7120 insertions(+), 5 deletions(-) create mode 100644 research/ablation_config.py create mode 100755 research/boot_resume.sh create mode 100644 research/build_skills_page.py create mode 100755 research/cron_runner.sh create mode 100644 research/data_decontam.py create mode 100644 research/datasets/grpo-math-prompts-v1/prepare_grpo_prompts.py create mode 100644 research/datasets/math-eval-v1/prepare_math_eval.py create mode 100644 research/datasets/math-reasoning-openr1-math-220k/prepare_openr1-math-220k.py create mode 100644 research/distributed_correctness.py create mode 100644 research/eval_math_acc.py create mode 100644 research/harness_search/tasks/codeharness/hard_benchmark.py create mode 100644 research/harness_search/tasks/codeharness/probe_headroom.py create mode 100644 research/harness_search/tasks/codeharness/probe_repair.py create mode 100644 research/harness_search/tasks/codeharness/repair_harness.py create mode 100644 research/interp/cka_probe.py create mode 100644 research/kernel/cce_linear_ce.py create mode 100644 research/kernel/cce_torch.py create mode 100644 research/kernel/cce_triton.py create mode 100644 research/kernel/gb10_cce_bench.py create mode 100755 research/kernel/pause_bench_resume.sh create mode 100644 research/kernel_oracle.py create mode 100755 research/liveness_cron.sh create mode 100644 research/papers/qwen3-imu1-matched-compute/figures/make_figures.py create mode 100644 research/posttrain_losses.py create mode 100644 research/recovery/.gitignore create mode 100644 research/render_md.py create mode 100644 research/scaling_ladder.py create mode 100644 research/tests/test_ablation_config.py create mode 100644 research/tests/test_boot_resume.py create mode 100644 research/tests/test_cce_linear_ce.py create mode 100644 research/tests/test_cka_probe.py create mode 100644 research/tests/test_data_decontam.py create mode 100644 research/tests/test_distributed_correctness.py create mode 100644 research/tests/test_eval_math_acc.py create mode 100644 research/tests/test_framework.py create mode 100644 research/tests/test_kernel_oracle.py create mode 100644 research/tests/test_posttrain_losses.py create mode 100644 research/tests/test_scaling_ladder.py create mode 100644 research/thermal_log.py diff --git a/.gitignore b/.gitignore index 4e87bc3..835e2f3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,11 +40,31 @@ SmolLM2-134(base)/results/lm_eval/ # Claude Code state (skills, settings, scheduled tasks, session caches) .claude/ -# Research-loop working tree — churning state (ledger.json, loop_state.json), -# generated artifacts, concurrent-session output, manuscript packages. Kept -# local only; already-committed hardening files (ledger.py, sentinel tests, -# eval_stats.py, CI) stay tracked. Durable ledger backup: research/backup_ledger.sh. -research/ +# Research-loop working tree. Durable SOURCE is tracked by rule; churning STATE +# and generated artifacts stay local-only. +# +# The previous blanket `research/` made backup depend on accident of history: a +# file was safe only if it happened to be committed BEFORE that line was added. +# Everything written after it — ~70 load-bearing, unit-tested .py/.sh files +# including posttrain_losses.py, scaling_ladder.py, boot_resume.sh, thermal_log.py, +# cce_*.py and 11 test modules — had NO backup anywhere, while their siblings +# (ledger.py, sentinel tests, eval_stats.py, CI) stayed tracked. It also silently +# overrode research/.gitignore, whose own stated policy is to "keep the durable +# record tracked; exclude only transient/runtime files". Source now follows that +# policy again; state does not. +# +# Deliberately still local-only: ledger.json / loop_state.json (churning state — +# a stale tracked copy is worse than none; durable backup = research/backup_ledger.sh), +# generated artifacts (digests/, pulse/, radar/, provenance/), prepared datasets, +# manuscript build output, and harness-search's machine-GENERATED candidates +# (search output, not authored source). +research/** +# re-include directories so git descends and the rules below can match +!research/**/ +!research/**/*.py +!research/**/*.sh +research/harness_search/archive/ +research/harness_search/targets/*/candidates/ # Skills showcase (Claude skill .md files + demo site) — kept local only skills_showcase/ diff --git a/research/ablation_config.py b/research/ablation_config.py new file mode 100644 index 0000000..693033b --- /dev/null +++ b/research/ablation_config.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""research/ablation_config.py — the validated cohort builder for an RS-rigorous +ablation, so the NEXT run is a 'clean win', not a 'scoped win'. + +The audit's three confounds in the prior ablation were: + 1. n<5 seeds -> the seed CI was too wide to separate a real effect from noise. + 2. a SHARED hyperparameter (one wd / one LR for both arms) -> a technique that + merely shifts the optimal wd/LR looks like a win or loss for the wrong reason. + 3. a FIXED train/val split -> seeds only captured init+shuffle noise, never the + corpus-resampling variance, so the CI understated the true uncertainty. + +This module builds + VALIDATES the cohort that closes all three: + * each ARM carries its OWN hyperparameters (per-arm wd/LR, etc.), so every arm + runs at ITS optimum and the comparison is confound-free. + * n_seeds is first-class; a 'clean' verdict tier REQUIRES n_seeds >= 5, and + validate() hard-REFUSES n_seeds < 3 (one or two seeds can never be a result). + * randomized_split toggles data_decontam's per-cell split: randomized=True => + every (arm, seed) cell takes a DIFFERENT doc-disjoint split (full end-to-end + variance); False => the legacy fixed split (init+shuffle noise only). + * cells() enumerates the (arm, seed) grid with the EXACT split seed each cell + must use (via data_decontam.cell_split_seed), so the launcher is deterministic. + +Stdlib-only, pure, deterministic — no torch, no network, no model loads. The +contract is unit-tested (test_ablation_config.py) before any GPU time is spent. +""" +from __future__ import annotations + +from dataclasses import dataclass, field, asdict + +from data_decontam import cell_split_seed + +# Verdict tiers by seed count. A 'clean' (publishable, confound-free) result needs +# a real seed cohort; below that the verdict is explicitly demoted. +CLEAN_MIN_SEEDS = 5 # >=5 seeds -> eligible for the 'clean' tier +MIN_SEEDS = 3 # <3 seeds -> validate() refuses to build the cohort + + +@dataclass(frozen=True) +class Arm: + """One arm of the ablation. `name` identifies it; `is_baseline` marks the + control. `hparams` are PER-ARM (e.g. {'lr': 3e-4, 'wd': 0.1}) so each arm runs + at its own tuned optimum — that is what turns a confounded comparison into a + clean one. `hparams` must be a non-empty mapping of finite numeric (or string) + values; an empty hparams means 'shared/untuned', which is exactly the confound + we forbid.""" + name: str + hparams: dict = field(default_factory=dict) + is_baseline: bool = False + + def validate(self) -> None: + if not isinstance(self.name, str) or not self.name.strip(): + raise ValueError("arm name must be a non-empty string") + if not isinstance(self.hparams, dict) or not self.hparams: + raise ValueError( + f"arm {self.name!r} has no per-arm hparams — every arm must carry " + "its OWN tuned hyperparameters (e.g. lr/wd) to avoid the shared-hparam confound") + for k, v in self.hparams.items(): + if not isinstance(k, str) or not k: + raise ValueError(f"arm {self.name!r}: hparam keys must be non-empty strings") + if isinstance(v, bool): + continue + if isinstance(v, (int, float)): + if v != v or v in (float("inf"), float("-inf")): + raise ValueError(f"arm {self.name!r}: hparam {k!r} is non-finite") + elif not isinstance(v, str): + raise ValueError( + f"arm {self.name!r}: hparam {k!r} must be number/bool/str, got {type(v).__name__}") + + +@dataclass +class CohortConfig: + """A multi-seed, per-arm-tuned ablation cohort. + + Fields: + arms : list[Arm] — exactly one must be the baseline; >=1 treatment. + n_seeds : int — training seeds PER arm (>=3 to build at all; + >=5 for the 'clean' tier). + base_split_seed : int — the run's base doc-disjoint split seed. + val_fraction : float — fraction of docs routed to val (0,1). + randomized_split : bool — True => a DIFFERENT split per (arm,seed) cell + (full variance); False => fixed split. + seed_start : int — first training seed; cell seeds are + seed_start .. seed_start+n_seeds-1. + direction : str — 'lower_is_better' (PPL/loss) or 'higher_is_better'. + notes : str — free-form provenance. + """ + arms: list + n_seeds: int = CLEAN_MIN_SEEDS + base_split_seed: int = 0 + val_fraction: float = 0.05 + randomized_split: bool = False + seed_start: int = 0 + direction: str = "lower_is_better" + notes: str = "" + + def validate(self) -> "CohortConfig": + """Refuse a cohort that cannot yield an honest stat. Returns self so it + chains. Raises ValueError on any violation; NEVER silently downgrades.""" + if self.n_seeds < MIN_SEEDS: + raise ValueError( + f"n_seeds={self.n_seeds} < {MIN_SEEDS}: one or two seeds is not a " + "result — a cohort needs at least 3 training seeds to estimate variance") + if not isinstance(self.arms, (list, tuple)) or len(self.arms) < 2: + raise ValueError("a cohort needs >=2 arms (one baseline + >=1 treatment)") + arms = list(self.arms) + for a in arms: + if not isinstance(a, Arm): + raise ValueError(f"each arm must be an Arm, got {type(a).__name__}") + a.validate() + names = [a.name for a in arms] + if len(set(names)) != len(names): + raise ValueError(f"duplicate arm names: {names}") + n_base = sum(1 for a in arms if a.is_baseline) + if n_base != 1: + raise ValueError(f"exactly one arm must have is_baseline=True (found {n_base})") + if not (0.0 < self.val_fraction < 1.0): + raise ValueError("val_fraction must be in (0,1)") + if self.direction not in ("lower_is_better", "higher_is_better"): + raise ValueError( + f"direction must be lower_is_better|higher_is_better, got {self.direction!r}") + if not isinstance(self.randomized_split, bool): + raise ValueError("randomized_split must be a bool") + return self + + # ----- derived views (all assume validate() has been or will be called) ----- + + @property + def is_clean_tier(self) -> bool: + """True iff this cohort is eligible for a 'clean' (publishable) verdict: + >=5 seeds AND a randomized per-cell split (so the CI reflects corpus + resampling, not just init+shuffle).""" + return self.n_seeds >= CLEAN_MIN_SEEDS and self.randomized_split + + def verdict_tier(self) -> str: + """The verdict tier this cohort's DESIGN can support (independent of the + eventual numbers): 'clean' (>=5 seeds + randomized split), 'scoped' + (enough seeds but fixed split and/or <5 seeds), or 'invalid' (<3 seeds).""" + if self.n_seeds < MIN_SEEDS: + return "invalid" + if self.is_clean_tier: + return "clean" + return "scoped" + + def baseline_arm(self) -> Arm: + for a in self.arms: + if a.is_baseline: + return a + raise ValueError("no baseline arm — call validate() first") + + def treatment_arms(self) -> list: + return [a for a in self.arms if not a.is_baseline] + + def seeds(self) -> list: + return list(range(self.seed_start, self.seed_start + self.n_seeds)) + + def cells(self) -> list: + """Enumerate the full (arm, seed) grid as cell dicts the launcher consumes. + Each cell carries its arm name, the arm's per-arm hparams, the training + seed, and the EXACT split seed to use (fixed => base_split_seed for every + cell; randomized => a distinct per-cell split seed via cell_split_seed). + Deterministic and order-stable (arms outer, seeds inner).""" + out = [] + for arm in self.arms: + for s in self.seeds(): + split_seed = cell_split_seed(self.base_split_seed, s, + randomized=self.randomized_split) + out.append({ + "arm": arm.name, + "is_baseline": arm.is_baseline, + "hparams": dict(arm.hparams), + "train_seed": s, + "split_seed": split_seed, + "val_fraction": self.val_fraction, + "randomized_split": self.randomized_split, + }) + return out + + def to_dict(self) -> dict: + """Serializable form for the ledger / on-disk cohort spec (provenance).""" + d = asdict(self) + d["verdict_tier"] = self.verdict_tier() + d["is_clean_tier"] = self.is_clean_tier + d["n_cells"] = len(self.arms) * self.n_seeds + return d + + +def build_cohort(arms, *, n_seeds: int = CLEAN_MIN_SEEDS, base_split_seed: int = 0, + val_fraction: float = 0.05, randomized_split: bool = False, + seed_start: int = 0, direction: str = "lower_is_better", + notes: str = "") -> CohortConfig: + """Build AND validate a cohort in one call (raises on any invalid spec). + `arms` is a list of Arm objects OR plain dicts ({'name','hparams','is_baseline'}) + for convenience from a yaml/json config.""" + norm = [] + for a in arms: + if isinstance(a, Arm): + norm.append(a) + elif isinstance(a, dict): + norm.append(Arm(name=a.get("name"), + hparams=a.get("hparams", {}) or {}, + is_baseline=bool(a.get("is_baseline", False)))) + else: + raise ValueError(f"arm must be Arm or dict, got {type(a).__name__}") + cfg = CohortConfig(arms=norm, n_seeds=n_seeds, base_split_seed=base_split_seed, + val_fraction=val_fraction, randomized_split=randomized_split, + seed_start=seed_start, direction=direction, notes=notes) + return cfg.validate() diff --git a/research/boot_resume.sh b/research/boot_resume.sh new file mode 100755 index 0000000..6fc6756 --- /dev/null +++ b/research/boot_resume.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# research/boot_resume.sh — GENERIC @reboot cross-reboot recovery for the research loop. +# +# The GB10 silently hard-locks under sustained training load every ~5-10h (thermal, no +# kernel trace). In-process resume (ablation-runner Phase 0) only fires when the loop is +# re-invoked; after a HARD-LOCK REBOOT nothing re-invokes it. This hook — fired ONCE per +# boot from crontab `@reboot` — re-adopts the one §C5-approved in-flight run recorded in +# research/loop_state.json and relaunches it from its checkpoint. It is the promotion of +# the ladder-local boot_resume.sh (the stack that delivered the 420M sweep) into a generic, +# loop-state-driven hook the sanctioned launcher (/ablation-runner) can rely on. +# +# It NEVER initiates a NEW run (that is S4 selection + the full §C5 contract). It only +# resumes a run the launcher already approved and recorded. Fully guarded so it can never +# double-launch (§C4.5), never resume past the §C5 auto-resume cap (no resume-loop that +# re-crashes the box), and never blindly resume after a sentinel kill. +# +# Guards (hardened 2026-07-19 after an adversarial review): +# - MUTUAL EXCLUSION: holds an flock on the SHARED /tmp/research-loop.lock (the same lock +# cron_runner.sh uses), so duplicate @reboot lines AND the liveness->/research-loop +# resume agent path can never both relaunch — exactly one recovery runs per boot. +# - cd's into the repo root first: @reboot cron runs with cwd=$HOME, but resume_cmd is +# repo-relative, so without this the trainer silently never launches. +# - Settles (BOOT_SETTLE_SECS, default 90s) BEFORE preflight, so a post-reboot GPU/driver +# storm does not fail the health gate and forfeit the once-per-boot recovery. +# - Refuses on a sentinel_kill marker (a memory/thermal kill is not auto-resumable at the +# same config — ablation-runner Phase 0 discipline). +# - Shares the §C5 auto_resumes cap via the tested `loop_state.py record-resume`. +# - Arms the sentinel on the PYTHON trainer child, never the run_arms.sh/run_ladder.sh +# wrapper (sentinel signals one pid; watching the wrapper orphans the trainer on SIGKILL). +# +# The agent NEVER auto-installs this (repo policy §C4.2). /ablation-runner PRINTS the +# `@reboot` crontab line at launch; the user pastes it once. It is idempotent — a second +# identical line is harmless (the flock lets only one instance act), and it self-nops once +# loop_state has no in-flight run. +# +# Usage: +# bash research/boot_resume.sh # real: guard -> resume the in-flight run +# bash research/boot_resume.sh --dry-run # print the DECISION only; no lock, no launch, no cap write +# Test hooks (never set in production): BFS_ROOT, BOOT_SETTLE_SECS, BOOT_LOCK. +# +# Exit: 0 = handled (resumed, or a clean reason not to); 1 = preflight failed / cannot cd. +set -u + +# @reboot cron runs with a bare PATH — make python3 / pgrep / flock / the miniforge +# interpreter resolve, or the hook dies before it can resume. Box-specific (single GB10). +export PATH="/opt/miniforge3/bin:$HOME/.local/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH" + +ROOT="${BFS_ROOT:-/home/yashb98/Downloads/BuildFromScratch}" +STATE="$ROOT/research/loop_state.json" +MARKER="$STATE.sentinel_kill" # sentinel.py writes this on a kill (§C6) +RECOVERY="$ROOT/research/recovery" +BOOTLOG="$RECOVERY/boot_resume.log" +CAP=2 # §C5 shared auto-resume cap (== loop_state.MAX_AUTO_RESUMES) +SETTLE="${BOOT_SETTLE_SECS:-90}" # seconds to settle after boot before the health gate +LOCK="${BOOT_LOCK:-/tmp/research-loop.lock}" # SHARED with cron_runner.sh -> excludes the agent-resume path too +# GUARD pattern (broad, fail-safe): any experiment trainer (train_.py) + the wrappers. +# Deliberately matches ablation-runner's per-run train_.py, not a static basename list. +# Env-overridable (BOOT_TRAINER_RE) for hermetic tests only — never set in production. +TRAINER_RE="${BOOT_TRAINER_RE:-train_[A-Za-z0-9_]*\.py|run_arms\.sh|run_ladder\.sh}" +# PID-CAPTURE pattern (precise): the PYTHON trainer child, so the sentinel is armed on the +# memory-holding process, never the bash -lc / run_arms.sh wrapper. Env-overridable for tests. +PIDCAP_RE="${BOOT_PIDCAP_RE:-python[0-9.]* .*train_[A-Za-z0-9_]*\.py}" + +DRY=0 +[ "${1:-}" = "--dry-run" ] && DRY=1 + +mkdir -p "$RECOVERY" +# @reboot cwd is $HOME; resume_cmd (and run_arms.sh's own paths) are repo-relative. +cd "$ROOT" || { echo "DECISION=cannot-cd"; echo "$(date '+%F %T') [boot_resume] cannot cd to $ROOT" >>"$BOOTLOG" 2>/dev/null; exit 1; } + +log(){ echo "$(date '+%F %T') [boot_resume] $*" | tee -a "$BOOTLOG" >&2; } +decide(){ echo "DECISION=$1"; log "decision=$1 ${2:-}"; } +preflight_ok(){ python3 "$ROOT/sentinel.py" preflight >/dev/null 2>&1; } + +# --- read the recovery pointer from loop_state.json (the ONE source of truth) ----------- +# Uses loop_state.py's fail-open load, so a truncated/corrupt state (a lock mid-write) +# yields the canonical schema instead of crashing this hook. +read_state(){ PYTHONPATH="$ROOT/research" python3 - "$STATE" <<'PY' +import sys +try: + import loop_state as ls + st = ls.load(sys.argv[1]) +except Exception: + st = {} +def g(k, d=""): + v = st.get(k, d) + return "" if v is None else str(v) +# \x1f (unit separator): a NON-whitespace delimiter, so bash `read` cannot collapse +# empty leading fields (a tab would — in_flight_run=None then reads as the next field). +print("\x1f".join([g("in_flight_run"), g("resume_cmd"), + g("auto_resumes", "0"), g("train_pid")])) +PY +} + +IFS=$'\x1f' read -r RUN_ID RESUME_CMD AUTO_RESUMES TRAIN_PID < <(read_state) +AUTO_RESUMES="${AUTO_RESUMES:-0}" + +# --- PURE-STATE GUARDS (identical in dry-run and real; evaluated before any side effect) --- +if [ -z "$RUN_ID" ] || [ -z "$RESUME_CMD" ]; then + decide nothing-to-resume "(no in_flight_run / resume_cmd in loop_state)"; exit 0 +fi +if [ -e "$MARKER" ]; then + decide refuse-sentinel-kill "(marker $MARKER present — a memory/thermal kill is NOT auto-resumable at the same config; human must inspect + clear the marker)" + exit 0 +fi +if pgrep -f "$TRAINER_RE" >/dev/null 2>&1; then + decide already-running "(a trainer is alive — no double-launch, §C4.5)"; exit 0 +fi +if [ "$AUTO_RESUMES" -ge "$CAP" ] 2>/dev/null; then + decide cap-reached "(auto_resumes=$AUTO_RESUMES >= $CAP — refusing; never loop-crash the box, §C5)" + exit 0 +fi + +if [ "$DRY" -eq 1 ]; then + # Dry-run runs preflight WITHOUT the settle (fast, read-only) purely to report the decision. + preflight_ok || { decide preflight-fail "(sentinel preflight != 0)"; exit 1; } + decide would-resume "(RUN_ID=$RUN_ID auto_resumes=$AUTO_RESUMES -> would relaunch resume_cmd)" + exit 0 +fi + +# --- REAL RESUME -------------------------------------------------------------------------- +# MUTUAL EXCLUSION: hold the shared lock for the whole critical section (settle -> preflight +# -> record-resume -> spawn). A duplicate @reboot boot_resume, or a liveness-triggered +# cron_runner (which flocks the same file), cannot also relaunch. Released when this exits. +exec 9>"$LOCK" 2>/dev/null || { decide lock-unavailable "(cannot open $LOCK)"; exit 0; } +if ! flock -n 9; then + decide already-locked "(another recovery holds $LOCK — no double-launch, §C4.5)"; exit 0 +fi + +[ "$SETTLE" -gt 0 ] 2>/dev/null && sleep "$SETTLE" # settle BEFORE the health gate so the GPU driver is up + +# Re-probe the guards that can change during the settle window (a kill marker appearing, or +# a trainer coming up via another path). Now holding the lock, so this is race-free. +if pgrep -f "$TRAINER_RE" >/dev/null 2>&1; then + decide already-running "(trainer appeared during settle — no double-launch)"; exit 0 +fi +[ -e "$MARKER" ] && { decide refuse-sentinel-kill "(marker appeared during settle)"; exit 0; } +preflight_ok || { decide preflight-fail "(sentinel preflight != 0 after settle — not launching)"; exit 1; } + +# Account this recovery through the TESTED atomic cap counter (the ONE shared counter); +# `crashed` means the cap is now reached — refuse without launching. +DEC=$(PYTHONPATH="$ROOT/research" python3 "$ROOT/research/loop_state.py" \ + --path "$STATE" record-resume --ts "$(date -Is)" 2>/dev/null \ + | python3 -c 'import sys,json; print(json.load(sys.stdin).get("decision","crashed"))' 2>/dev/null) +if [ "$DEC" != "resume" ]; then + decide cap-reached "(record-resume returned '$DEC' — auto-resume budget exhausted, §C5)"; exit 0 +fi + +# Dense thermal forensics beside the run (pure observability; sentinel.py owns the kill). +pgrep -f 'thermal_log.py' >/dev/null 2>&1 || \ + setsid nohup python3 "$ROOT/research/thermal_log.py" --interval 10 \ + --out "$RECOVERY/${RUN_ID}.thermal.log" >> "$RECOVERY/${RUN_ID}.thermal.stdout" 2>&1 < /dev/null & + +# Relaunch the recorded resume_cmd DETACHED (§C5 launch form), from cwd=$ROOT so its +# repo-relative paths resolve. resume_cmd is idempotent: a multi-arm wrapper skips completed +# arms; a single-arm trainer restores from --resume. +RESUME_LOG="$RECOVERY/${RUN_ID}.resume.log" +echo "===== $(date '+%F %T') @reboot auto-resume: $RUN_ID (auto_resumes now accounted) =====" >> "$RESUME_LOG" +setsid nohup bash -lc "$RESUME_CMD" >> "$RESUME_LOG" 2>&1 < /dev/null & +sleep 5 + +# Find the actual PYTHON trainer child (newest match), NOT the bash -lc / run_arms.sh wrapper +# — the sentinel must watch the memory-holding process or a SIGKILL orphans the real trainer. +NEWPID="$(pgrep -n -f "$PIDCAP_RE" | head -1)" +if [ -z "$NEWPID" ]; then + decide relaunch-unconfirmed "(resume_cmd launched but no python trainer found after 5s — check $RESUME_LOG)" + exit 0 +fi + +# Re-arm the sentinel on the trainer pid UNLESS a wrapper already self-armed one +# (multi-arm run_arms.sh arms a per-arm watcher — Phase 5.2). Never double-watch. +if ! pgrep -f 'sentinel.py watch' >/dev/null 2>&1; then + setsid nohup python3 "$ROOT/sentinel.py" watch --pid "$NEWPID" \ + --log "$RECOVERY/${RUN_ID}.sentinel.log" >/dev/null 2>&1 < /dev/null & + log "armed sentinel watch on trainer pid $NEWPID" +else + log "sentinel watch already alive (wrapper self-armed) — not double-arming" +fi + +# Write the new trainer pid back through loop_state.py (the only sanctioned writer) so the +# next Phase 0 adopt sees the live pid. record-resume already updated auto_resumes. +PYTHONPATH="$ROOT/research" python3 - "$STATE" "$NEWPID" <<'PY' 2>/dev/null || true +import sys, loop_state as ls +p, pid = sys.argv[1], int(sys.argv[2]) +st = ls.load(p); st["train_pid"] = pid; ls.save(p, st) +PY + +decide resumed "(RUN_ID=$RUN_ID new train_pid=$NEWPID auto_resumes=$((AUTO_RESUMES+1)) log=$RESUME_LOG)" +exit 0 diff --git a/research/build_skills_page.py b/research/build_skills_page.py new file mode 100644 index 0000000..bfe9881 --- /dev/null +++ b/research/build_skills_page.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Render every SKILL.md under .claude/skills/ into one self-contained HTML page. + +Re-run any time a skill changes: python3 research/build_skills_page.py +Output: research/skills_overview.html (open in any browser; works offline — +marked.js is inlined from research/.marked.min.js). +""" +import html +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] # BuildFromScratch/ +SKILLS_DIR = ROOT / ".claude" / "skills" +OUT = ROOT / "research" / "skills_overview.html" +MARKED_JS = ROOT / "research" / ".marked.min.js" + +GROUPS = [ + ("Orchestrator", ["research-loop"]), + ("Scouting & research", ["community-pulse", "model-radar", "ml-research"]), + ("Data & experiments", ["dataset-forge", "ablation-runner", "eval-harness"]), + ("Memory, safety & reporting", ["experiment-ledger", "resource-sentinel", "weekly-retro"]), + ("Pre-existing (interactive, approval-gated)", ["from-scratch-build", "finance-research-loop"]), +] +NEW_SKILLS = {n for g, names in GROUPS[:4] for n in names} +OWNED_SCRIPTS = { + "experiment-ledger": [ROOT / "research" / "ledger" / "ledger.py"], + "resource-sentinel": [ROOT / "sentinel.py"], +} + + +def parse_frontmatter(text): + meta, body = {}, text + m = re.match(r"^---\n(.*?)\n---\n(.*)$", text, re.S) + if m: + body = m.group(2) + for line in m.group(1).splitlines(): + kv = re.match(r"^([\w-]+):\s*(.*)$", line) + if kv: + meta[kv.group(1)] = kv.group(2).strip().strip('"') + return meta, body + + +def md_template(dom_id, markdown_text): + """Embed raw markdown for client-side rendering by marked.js.""" + safe = markdown_text.replace("{safe}\n
' + + +def code_block(path): + try: + src = path.read_text() + except OSError as e: + src = f"(unreadable: {e})" + return f"
{html.escape(src)}
" + + +sections, toc = [], [] +missing = [] +for group, names in GROUPS: + toc.append(f"
{html.escape(group)}
") + for name in names: + skill_md = SKILLS_DIR / name / "SKILL.md" + badge = "new" if name in NEW_SKILLS else "existing" + if not skill_md.exists(): + missing.append(name) + toc.append(f"{name}") + sections.append( + f"

/{name}

{badge}" + f"not written yet
" + f"

The authoring workflow has not finished this skill — regenerate the page once it lands.

" + ) + continue + + meta, body = parse_frontmatter(skill_md.read_text()) + lines = skill_md.read_text().count("\n") + 1 + toc.append(f"{name}") + + chips = [f"{lines} lines", + f"{skill_md.relative_to(ROOT)}"] + if meta.get("argument-hint"): + chips.insert(0, f"args: {html.escape(meta['argument-hint'])}") + + extras = [] + refs_dir = SKILLS_DIR / name / "references" + if refs_dir.is_dir(): + for ref in sorted(refs_dir.glob("*.md")): + extras.append( + f"
reference: {ref.name} ({ref.stat().st_size // 1024} KB)" + f"{md_template(f'ref-{name}-{ref.stem}', ref.read_text())}
" + ) + scripts_dir = SKILLS_DIR / name / "scripts" + if scripts_dir.is_dir(): + for s in sorted(scripts_dir.iterdir()): + extras.append(f"
script: scripts/{s.name}{code_block(s)}
") + for s in OWNED_SCRIPTS.get(name, []): + if s.exists(): + extras.append(f"
owned script: {s.relative_to(ROOT)}{code_block(s)}
") + + desc = html.escape(meta.get("description", "")) + sections.append( + f"
" + f"

/{name}

{badge}
" + f"
{''.join(chips)}
" + f"

{desc}

" + f"{md_template(f'md-{name}', body)}" + f"{''.join(extras)}" + f"
" + ) + +banner = "" +if missing: + banner = (f"") + +page = f""" + + +BuildFromScratch — research-loop skills + +
+ +
+{banner} +{''.join(sections)} +
+
+ + +""" + +OUT.write_text(page) +print(f"wrote {OUT} ({OUT.stat().st_size // 1024} KB), {len(missing)} skill(s) still pending: {missing or 'none'}") diff --git a/research/cron_runner.sh b/research/cron_runner.sh new file mode 100755 index 0000000..6c0afcd --- /dev/null +++ b/research/cron_runner.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Headless runner for the autonomous research loop. +# Usage: cron_runner.sh "/research-loop" (or "/weekly-retro") +# Installed via crontab; logs to research/cron_logs/. A flock guard skips the +# night if the previous session is still running. The 6h timeout is a backstop +# against a hung session — training runs are launched detached (nohup) by +# ablation-runner and survive this session exiting. +set -u + +REPO=/home/yashb98/Downloads/BuildFromScratch +CLAUDE=/home/yashb98/.local/bin/claude +PROMPT="${1:-/research-loop}" +LOG_DIR="$REPO/research/cron_logs" +LOCK=/tmp/research-loop.lock + +mkdir -p "$LOG_DIR" +SLUG=$(echo "$PROMPT" | tr -cd '[:alnum:]-' | cut -c1-32) +LOG="$LOG_DIR/$(date +%Y-%m-%d_%H%M)_${SLUG}.log" + +exec 9>"$LOCK" +if ! flock -n 9; then + echo "$(date -Is) previous loop session still running — skipping this fire" >>"$LOG" + exit 0 +fi + +cd "$REPO" || exit 1 +echo "$(date -Is) starting: $PROMPT" >>"$LOG" +timeout --signal=TERM 6h "$CLAUDE" -p "$PROMPT" --dangerously-skip-permissions >>"$LOG" 2>&1 +echo "$(date -Is) finished, exit=$?" >>"$LOG" + +# keep the last 60 logs +ls -1t "$LOG_DIR" | tail -n +61 | while read -r f; do rm -f "$LOG_DIR/$f"; done diff --git a/research/data_decontam.py b/research/data_decontam.py new file mode 100644 index 0000000..71767fd --- /dev/null +++ b/research/data_decontam.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""data_decontam.py — document-disjoint train/val splitting + n-gram decontamination. + +The audit found the build's eval split was the *sequential continuation* of the +train stream with NO independent dedup or benchmark decontamination, so any +reported val PPL/BPB was leak-suspect (an RS rejects it). This module is the +shared, UNIT-TESTED fix the data path calls: + + * is_val_doc(doc, seed, val_fraction) — deterministic, order-independent routing + of a WHOLE document to train or val by a seeded hash, so train/val are + DOCUMENT-DISJOINT (a given doc always lands in the same bucket; no doc spans + the boundary, unlike a token-count cut). + * decontaminate_val(val_docs, train_docs, n, threshold) — drops val docs whose + n-gram (default 13) Jaccard-style overlap with the train corpus exceeds + `threshold`, reusing eval_metrics' contamination primitives. + * decontam_report(...) — the on-disk evidence (docs_dropped, split_seed, n, + threshold) the §C8 ledger lineage + the §C22 provenance span point at. + +Stdlib-only, pure, deterministic — no torch, no network. The logic is tested so +that even before the (paid) run executes, the decontam guarantee is proven. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from eval_metrics import build_ngram_index, ngram_contamination + + +def _doc_hash01(doc: str, seed: int) -> float: + """Deterministic uniform-in-[0,1) hash of a document, mixed with `seed`. + Order-independent: the same document always maps to the same value, so its + train/val assignment never depends on where it appeared in the stream.""" + h = hashlib.sha256(f"{seed}\x00{doc}".encode("utf-8")).digest() + return int.from_bytes(h[:8], "big") / 2**64 + + +def is_val_doc(doc: str, seed: int, val_fraction: float) -> bool: + """Route a WHOLE document to val (True) or train (False). ~`val_fraction` of + documents land in val, disjoint from train, independent of stream order.""" + if not (0.0 < val_fraction < 1.0): + raise ValueError("val_fraction must be in (0,1)") + return _doc_hash01(doc, seed) < val_fraction + + +def decontaminate_val(val_docs, train_docs, n: int = 13, threshold: float = 0.8): + """Drop val docs that overlap the train corpus by more than `threshold` of + their n-grams. Returns (kept_docs, dropped_idx, per_item_overlap). Uses 13-gram + word overlap by default (the GPT-3-era decontamination convention).""" + idx = build_ngram_index(train_docs, n) + _, per_item = ngram_contamination(val_docs, idx, n, threshold=threshold) + kept, dropped = [], [] + for i, ov in enumerate(per_item): + (dropped if ov > threshold else kept).append(i) + kept_docs = [val_docs[i] for i in kept] + return kept_docs, dropped, per_item + + +def cell_split_seed(base_seed: int, cell_seed: int, *, randomized: bool) -> int: + """Derive the *split* seed for one cohort cell from the run's base split seed + and the cell's training seed. + + randomized=False -> the FIXED split: every cell gets the SAME doc-disjoint + split (return base_seed), so train/val membership is held constant and the + only thing the seeds vary is init + data-order shuffle. This is the existing + behaviour and the right choice when you want to isolate optimization noise. + + randomized=True -> a DIFFERENT doc-disjoint split per cell, derived by hashing + (base_seed, cell_seed) into a fresh split seed. Each cell then resamples + which documents are train vs val, so the seed spread captures FULL + end-to-end variance — corpus resampling included — not just init+shuffle. + This is the honest variance estimate for an RS 'clean win' (a result that + survives the train/val partition itself being a random draw). + + Deterministic: same (base_seed, cell_seed, randomized) -> same split seed, so + a cohort is exactly reproducible from its config.""" + if not randomized: + return int(base_seed) + h = hashlib.sha256(f"split\x00{int(base_seed)}\x00{int(cell_seed)}".encode("utf-8")).digest() + # 63-bit unsigned -> a stable, non-negative split seed distinct per cell. + return int.from_bytes(h[:8], "big") & ((1 << 63) - 1) + + +def split_and_decontaminate(docs, seed: int, val_fraction: float, + n: int = 13, threshold: float = 0.8, *, + cell_seed: int | None = None, + randomized: bool = False): + """Convenience: doc-disjoint split, then decontaminate the val side against + train. Returns (train_docs, kept_val_docs, report_dict). + + By default this is the FIXED-split path (unchanged): the doc-disjoint split is + keyed on `seed` alone, identical for every call. + + Pass randomized=True together with the cell's training seed (`cell_seed`) to + take a DIFFERENT doc-disjoint split for this cell — derived via + cell_split_seed(seed, cell_seed, randomized=True). The report records both the + base split seed and the effective per-cell split seed so the lineage is exact. + `randomized=True` with `cell_seed=None` is an error (the cell seed is what + makes the split differ).""" + if randomized and cell_seed is None: + raise ValueError("randomized=True requires a cell_seed to derive the per-cell split") + eff_seed = cell_split_seed(seed, cell_seed if cell_seed is not None else 0, + randomized=randomized) + train_docs, raw_val = [], [] + for d in docs: + (raw_val if is_val_doc(d, eff_seed, val_fraction) else train_docs).append(d) + kept_val, dropped, _ = decontaminate_val(raw_val, train_docs, n, threshold) + report = decontam_report(eff_seed, val_fraction, n, threshold, + n_train_docs=len(train_docs), n_val_raw=len(raw_val), + docs_dropped=len(dropped)) + # lineage: distinguish the base split seed from the effective per-cell one + report["base_split_seed"] = int(seed) + report["cell_seed"] = None if cell_seed is None else int(cell_seed) + report["randomized_split"] = bool(randomized) + if randomized: + report["method"] = ("randomized per-cell doc-disjoint seeded-hash split " + "+ 13-gram word-overlap decontam") + return train_docs, kept_val, report + + +def decontam_report(seed, val_fraction, n, threshold, *, n_train_docs, + n_val_raw, docs_dropped) -> dict: + """The on-disk decontamination evidence (write next to the token cache).""" + return { + "split_seed": seed, + "val_fraction": val_fraction, + "ngram_n": n, + "overlap_threshold": threshold, + "n_train_docs": n_train_docs, + "n_val_docs_raw": n_val_raw, + "docs_dropped": docs_dropped, + "n_val_docs_kept": n_val_raw - docs_dropped, + "method": "doc-disjoint seeded-hash split + 13-gram word-overlap decontam", + } + + +def write_decontam_report(path, report: dict) -> None: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(report, indent=2) + "\n") diff --git a/research/datasets/grpo-math-prompts-v1/prepare_grpo_prompts.py b/research/datasets/grpo-math-prompts-v1/prepare_grpo_prompts.py new file mode 100644 index 0000000..7ce06ac --- /dev/null +++ b/research/datasets/grpo-math-prompts-v1/prepare_grpo_prompts.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Prepare the GRPO TRAINING prompt+answer set (P1 of research/rlvr/plan.md §3, unlocked by +the Phase-1 GO on 2026-07-02): verifiable-reward math prompts for the exploratory Dr.GRPO arm +and its iso-compute RFT + random-reward controls. + +Sets (→ {prompt, gold} JSONL, same template + extractor as math-eval-v1 / math-acc-v1): + - GSM8K TRAIN (openai/gsm8k main:train, 7,473) — gold = number after '####'. + - MATH train levels 1-3 (EleutherAI/hendrycks_math, all subjects) — gold = last \\boxed{} + of the reference solution via the PINNED math-acc-v1 extractor. Skipped with a recorded + reason if the dataset is unavailable (GSM8K-only v1 is still runnable). + +DECONTAMINATION (direction matters for a TRAINING set): + - vs the EVAL sets (math-eval-v1 gsm8k_test + math500): 13-gram overlap > 0.5 → DROPPED + from *_clean.jsonl (training on an eval neighbor corrupts the decision metric). + - vs the SFT problems (OpenR1 used uuids): FLAGGED ONLY (`sft_overlap`) — seeing SFT data + again in RL is data reuse, not eval contamination; recorded for attribution honesty. + +CPU-only, no torch. Run: python3 research/datasets/grpo-math-prompts-v1/prepare_grpo_prompts.py +Smoke: --limit 20. +""" +from __future__ import annotations +import argparse, json, os, sys, pathlib, re + +ROOT = pathlib.Path("/home/yashb98/Downloads/BuildFromScratch") +sys.path.insert(0, str(ROOT)) +from research.eval_metrics import build_ngram_index, ngram_contamination +from research.eval_math_acc import _last_boxed, EXTRACTOR_VERSION + +HF_CACHE = "/home/yashb98/projects/qwen-distill/hf_cache" +OUT = ROOT / "research/datasets/grpo-math-prompts-v1" +EVAL_DIR = ROOT / "research/datasets/math-eval-v1" +SFT_META = ROOT / "research/datasets/math-reasoning-openr1-math-220k/train_meta.jsonl" +NGRAM_N, FLAG_THRESHOLD = 13, 0.5 +PROMPT_TMPL = "{q}\n\nPlease reason step by step, and put your final answer within \\boxed{{}}." +_NUM = re.compile(r"-?\d[\d,]*(?:\.\d+)?") +_LEVEL = re.compile(r"Level (\d)") + + +def gsm8k_gold(answer: str) -> str: + tail = answer.rsplit("####", 1)[-1] + m = _NUM.search(tail) + return (m.group(0).replace(",", "") if m else tail.strip()) + + +def load_gsm8k_train(limit=None): + os.environ["HF_HOME"] = HF_CACHE + from datasets import load_dataset + ds = load_dataset("openai/gsm8k", "main", split="train") + if limit: + ds = ds.select(range(min(limit, len(ds)))) + return [{"prompt": PROMPT_TMPL.format(q=r["question"]), "question": r["question"], + "gold": gsm8k_gold(r["answer"]), "source": "openai/gsm8k:train"} for r in ds] + + +def load_math_l13(limit=None): + """MATH train levels 1-3, all subjects; gold via the pinned math-acc-v1 boxed extractor. + Returns (items, err) — err recorded if unavailable (GSM8K-only v1 still valid).""" + os.environ["HF_HOME"] = HF_CACHE + from datasets import load_dataset + subjects = ["algebra", "counting_and_probability", "geometry", "intermediate_algebra", + "number_theory", "prealgebra", "precalculus"] + items = [] + try: + for sub in subjects: + ds = load_dataset("EleutherAI/hendrycks_math", sub, split="train", + trust_remote_code=False) + for r in ds: + m = _LEVEL.search(r.get("level") or "") + if not m or int(m.group(1)) > 3: + continue + gold = _last_boxed(r.get("solution") or "") + if not gold: + continue + items.append({"prompt": PROMPT_TMPL.format(q=r["problem"]), + "question": r["problem"], "gold": gold, + "level": int(m.group(1)), "subject": sub, + "source": f"EleutherAI/hendrycks_math/{sub}:train"}) + if limit and len(items) >= limit: + return items[:limit], None + return items, None + except Exception as e: + return items, f"{type(e).__name__}: {str(e)[:140]}" + + +def overlap_tag(items, index, key): + _, ov = ngram_contamination([it["question"] for it in items], index, NGRAM_N, FLAG_THRESHOLD) + n = 0 + for it, o in zip(items, ov): + it[key] = round(o, 4) + n += o > FLAG_THRESHOLD + return n + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=None) + args = ap.parse_args() + OUT.mkdir(parents=True, exist_ok=True) + log = open(OUT / "prep.log", "w") + def say(*a): print(*a, flush=True); print(*a, file=log) + + say(f"[1/5] GSM8K train{' SMOKE' if args.limit else ''}") + gsm = load_gsm8k_train(args.limit) + say(f" gsm8k_train={len(gsm)}") + + say("[2/5] MATH train levels 1-3 (hendrycks_math)") + math_items, math_err = load_math_l13(args.limit) + say(f" math_l13={len(math_items)}" + (f" (ERR: {math_err})" if math_err else "")) + + say("[3/5] eval-set decontam index (math-eval-v1 gsm8k_test + math500 questions)") + eval_qs = [] + for f in ("gsm8k_test.jsonl", "math500.jsonl"): + for line in open(EVAL_DIR / f): + r = json.loads(line) + eval_qs.append(r["prompt"].split("\n\nPlease reason", 1)[0]) + eval_idx = build_ngram_index(eval_qs, NGRAM_N) + say(f" eval questions={len(eval_qs)} index_13grams={len(eval_idx):,}") + n_gsm_drop = overlap_tag(gsm, eval_idx, "eval_overlap") + n_math_drop = overlap_tag(math_items, eval_idx, "eval_overlap") if math_items else 0 + say(f" eval-overlap DROPS: gsm8k={n_gsm_drop} math_l13={n_math_drop}") + + say("[4/5] vs-SFT flag (OpenR1 used problems; info-only)") + sft_status = "done" + try: + os.environ["HF_HOME"] = HF_CACHE + from datasets import load_dataset + used = {json.loads(l)["source_uuid"] for l in open(SFT_META)} + ds = load_dataset("open-r1/OpenR1-Math-220k", "default", split="train") + ds = ds.select_columns(["uuid", "problem"]) + sft_idx = build_ngram_index([r["problem"] for r in ds if r["uuid"] in used and r["problem"]], NGRAM_N) + n_gsm_sft = overlap_tag(gsm, sft_idx, "sft_overlap") + n_math_sft = overlap_tag(math_items, sft_idx, "sft_overlap") if math_items else 0 + say(f" sft-overlap FLAGS (kept): gsm8k={n_gsm_sft} math_l13={n_math_sft}") + except Exception as e: + sft_status = f"PENDING ({type(e).__name__}: {str(e)[:100]})" + n_gsm_sft = n_math_sft = None + say(f" vs-SFT flag unavailable: {sft_status}") + + say(f"[5/5] writing {OUT}") + def clean(items): + return [x for x in items if x["eval_overlap"] <= FLAG_THRESHOLD] + keys = ["prompt", "gold", "source", "eval_overlap", "sft_overlap", "level", "subject"] + def w(path, items): + with open(path, "w") as f: + for it in items: + f.write(json.dumps({k: it[k] for k in keys if k in it}) + "\n") + w(OUT / "gsm8k_train.jsonl", gsm) + w(OUT / "gsm8k_train_clean.jsonl", clean(gsm)) + if math_items: + w(OUT / "math_l13_train.jsonl", math_items) + w(OUT / "math_l13_train_clean.jsonl", clean(math_items)) + report = { + "prepared_for": "research/rlvr/plan.md P1 (GRPO prompt set) — unlocked by Phase-1 GO 2026-07-02", + "extractor": EXTRACTOR_VERSION, "ngram_n": NGRAM_N, "flag_threshold": FLAG_THRESHOLD, + "eval_decontam": "vs math-eval-v1 (gsm8k_test+math500) — overlaps DROPPED from *_clean", + "sft_flag_status": sft_status, + "gsm8k_train": {"n": len(gsm), "eval_dropped": n_gsm_drop, "clean": len(clean(gsm)), + "sft_flagged": n_gsm_sft}, + "math_l13_train": {"n": len(math_items), "eval_dropped": n_math_drop, + "clean": len(clean(math_items)), "sft_flagged": n_math_sft, + "load_error": math_err}, + "smoke_limit": args.limit, + } + json.dump(report, open(OUT / "decontam_report.json", "w"), indent=2) + say("DONE", json.dumps({k: report[k] for k in ("gsm8k_train", "math_l13_train")})) + log.close() + + +if __name__ == "__main__": + main() diff --git a/research/datasets/math-eval-v1/prepare_math_eval.py b/research/datasets/math-eval-v1/prepare_math_eval.py new file mode 100644 index 0000000..4271be4 --- /dev/null +++ b/research/datasets/math-eval-v1/prepare_math_eval.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Prepare the held-out math-accuracy EVAL band (P2 of research/rlvr/plan.md §C27): the +decision-metric sets for the Phase-1 pass@k go/no-go and every downstream reasoning run. + +Two sets → {prompt, gold} JSONL that research.eval_math_acc.run_math_acc consumes directly: + - GSM8K test (openai/gsm8k, 1319 items) — gold = the number after '####'. + - MATH-500 (HuggingFaceH4/MATH-500, 500 items) — gold = the dataset `answer` field. + +DECONTAMINATION (the load-bearing hygiene): both are checked with a 13-gram overlap against +the ACTUAL 35,231 OpenR1-Math-220k problems the model was SFT'd on (identified by the +source_uuids in the prepared SFT set). This matters because OpenR1-Math contains MATH-style +problems, so MATH-500 could overlap the SFT data — scoring on a memorized problem would fake +a reasoning signal. Items over the flag threshold are marked `contaminated: true` and excluded +from the `*_clean.jsonl` the go/no-go scores; the full set + a decontam_report are kept. + +CPU-only, no torch (forge convention). Offline-first via the on-box HF cache. Run: + python3 research/datasets/math-eval-v1/prepare_math_eval.py +Smoke: add --limit 20 (tiny, for a fast wiring check — does NOT gate the frozen sets). +""" +from __future__ import annotations +import argparse, json, os, re, sys, pathlib + +ROOT = pathlib.Path("/home/yashb98/Downloads/BuildFromScratch") +sys.path.insert(0, str(ROOT)) +from research.eval_metrics import build_ngram_index, ngram_contamination + +HF_CACHE = "/home/yashb98/projects/qwen-distill/hf_cache" # GSM8K + OpenR1 already cached here +OUT = ROOT / "research/datasets/math-eval-v1" +SFT_META = ROOT / "research/datasets/math-reasoning-openr1-math-220k/train_meta.jsonl" +NGRAM_N = 13 # the plan's 13-gram decontam +FLAG_THRESHOLD = 0.5 # eval item flagged if >50% of its 13-grams appear in SFT +PROMPT_TMPL = "{q}\n\nPlease reason step by step, and put your final answer within \\boxed{{}}." +_NUM = re.compile(r"-?\d[\d,]*(?:\.\d+)?") + + +def gsm8k_gold(answer: str) -> str: + tail = answer.rsplit("####", 1)[-1] + m = _NUM.search(tail) + return (m.group(0).replace(",", "") if m else tail.strip()) + + +def load_sets(limit=None): + os.environ["HF_HOME"] = HF_CACHE + from datasets import load_dataset + gsm = load_dataset("openai/gsm8k", "main", split="test") + m500 = load_dataset("HuggingFaceH4/MATH-500", split="test") + if limit: + gsm, m500 = gsm.select(range(min(limit, len(gsm)))), m500.select(range(min(limit, len(m500)))) + gsm_items = [{"prompt": PROMPT_TMPL.format(q=r["question"]), "question": r["question"], + "gold": gsm8k_gold(r["answer"]), "source": "openai/gsm8k:test"} for r in gsm] + m500_items = [{"prompt": PROMPT_TMPL.format(q=r["problem"]), "question": r["problem"], + "gold": str(r["answer"]).strip(), "level": r.get("level"), + "subject": r.get("subject"), "source": "HuggingFaceH4/MATH-500:test"} for r in m500] + return gsm_items, m500_items + + +def sft_problem_texts(): + """The raw text of the OpenR1 problems actually used in the SFT set (by uuid), for the + vs-SFT decontam. OpenR1 was STREAMED by the SFT prep (no processed offline cache), so this + succeeds only if OpenR1 is loadable; on offline-miss it returns ([], n_used, reason) and the + caller records the vs-SFT decontam as PENDING (online) rather than silently skipping it.""" + os.environ["HF_HOME"] = HF_CACHE + from datasets import load_dataset + used = {json.loads(l)["source_uuid"] for l in open(SFT_META)} + try: + ds = load_dataset("open-r1/OpenR1-Math-220k", "default", split="train") + ds = ds.select_columns(["uuid", "problem"]) # project away the huge generations/messages + texts = [r["problem"] for r in ds if r["uuid"] in used and r["problem"]] + return texts, len(used), None + except Exception as e: + return [], len(used), f"{type(e).__name__}: {str(e)[:120]}" + + +def decontam(items, index): + _, overlap = ngram_contamination([it["question"] for it in items], index, NGRAM_N, FLAG_THRESHOLD) + n_flag = 0 + for it, ov in zip(items, overlap): + it["ngram_overlap"] = round(ov, 4) + it["contaminated"] = ov > FLAG_THRESHOLD + n_flag += it["contaminated"] + return n_flag + + +def write_jsonl(path, items, keys): + with open(path, "w") as f: + for it in items: + f.write(json.dumps({k: it[k] for k in keys if k in it}) + "\n") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=None, help="tiny smoke subset; does not gate the frozen sets") + args = ap.parse_args() + OUT.mkdir(parents=True, exist_ok=True) + log = open(OUT / "prep.log", "w") + def say(*a): print(*a); print(*a, file=log) + + say(f"[1/4] loading GSM8K-test + MATH-500 (offline cache {HF_CACHE}){' SMOKE' if args.limit else ''}") + gsm, m500 = load_sets(args.limit) + say(f" gsm8k_test={len(gsm)} math500={len(m500)}") + + say(f"[2/4] building 13-gram decontam index from the SFT problems (OpenR1, used uuids)") + sft_texts, n_uuid, sft_err = sft_problem_texts() + if sft_texts: + index = build_ngram_index(sft_texts, NGRAM_N) + decontam_target = f"{len(sft_texts)} OpenR1 SFT problems (of {n_uuid} used uuids)" + sft_status = "done" + say(f" sft_problems={len(sft_texts)} (of {n_uuid} used uuids) index_13grams={len(index):,}") + else: + index = None # vs-SFT decontam PENDING (OpenR1 not offline-cached) + decontam_target = f"cross-eval only (vs-SFT PENDING online: {sft_err})" + sft_status = f"PENDING-ONLINE ({sft_err})" + say(f" OpenR1 unavailable offline ({sft_err}); vs-SFT decontam DEFERRED — cross-eval check only") + + say(f"[3/4] decontaminating (n={NGRAM_N}, flag>{FLAG_THRESHOLD}); target={decontam_target}") + if index is not None: + n_gsm_flag, n_m5_flag = decontam(gsm, index), decontam(m500, index) + else: # cross-eval leakage: gsm vs math500 and vice versa + n_gsm_flag = decontam(gsm, build_ngram_index([x["question"] for x in m500], NGRAM_N)) + n_m5_flag = decontam(m500, build_ngram_index([x["question"] for x in gsm], NGRAM_N)) + say(f" gsm8k flagged={n_gsm_flag}/{len(gsm)} math500 flagged={n_m5_flag}/{len(m500)}") + + say(f"[4/4] writing {OUT}") + gk = ["prompt", "gold", "source", "ngram_overlap", "contaminated"] + mk = gk + ["level", "subject"] + write_jsonl(OUT / "gsm8k_test.jsonl", gsm, mk) + write_jsonl(OUT / "math500.jsonl", m500, mk) + write_jsonl(OUT / "gsm8k_test_clean.jsonl", [x for x in gsm if not x["contaminated"]], gk) + write_jsonl(OUT / "math500_clean.jsonl", [x for x in m500 if not x["contaminated"]], mk) + report = { + "prepared_for": "research/rlvr/plan.md Phase-1 pass@k go/no-go", + "consumer": "research.eval_math_acc.run_math_acc (extractor math-acc-v1)", + "ngram_n": NGRAM_N, "flag_threshold": FLAG_THRESHOLD, + "decontam_target": decontam_target, + "sft_decontam_status": sft_status, + "rerun_vs_sft_online": "HF_HUB_OFFLINE=0 python3 research/datasets/math-eval-v1/prepare_math_eval.py (fetches OpenR1 problems online for the vs-SFT check)", + "index_13grams": len(index) if index is not None else 0, + "gsm8k_test": {"n": len(gsm), "contaminated": n_gsm_flag, "clean": len(gsm) - n_gsm_flag}, + "math500": {"n": len(m500), "contaminated": n_m5_flag, "clean": len(m500) - n_m5_flag}, + "max_overlap_gsm8k": max((x["ngram_overlap"] for x in gsm), default=0.0), + "max_overlap_math500": max((x["ngram_overlap"] for x in m500), default=0.0), + "smoke_limit": args.limit, + } + json.dump(report, open(OUT / "decontam_report.json", "w"), indent=2) + say("DONE", json.dumps(report["gsm8k_test"]), json.dumps(report["math500"])) + log.close() + + +if __name__ == "__main__": + main() diff --git a/research/datasets/math-reasoning-openr1-math-220k/prepare_openr1-math-220k.py b/research/datasets/math-reasoning-openr1-math-220k/prepare_openr1-math-220k.py new file mode 100644 index 0000000..712060e --- /dev/null +++ b/research/datasets/math-reasoning-openr1-math-220k/prepare_openr1-math-220k.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""dataset-forge prep — OpenR1-Math-220k SFT shards for Qwen3-0.6B (math-reasoning). + +Streaming-only (§C1: one ~119 GB unified pool; never list() a split, never hold +the corpus in RAM). No torch import -> no safe_cuda header needed (tokenization +is pure-CPU; the crash mode here is RAM accumulation, guarded by streaming + +bounded array.array buffers per recipes §R4). + +Produces, all under this dir (research/datasets/math-reasoning-openr1-math-220k/): + - shard_*.bin : SFT train tokens, uint32 flat stream, eos-separated + - train_meta.jsonl : one line per SFT sample {n_tokens, prompt_len, source_uuid} + so /ablation-runner's masked-completion CE can mask the prompt + - meta.json : dtype/vocab/tokenizer/dataset id+sha/eos + counts + - eval/eval_docs.jsonl : held-out in-domain reasoning eval docs (post-13-gram-dedup) + - eval/eval_tokens.bin : tokenized eval split (same meta) + - eval/forgetting_docs.jsonl : held-out FineWeb-Edu general probe (catastrophic-forgetting, §C13) + - eval/forgetting_tokens.bin : tokenized forgetting probe (same meta) + - stats.json : final counts + +Hygiene (finance contracts §6 / recipes §R5): + - document-level seeded split FIRST (sha256 mod), seed "forge-v1", eval frac 0.005 + - 13-gram Jaccard dedup of eval docs vs the streamed train side, drop > 0.8 +Re-runnable: completed shards on disk are skipped on resume. +""" +import argparse, array, hashlib, json, sys +from pathlib import Path + +import numpy as np +from datasets import load_dataset +from transformers import AutoTokenizer + +# ----- pins (verified live 2026-06-17; see card.md / selection.md) ----- +REPO = "Qwen/Qwen3-0.6B-Base" +SFT_ID = "open-r1/OpenR1-Math-220k" +SFT_REV = "e4e141ec9dea9f8326f4d347be56105859b2bd68" +SFT_CONFIG = "default" +FORGET_ID = "HuggingFaceFW/fineweb-edu" +FORGET_REV = "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9" +FORGET_CONFIG = "sample-10BT" + +VOCAB_SIZE = 151_936 +DTYPE = np.uint32 # vocab > 65536 -> 4 bytes/token (§R4) +ACODE = "I" # 4-byte unsigned C int +MAX_SEQ = 4096 # build seq_len (train_qwen3.py) +SHARD_TOKENS = 50_000_000 # §R4 +EVAL_FRAC = 0.005 # Constants +EVAL_CAP_TOK = 5_000_000 # Constants +FORGET_CAP_TOK = 5_000_000 +SEED = "forge-v1" # Constants +DEDUP_THRESH = 0.8 # 13-gram Jaccard (finance §6) +NGRAM = 13 + +HERE = Path(__file__).resolve().parent +EVAL_DIR = HERE / "eval"; EVAL_DIR.mkdir(exist_ok=True) + + +def is_eval(doc_key: str, frac=EVAL_FRAC, seed=SEED) -> bool: + h = int(hashlib.sha256(f"{seed}:{doc_key}".encode()).hexdigest(), 16) + return (h % 10_000) < int(frac * 10_000) + + +def grams(text, n=NGRAM): + t = text.split() + return {hash(tuple(t[i:i + n])) for i in range(max(0, len(t) - n + 1))} + + +def pick_verified_trace(row): + """First generation whose correctness_math_verify is True (answer-checked).""" + gens = row.get("generations") or [] + flags = row.get("correctness_math_verify") or [] + for g, f in zip(gens, flags): + if f and g: + return g + return None + + +def sft_text(row, trace): + """prompt + blank line + verified reasoning trace. Returns (full_text, prompt_text).""" + prompt = (row.get("problem") or "").strip() + return prompt + "\n\n" + trace, prompt + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit_tokens", type=int, default=0, + help="bounded slice for the 60s smoke run; 0 = full TOKEN_TARGET") + ap.add_argument("--token_target", type=int, default=125_000_000) + args = ap.parse_args() + TOKEN_TARGET = args.limit_tokens if args.limit_tokens > 0 else args.token_target + smoke = args.limit_tokens > 0 + + tok = AutoTokenizer.from_pretrained(REPO) + eos = tok.eos_token_id + assert eos == 151643, f"unexpected eos {eos}" + print(f"[forge] tokenizer {REPO} len={len(tok)} eos={eos} target={TOKEN_TARGET} smoke={smoke}", + flush=True) + + # ---------- PASS 1: build the held-out in-domain eval set FIRST (doc-level split) ---------- + # Stream a bounded prefix; route eval-bucket rows to the eval side, capped at EVAL_CAP_TOK. + eval_docs, eval_keys = [], [] + eval_tok_total = 0 + eval_scan_cap = 200 if smoke else 20000 # bounded scan to fill the small eval cap + sft = load_dataset(SFT_ID, SFT_CONFIG, split="train", streaming=True, + revision=SFT_REV, trust_remote_code=False) + n_scanned = 0 + for row in sft: + n_scanned += 1 + if n_scanned > eval_scan_cap: + break + key = row.get("uuid") or hashlib.sha256((row.get("problem") or "").encode()).hexdigest() + if not is_eval(key): + continue + trace = pick_verified_trace(row) + if trace is None: + continue + full, _ = sft_text(row, trace) + ids = tok(full).input_ids[:MAX_SEQ] + if eval_tok_total + len(ids) > EVAL_CAP_TOK: + continue + eval_docs.append({"uuid": key, "text": full, "n_tokens": len(ids)}) + eval_keys.append(key) + eval_tok_total += len(ids) + print(f"[forge] eval candidates collected: {len(eval_docs)} docs, {eval_tok_total} tok " + f"(scanned {n_scanned} rows)", flush=True) + + # build the eval gram index for streaming dedup-vs-train + eval_grams = {i: grams(d["text"]) for i, d in enumerate(eval_docs)} + index = {} + for i, gs in eval_grams.items(): + for g in gs: + index.setdefault(g, []).append(i) + + # ---------- PASS 2: stream train side, dedup eval vs train, write shards ---------- + overlap = {} # (eval_i) -> max overlap count seen + train_doc_grams_lens = {} # per train doc gram-count (for jaccard denom) + buf = array.array(ACODE) + shard_idx, train_total, n_train_samples = 0, 0, 0 + eval_key_set = set(eval_keys) + meta_f = open(HERE / "train_meta.jsonl", "w") + + def flush_full_shards(): + nonlocal buf, shard_idx + while len(buf) >= SHARD_TOKENS: + out = HERE / f"shard_{shard_idx:05d}.bin" + np.frombuffer(buf, dtype=DTYPE, count=SHARD_TOKENS).tofile(out) + print(f"[forge] wrote {out.name} ({SHARD_TOKENS} tok) total={train_total}", flush=True) + buf = buf[SHARD_TOKENS:] + shard_idx += 1 + + sft2 = load_dataset(SFT_ID, SFT_CONFIG, split="train", streaming=True, + revision=SFT_REV, trust_remote_code=False) + for row in sft2: + key = row.get("uuid") or hashlib.sha256((row.get("problem") or "").encode()).hexdigest() + if key in eval_key_set: + continue # never let an eval doc into train + trace = pick_verified_trace(row) + if trace is None: + continue # data hygiene: verified traces only + full, prompt = sft_text(row, trace) + + # streaming 13-gram dedup contribution from this train doc + if index: + tg = grams(full) + hit = tg & index.keys() + if hit: + tg_len = len(tg) + for g in hit: + for i in index[g]: + c = overlap.get(i, 0) + 1 + overlap[i] = c + # track the train doc gram-len that produced the best overlap + prev = train_doc_grams_lens.get(i, (0, 0)) + if c >= prev[0]: + train_doc_grams_lens[i] = (c, tg_len) + + full_ids = tok(full).input_ids[:MAX_SEQ] + prompt_ids = tok(prompt).input_ids + prompt_len = min(len(prompt_ids), len(full_ids)) # masked span (clamped to window) + ids = full_ids + [eos] + buf.extend(ids) + train_total += len(ids) + meta_f.write(json.dumps({"n_tokens": len(ids), "prompt_len": prompt_len, + "source_uuid": key}) + "\n") + n_train_samples += 1 + flush_full_shards() + if train_total >= TOKEN_TARGET: + break + # flush remainder + if len(buf): + out = HERE / f"shard_{shard_idx:05d}.bin" + np.frombuffer(buf, dtype=DTYPE).tofile(out) + print(f"[forge] wrote {out.name} ({len(buf)} tok, remainder) total={train_total}", flush=True) + shard_idx += 1 + meta_f.close() + + # ---------- apply dedup verdicts: drop eval docs that overlap train > 0.8 ---------- + docs_dropped = 0 + kept_eval = [] + for i, d in enumerate(eval_docs): + c = overlap.get(i, 0) + if c > 0: + e_len = len(eval_grams[i]) + t_len = train_doc_grams_lens.get(i, (c, c))[1] + jacc = c / max(1, (e_len + t_len - c)) + if jacc > DEDUP_THRESH: + docs_dropped += 1 + continue + kept_eval.append(d) + print(f"[forge] eval dedup: kept {len(kept_eval)}, dropped {docs_dropped} (13-gram jaccard>{DEDUP_THRESH})", + flush=True) + + # write the kept eval split (untokenized + tokenized, same meta) + eval_tok_buf = array.array(ACODE) + eval_tokens = 0 + with open(EVAL_DIR / "eval_docs.jsonl", "w") as f: + for d in kept_eval: + f.write(json.dumps({"uuid": d["uuid"], "text": d["text"]}) + "\n") + ids = tok(d["text"]).input_ids[:MAX_SEQ] + [eos] + eval_tok_buf.extend(ids); eval_tokens += len(ids) + np.frombuffer(eval_tok_buf, dtype=DTYPE).tofile(EVAL_DIR / "eval_tokens.bin") + + # ---------- forgetting probe: FineWeb-Edu general-distribution held-out (§C13) ---------- + forget_scan_cap = 200 if smoke else 40000 + forget_docs, forget_keys = [], [] + forget_tok = 0 + fw = load_dataset(FORGET_ID, FORGET_CONFIG, split="train", streaming=True, + revision=FORGET_REV, trust_remote_code=False) + n = 0 + for row in fw: + n += 1 + if n > forget_scan_cap or forget_tok >= FORGET_CAP_TOK: + break + text = row.get("text") or "" + key = row.get("id") or hashlib.sha256(text.encode()).hexdigest() + if not is_eval(key): # same doc-level seeded split + continue + ids = tok(text).input_ids[:MAX_SEQ] + if forget_tok + len(ids) > FORGET_CAP_TOK: + continue + forget_docs.append({"id": key, "text": text}); forget_keys.append(key) + forget_tok += len(ids) + # the forgetting probe is from a DIFFERENT corpus than train, so train-leak is structurally + # impossible; we still apply the doc-level seeded split for held-out hygiene. Record 0 dropped. + forget_buf = array.array(ACODE); forgetting_tokens = 0 + with open(EVAL_DIR / "forgetting_docs.jsonl", "w") as f: + for d in forget_docs: + f.write(json.dumps(d) + "\n") + ids = tok(d["text"]).input_ids[:MAX_SEQ] + [eos] + forget_buf.extend(ids); forgetting_tokens += len(ids) + np.frombuffer(forget_buf, dtype=DTYPE).tofile(EVAL_DIR / "forgetting_tokens.bin") + print(f"[forge] forgetting probe: {len(forget_docs)} FineWeb-Edu docs, {forgetting_tokens} tok", + flush=True) + + # ---------- meta + stats ---------- + meta = { + "dtype": "uint32", "vocab_size": VOCAB_SIZE, "tokenizer_repo": REPO, + "dataset_id": SFT_ID, "revision": SFT_REV, "dataset_config": SFT_CONFIG, + "eos_token_id": eos, "shard_tokens": SHARD_TOKENS, "max_seq": MAX_SEQ, + "doc_separator": "eos", "sft_format": "prompt + '\\n\\n' + verified_trace; prompt_len in train_meta.jsonl", + "forgetting_source": {"id": FORGET_ID, "config": FORGET_CONFIG, "revision": FORGET_REV}, + } + (HERE / "meta.json").write_text(json.dumps(meta, indent=2)) + + # ---------- readback verification (§R4 gate) ---------- + shard_files = sorted(HERE.glob("shard_*.bin")) + read_total, maxid = 0, 0 + for sf in shard_files: + a = np.fromfile(sf, dtype=DTYPE) + read_total += len(a) + if len(a): + maxid = max(maxid, int(a.max())) + assert read_total == train_total, f"readback {read_total} != written {train_total}" + assert maxid < VOCAB_SIZE, f"max id {maxid} >= vocab {VOCAB_SIZE}" + # decode one 256-token window verbatim into the log + if shard_files: + win = np.fromfile(shard_files[0], dtype=DTYPE)[:256].tolist() + print("[forge] DECODED WINDOW (first shard, 256 tok):", flush=True) + print(tok.decode(win), flush=True) + + stats = { + "train_tokens": train_total, "n_train_samples": n_train_samples, + "shards": len(shard_files), "eval_tokens": eval_tokens, + "eval_docs_kept": len(kept_eval), "docs_dropped": docs_dropped, + "forgetting_tokens": forgetting_tokens, "forgetting_docs": len(forget_docs), + "fertility": 1.92, "max_token_id": maxid, "token_target": TOKEN_TARGET, + "smoke": smoke, + } + (HERE / "stats.json").write_text(json.dumps(stats, indent=2)) + print(f"[forge] DONE {json.dumps(stats)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/research/distributed_correctness.py b/research/distributed_correctness.py new file mode 100644 index 0000000..8782ae9 --- /dev/null +++ b/research/distributed_correctness.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""research/distributed_correctness.py — the §C21 distributed-correctness GATE. + +A multi-GPU (FSDP2 / TP / CP) training run is `inconclusive` BY CONSTRUCTION +unless it first proves it computes the SAME thing the single-GPU baseline does. +A distributed run can be fast AND wrong — a sharding/all-reduce/cast bug shifts +the loss while still descending plausibly — and an MFU number from a wrong run +is a lie. So this gate runs FIRST: before any tokens/sec or MFU figure from a +distributed run is trusted, its loss (or any eval metric) must match the +single-GPU baseline within a noise floor. + +The law (§C21): + + |distributed_metric − single_gpu_baseline| <= tolerance + +where `tolerance` is NOT invented — it is derived from the §C17 seed-noise floor +(the same machinery `eval_stats.py` uses for win/loss verdicts). The reason: a +distributed run that lands within the run-to-run training-seed scatter of the +baseline is indistinguishable from "the same computation, different RNG"; a run +that lands OUTSIDE that scatter is doing something different and must be caught. + +This module is PURE and CPU-testable: it takes synthetic numbers (no torch, no +CUDA, no model, no I/O) and returns a structured pass/fail verdict. The GPU work +that PRODUCES the numbers lives off-box (`remote-launcher` / torchtitan); this is +the correctness logic that JUDGES them, fully unit-tested on CPU. + +It reuses `eval_stats.py` for the noise-floor estimate so the tolerance is the +same statistic the rest of the system already trusts. +""" +from __future__ import annotations + +import math + +import eval_stats as es + +# Below this absolute floor we never let the tolerance collapse to ~0. Two +# identical-seed baseline replicates can have near-zero scatter purely by luck; +# a literally-zero tolerance would then reject a correct distributed run for a +# 1e-7 floating-point reassociation difference (all-reduce changes summation +# order — bit-exactness is NOT expected across a different reduction tree). This +# is a relative floor applied to the baseline magnitude. +_MIN_REL_TOLERANCE = 1e-4 + + +def noise_floor_from_seeds(baseline_seeds, k_sigma=2.0): + """Derive a correctness tolerance from replicate single-GPU baseline seeds. + + The tolerance is `k_sigma` standard deviations of the baseline's seed-to-seed + scatter (default 2σ ≈ the 95% band of run-to-run training noise). A + distributed run inside this band is statistically indistinguishable from the + baseline computation under a different RNG; outside it, it is doing something + different. + + Requires >= 2 seeds (one run gives no variance estimate — and a single-GPU + baseline you cannot vary is not a baseline you can gate against). Raises + ValueError otherwise, so a caller can never silently gate against a fabricated + zero-width floor. + + Returns (tolerance, baseline_mean, baseline_std). + """ + seeds = list(baseline_seeds) + if len(seeds) < 2: + raise ValueError( + f"need >= 2 baseline seeds to estimate a noise floor (got {len(seeds)}); " + "a one-seed baseline gives no variance and cannot gate a distributed run" + ) + if k_sigma <= 0: + raise ValueError(f"k_sigma must be > 0, got {k_sigma!r}") + mean, std, _ = es.mean_std_sem(seeds) # raises on NaN/inf (diverged run) + tolerance = k_sigma * std + return tolerance, mean, std + + +def check_distributed_correctness( + distributed_metric, + single_gpu_baseline, + tolerance=None, + baseline_seeds=None, + k_sigma=2.0, + metric_name="loss", +): + """The §C21 gate. Decide whether a distributed run matches its single-GPU + baseline within the noise floor. + + Exactly ONE source of the tolerance must be supplied: + * `tolerance=` — an explicit absolute tolerance (e.g. a pre-computed + noise floor), OR + * `baseline_seeds=[...]` — replicate single-GPU baseline metrics, from which + a `k_sigma`-σ tolerance is derived via `noise_floor_from_seeds`. + + `single_gpu_baseline` is the reference value the distributed run must match. + When `baseline_seeds` is given and `single_gpu_baseline` is None, the seed + MEAN is used as the reference (the natural single-GPU expectation). + + Returns a dict: + passed bool — |Δ| <= tolerance (False also on any non-finite input) + verdict str — "pass" | "fail" + metric_name str + distributed_metric, single_gpu_baseline, tolerance floats (as resolved) + abs_delta float — |distributed − baseline| + rel_delta float — |Δ| / |baseline| (inf if baseline == 0) + margin float — tolerance − |Δ| (>=0 passes; how much headroom) + n_baseline_seeds int|None + reason str — human-readable one-liner for the ledger/post-mortem + + HARD RULES: + * A NaN/inf distributed_metric => FAIL (a diverged distributed run is the + canonical fast-but-wrong case and must never pass by accident). + * You may not pass BOTH `tolerance` and `baseline_seeds` (ambiguous source + of truth), nor NEITHER (no floor => not gateable, §C21). + * The effective tolerance is floored at `_MIN_REL_TOLERANCE * |baseline|` + so a fluky zero-scatter baseline cannot reject a bit-reassociation-level + difference that is correct-by-design across a different reduction tree. + """ + if (tolerance is None) == (baseline_seeds is None): + raise ValueError( + "supply exactly one of tolerance= or baseline_seeds=[...]; " + "two sources is ambiguous, zero sources is ungateable (§C21)" + ) + + n_seeds = None + if baseline_seeds is not None: + derived_tol, seed_mean, seed_std = noise_floor_from_seeds( + baseline_seeds, k_sigma=k_sigma + ) + tolerance = derived_tol + n_seeds = len(list(baseline_seeds)) + if single_gpu_baseline is None: + single_gpu_baseline = seed_mean + else: + if single_gpu_baseline is None: + raise ValueError( + "single_gpu_baseline is required when tolerance is given explicitly" + ) + tolerance = float(tolerance) + if tolerance < 0: + raise ValueError(f"tolerance must be >= 0, got {tolerance!r}") + + baseline = float(single_gpu_baseline) + dist = float(distributed_metric) + + # Apply the relative floor so a bit-exact-impossible reduction-order diff + # cannot fail against a coincidentally-zero-width baseline scatter. + eff_tolerance = max(tolerance, _MIN_REL_TOLERANCE * abs(baseline)) + + # Non-finite distributed metric => diverged run => hard FAIL (never silently + # NaN-compares to True). + if not math.isfinite(dist): + return { + "passed": False, + "verdict": "fail", + "metric_name": metric_name, + "distributed_metric": dist, + "single_gpu_baseline": baseline, + "tolerance": eff_tolerance, + "abs_delta": float("inf"), + "rel_delta": float("inf"), + "margin": float("-inf"), + "n_baseline_seeds": n_seeds, + "reason": ( + f"distributed {metric_name} is non-finite ({dist}) — diverged " + "distributed run; fast-but-wrong, gate FAILS (§C21)" + ), + } + + abs_delta = abs(dist - baseline) + rel_delta = abs_delta / abs(baseline) if baseline != 0 else float("inf") + margin = eff_tolerance - abs_delta + passed = abs_delta <= eff_tolerance + + if passed: + reason = ( + f"distributed {metric_name}={dist:.6g} matches single-GPU " + f"baseline={baseline:.6g} within tolerance={eff_tolerance:.6g} " + f"(|Δ|={abs_delta:.6g}); MFU may now be trusted (§C21 pass)" + ) + else: + reason = ( + f"distributed {metric_name}={dist:.6g} DIVERGES from single-GPU " + f"baseline={baseline:.6g}: |Δ|={abs_delta:.6g} > tolerance=" + f"{eff_tolerance:.6g} — fast-but-wrong; run is INCONCLUSIVE, do NOT " + "trust its MFU (§C21 fail)" + ) + + return { + "passed": passed, + "verdict": "pass" if passed else "fail", + "metric_name": metric_name, + "distributed_metric": dist, + "single_gpu_baseline": baseline, + "tolerance": eff_tolerance, + "abs_delta": abs_delta, + "rel_delta": rel_delta, + "margin": margin, + "n_baseline_seeds": n_seeds, + "reason": reason, + } + + +def gate_or_raise(*args, **kwargs): + """Convenience wrapper: run the gate and raise CorrectnessGateError on FAIL. + + For callers (the scaling-report driver) that want a hard stop — "do not even + compute MFU if correctness failed". Returns the verdict dict on pass. + """ + result = check_distributed_correctness(*args, **kwargs) + if not result["passed"]: + raise CorrectnessGateError(result["reason"], result) + return result + + +class CorrectnessGateError(RuntimeError): + """Raised by gate_or_raise when the §C21 distributed-correctness gate fails.""" + + def __init__(self, message, result): + super().__init__(message) + self.result = result + + +__all__ = [ + "noise_floor_from_seeds", + "check_distributed_correctness", + "gate_or_raise", + "CorrectnessGateError", +] diff --git a/research/eval_math_acc.py b/research/eval_math_acc.py new file mode 100644 index 0000000..0260d47 --- /dev/null +++ b/research/eval_math_acc.py @@ -0,0 +1,225 @@ +"""Math-accuracy scorer — the `math-acc-v1` capability axis (the decision metric the +rlvr stage needs, per research/rlvr/plan.md §C27). The MISSING piece: eval-harness has +PPL + multiple-choice but NO exact-match / pass@k, and the SFT cohort proved PPL cannot +separate reasoning arms — so held-out math EXACT-MATCH pass@1 + pass@k is the only valid +win metric for SFT/distillation/GRPO reasoning runs. + +DESIGN — model-agnostic & CPU-testable, mirroring research/eval_downstream.py. The scorer +takes a `generate_fn` callable (`(prompt:str, n:int) -> list[str]` of n sampled +completions), NOT a model, so all of extraction / equivalence / pass@k is unit-tested here +with a deterministic stub and the SAME code runs on the real checkpoint under eval-harness. +This module NEVER trains, NEVER loads a model, NEVER allocates CUDA — the CALLER +(eval-harness template) loads the checkpoint behind `safe_cuda.guard(...)` and passes +`generate_fn`. + +The three §C25 `rlvr` battery items this module supplies: + - `pass1_wilson_ci` — accuracy_wilson_ci over all (item×sample) trials. + - `passk_chen2021` — the unbiased HumanEval/Chen-2021 pass@k estimator, mean over items. + - `extractor_pinned` — EXTRACTOR_VERSION is stamped into every result; the extractor + + verifier are frozen here and version-bumped on any change, so a + number is comparable only within one extractor version. +Plus `verifier_honesty_ipt` support: `verifier_false_positive_rate()` runs the verifier on +KNOWN-wrong (question, answer) pairs — it MUST stay near 0, else the reward is gameable. + +HONEST SCALE CAVEAT (carried into every result): at 596M params / ~1.19B training tokens +the SFT'd model very likely sits near pass@k≈0 on any non-trivial math band. A pass@k of 0 +is a REAL, decisive result (RL is null-by-construction — nothing to sharpen), not a bug. +Report the number; do not over-read a near-zero accuracy as a failure of the harness. +""" +from __future__ import annotations +import re +from typing import Callable, Sequence + +from research.eval_metrics import accuracy_wilson_ci + +# Pinned extractor+verifier identity. BUMP on ANY change to _extract/_normalize/is_equiv — +# numbers are only comparable within one version (the `extractor_pinned` §C25 item). +EXTRACTOR_VERSION = "math-acc-v1" + +GenerateFn = Callable[[str, int], Sequence[str]] + +# --------------------------------------------------------------- pass@k (Chen 2021) + +def pass_at_k(n: int, c: int, k: int) -> float: + """Unbiased pass@k estimator (Chen et al. 2021, the HumanEval formula): + the probability that at least one of k samples drawn WITHOUT replacement from n + total samples (of which c are correct) is correct = 1 - C(n-c,k)/C(n,k), computed + in the numerically-stable product form. k=1 ⇒ c/n (plain pass@1).""" + if k <= 0: + raise ValueError("k must be >= 1") + if n <= 0 or k > n: + raise ValueError(f"need 1 <= k <= n; got n={n} k={k}") + if c <= 0: + return 0.0 + if n - c < k: # not enough wrong samples to fill k slots ⇒ guaranteed a hit + return 1.0 + prod = 1.0 + for i in range(n - c + 1, n + 1): + prod *= 1.0 - k / i + return 1.0 - prod + +# --------------------------------------------------------------- answer extraction + +def _last_boxed(text: str) -> str | None: + """Return the content of the LAST \\boxed{...} (balanced braces), or None.""" + idx = text.rfind(r"\boxed") + if idx < 0: + return None + i = idx + len(r"\boxed") + while i < len(text) and text[i] in " \t": + i += 1 + if i >= len(text) or text[i] != "{": + return None + depth, start = 0, i + for j in range(i, len(text)): + if text[j] == "{": + depth += 1 + elif text[j] == "}": + depth -= 1 + if depth == 0: + return text[start + 1:j] + return None + +_ANS_IS = re.compile(r"(?:final answer|answer)\s*(?:is|:|=)\s*\$?([^\n.$]+)", re.IGNORECASE) +_NUM = re.compile(r"-?\d[\d,]*(?:\.\d+)?") + +def extract_answer(text: str) -> str | None: + """Pinned+versioned answer extractor. Priority: \\boxed{} → GSM8K '#### X' → + 'the answer is X' → last number in the text. Returns the raw span (normalized at + compare time), or None if nothing parseable. Deterministic, no model, no network.""" + if not text: + return None + boxed = _last_boxed(text) + if boxed is not None: + return boxed.strip() + if "####" in text: # GSM8K gold/solution convention + tail = text.rsplit("####", 1)[1] + m = _NUM.search(tail) + if m: + return m.group(0).strip() + return tail.strip().splitlines()[0].strip() if tail.strip() else None + m = _ANS_IS.search(text) + if m: + return m.group(1).strip() + nums = _NUM.findall(text) # last-number fallback + return nums[-1].strip() if nums else None + +# --------------------------------------------------------------- normalization + verifier + +_TEXT_CMD = re.compile(r"\\(?:text|mathrm|mbox|mathbf|mathit)\s*\{([^}]*)\}") +_FRAC = re.compile(r"\\d?frac\s*\{([^{}]*)\}\s*\{([^{}]*)\}") + +def _normalize(s: str) -> str: + """Canonicalize a math answer string for a fast string-equality path (the classic + MATH/Minerva cleanup): strip $, \\left/\\right, \\text{}, units, spaces, thousands + commas, trailing punctuation; \\frac{a}{b} → (a)/(b); ^ stays for the sympy path.""" + if s is None: + return "" + s = s.strip() + s = _TEXT_CMD.sub(r"\1", s) + s = s.replace(r"\boxed", "").replace(r"\left", "").replace(r"\right", "") + s = _FRAC.sub(r"(\1)/(\2)", s) # \frac{a}{b} → (a)/(b) BEFORE braces are stripped + for a, b in ((r"\!", ""), (r"\,", ""), (r"\ ", " "), ("$", ""), (r"\%", ""), ("%", ""), + (r"^\circ", ""), (r"\circ", ""), (r"\cdot", "*"), (r"\times", "*"), + ("{", ""), ("}", "")): + s = s.replace(a, b) + s = s.replace("\\", "").replace(" ", "") + s = s.rstrip(".") + if re.fullmatch(r"-?\d{1,3}(?:,\d{3})+(?:\.\d+)?", s): # 1,234 → 1234 (only true separators) + s = s.replace(",", "") + return s.lower() + +def _as_float(s: str): + try: + return float(s) + except (ValueError, TypeError): + return None + +def _sympy_equal(a: str, b: str) -> bool: + """Symbolic equivalence via sympy (1/2 == 0.5, (1)/(2) == 0.5, 2^3 == 8). Fully + guarded: any parse/timeout/exception → False (a verifier must never crash training). + Length-capped to avoid pathological sympify blow-ups.""" + if len(a) > 100 or len(b) > 100: + return False + try: + from sympy import simplify + from sympy.parsing.sympy_parser import parse_expr + expr = f"({a.replace('^', '**')})-({b.replace('^', '**')})" + return simplify(parse_expr(expr, evaluate=True)) == 0 + except Exception: + return False + +def is_equiv(pred: str | None, gold: str | None) -> bool: + """The VERIFIER (verifiable-reward, no neural RM). True iff the extracted prediction + is mathematically equivalent to the gold answer. Order: normalized string equality → + numeric equality (tol 1e-6) → sympy symbolic equality. Conservative: unparseable ⇒ False.""" + if pred is None or gold is None: + return False + np_, ng = _normalize(pred), _normalize(gold) + if np_ == "" or ng == "": + return False + if np_ == ng: + return True + fp, fg = _as_float(np_), _as_float(ng) + if fp is not None and fg is not None: + return abs(fp - fg) <= 1e-6 * max(1.0, abs(fg)) + return _sympy_equal(np_, ng) + +def verifier_false_positive_rate(wrong_pairs: Sequence[tuple[str, str]]) -> float: + """`verifier_honesty_ipt` support: fraction of KNOWN-WRONG (pred, gold) pairs the + verifier wrongly accepts. MUST be ~0 — a nonzero rate means the reward is gameable.""" + if not wrong_pairs: + return 0.0 + fp = sum(1 for pred, gold in wrong_pairs if is_equiv(pred, gold)) + return fp / len(wrong_pairs) + +# --------------------------------------------------------------- scoring harness + +def score_item(generate_fn: GenerateFn, prompt: str, gold: str, n_samples: int) -> dict: + """Generate n_samples completions for one prompt, extract+verify each. Returns + {n, c, correct: [0/1...]} — c = number of verifier-correct samples.""" + comps = list(generate_fn(prompt, n_samples)) + correct = [1 if is_equiv(extract_answer(c), gold) else 0 for c in comps] + return {"n": len(correct), "c": sum(correct), "correct": correct} + +def run_math_acc(generate_fn: GenerateFn, items: Sequence[dict], + n_samples: int = 16, k_list: Sequence[int] = (1, 8, 16)) -> dict: + """Score a held-out math set. `items` = [{"prompt", "gold"}...]. For each item sample + n_samples completions; aggregate pass@1 (Wilson CI over all item×sample trials) and + pass@k (Chen-2021, mean over items). Returns a flat metrics dict stamped with + EXTRACTOR_VERSION — the §C25 rlvr items pass1_wilson_ci / passk_chen2021 / extractor_pinned.""" + if not items: + raise ValueError("no items to score") + k_list = [k for k in k_list if k <= n_samples] + per_item, flat = [], [] + for it in items: + r = score_item(generate_fn, it["prompt"], it["gold"], n_samples) + per_item.append(r) + flat.extend(r["correct"]) + acc, lo, hi = accuracy_wilson_ci(flat) + passk = {k: sum(pass_at_k(r["n"], r["c"], k) for r in per_item) / len(per_item) + for k in k_list} + return { + "extractor_version": EXTRACTOR_VERSION, # → extractor_pinned + "n_items": len(items), + "n_samples": n_samples, + "pass1_wilson_ci": {"acc": acc, "ci_low": lo, "ci_high": hi}, + "passk_chen2021": passk, + "solved_items": sum(1 for r in per_item if r["c"] > 0), + } + +def _self_test() -> None: + """Deterministic CPU smoke — no model, no network. Run: python3 -m research.eval_math_acc""" + assert extract_answer(r"so the answer is \boxed{72}.") == "72" + assert extract_answer("...\n#### 18") == "18" + assert is_equiv(r"\boxed{1/2}", "0.5") and is_equiv("72", "72") and not is_equiv("72", "73") + assert abs(pass_at_k(4, 1, 1) - 0.25) < 1e-9 and pass_at_k(4, 0, 2) == 0.0 and pass_at_k(4, 4, 2) == 1.0 + # a stub policy that always emits the gold answer ⇒ pass@1 == 1 + items = [{"prompt": "2+2?", "gold": "4"}, {"prompt": "3+3?", "gold": "6"}] + gen = lambda p, n: [f"the answer is \\boxed{{{ '4' if '2+2' in p else '6' }}}"] * n + out = run_math_acc(gen, items, n_samples=4, k_list=(1, 4)) + assert out["pass1_wilson_ci"]["acc"] == 1.0 and out["passk_chen2021"][4] == 1.0 + print("SELF-TEST OK", out["extractor_version"], out["passk_chen2021"]) + +if __name__ == "__main__": + _self_test() diff --git a/research/harness_search/tasks/codeharness/hard_benchmark.py b/research/harness_search/tasks/codeharness/hard_benchmark.py new file mode 100644 index 0000000..567da6c --- /dev/null +++ b/research/harness_search/tasks/codeharness/hard_benchmark.py @@ -0,0 +1,221 @@ +"""HARD coding tasks for the codeharness headroom check. Unlike benchmark.py (10 +trivial tasks a 9B aces under any harness -> zero headroom), each task here has a +known EDGE-CASE TRAP a capable model often misses on the first attempt (overflow +clamping, truncate-toward-zero division, the abba sliding-window reset, spiral +boundary management, …). That first-attempt failure is what a TRACE-USING +self-repair harness can fix and a scalar-only harness cannot — so these tasks are +where "do traces beat scalar feedback" is actually testable. + +Each task carries: + prompt – function stub shown to the model (the only thing the harness sees) + reference – a correct solution (runner self-test ONLY; never shown) + public_tests – a few asserts the SELF-REPAIR harness may run + read failures from + tests – the HIDDEN grading suite (superset, more edge cases); never shown + +public_tests deliberately INCLUDE an edge case or two, so repairing against them can +actually raise the hidden pass rate. Pure data; correctness is verified on CPU by +verify_hard_benchmark() before any GPU run. +""" + +TASKS = [ + { + "id": "my_atoi", "entry_point": "my_atoi", + "prompt": ('def my_atoi(s):\n' + ' """Convert string s to a 32-bit signed integer (C atoi / LeetCode 8):\n' + ' skip leading spaces, an optional single +/- sign, read digits until a\n' + ' non-digit, ignore the rest. Clamp the result to [-2**31, 2**31-1].\n' + ' Return 0 if no digits are read."""'), + "reference": ( + "def my_atoi(s):\n" + " i, n = 0, len(s)\n" + " while i < n and s[i] == ' ':\n" + " i += 1\n" + " sign = 1\n" + " if i < n and s[i] in '+-':\n" + " sign = -1 if s[i] == '-' else 1\n" + " i += 1\n" + " num = 0\n" + " while i < n and s[i].isdigit():\n" + " num = num * 10 + int(s[i])\n" + " i += 1\n" + " num *= sign\n" + " return max(-2**31, min(2**31 - 1, num))\n"), + "public_tests": ( + 'assert my_atoi("42") == 42\n' + 'assert my_atoi(" -42") == -42\n' + 'assert my_atoi("4193 with words") == 4193\n' + 'assert my_atoi("words and 987") == 0\n' + 'assert my_atoi("-91283472332") == -2147483648\n'), + "tests": ( + 'assert my_atoi("42") == 42\n' + 'assert my_atoi(" -42") == -42\n' + 'assert my_atoi("4193 with words") == 4193\n' + 'assert my_atoi("words and 987") == 0\n' + 'assert my_atoi("-91283472332") == -2147483648\n' + 'assert my_atoi("91283472332") == 2147483647\n' + 'assert my_atoi("+-12") == 0\n' + 'assert my_atoi(" +0 123") == 0\n' + 'assert my_atoi("") == 0\n' + 'assert my_atoi("2147483648") == 2147483647\n'), + }, + { + "id": "eval_rpn", "entry_point": "eval_rpn", + "prompt": ('def eval_rpn(tokens):\n' + ' """Evaluate a Reverse Polish Notation expression (list of string\n' + ' tokens). Operators: + - * /. Division TRUNCATES TOWARD ZERO\n' + ' (so -7 / 2 == -3, not -4). Return the integer result."""'), + "reference": ( + "def eval_rpn(tokens):\n" + " st = []\n" + " for t in tokens:\n" + " if t in ('+', '-', '*', '/'):\n" + " b = st.pop(); a = st.pop()\n" + " if t == '+': st.append(a + b)\n" + " elif t == '-': st.append(a - b)\n" + " elif t == '*': st.append(a * b)\n" + " else:\n" + " q = abs(a) // abs(b)\n" + " st.append(q if (a < 0) == (b < 0) else -q)\n" + " else:\n" + " st.append(int(t))\n" + " return st[-1]\n"), + "public_tests": ( + 'assert eval_rpn(["2","1","+","3","*"]) == 9\n' + 'assert eval_rpn(["4","13","5","/","+"]) == 6\n' + 'assert eval_rpn(["-7","2","/"]) == -3\n'), + "tests": ( + 'assert eval_rpn(["2","1","+","3","*"]) == 9\n' + 'assert eval_rpn(["4","13","5","/","+"]) == 6\n' + 'assert eval_rpn(["-7","2","/"]) == -3\n' + 'assert eval_rpn(["3","-4","/"]) == 0\n' + 'assert eval_rpn(["-22","5","/"]) == -4\n' + 'assert eval_rpn(["10","6","9","3","+","-11","*","/","*","17","+","5","+"]) == 22\n'), + }, + { + "id": "decode_string", "entry_point": "decode_string", + "prompt": ('def decode_string(s):\n' + ' """Decode a string with the rule k[encoded] = encoded repeated k\n' + ' times; brackets may NEST and k may be multi-digit. Examples:\n' + " '3[a2[c]]' -> 'accaccacc', '2[abc]3[cd]ef' -> 'abcabccdcdcdef'.\n" + ' Input is always valid."""'), + "reference": ( + "def decode_string(s):\n" + " cur, num, st = '', 0, []\n" + " for ch in s:\n" + " if ch.isdigit():\n" + " num = num * 10 + int(ch)\n" + " elif ch == '[':\n" + " st.append((cur, num)); cur, num = '', 0\n" + " elif ch == ']':\n" + " prev, k = st.pop(); cur = prev + cur * k\n" + " else:\n" + " cur += ch\n" + " return cur\n"), + "public_tests": ( + 'assert decode_string("3[a]2[bc]") == "aaabcbc"\n' + 'assert decode_string("3[a2[c]]") == "accaccacc"\n' + 'assert decode_string("10[a]") == "aaaaaaaaaa"\n'), + "tests": ( + 'assert decode_string("3[a]2[bc]") == "aaabcbc"\n' + 'assert decode_string("3[a2[c]]") == "accaccacc"\n' + 'assert decode_string("2[abc]3[cd]ef") == "abcabccdcdcdef"\n' + 'assert decode_string("abc") == "abc"\n' + 'assert decode_string("10[a]") == "aaaaaaaaaa"\n' + 'assert decode_string("2[2[2[a]]]") == "aaaaaaaa"\n'), + }, + { + "id": "merge_intervals", "entry_point": "merge_intervals", + "prompt": ('def merge_intervals(intervals):\n' + ' """Merge all overlapping intervals (a list of [start, end]) and\n' + ' return the merged list sorted by start. Touching intervals such as\n' + ' [1,4] and [4,5] merge into [1,5]. Input may be unsorted."""'), + "reference": ( + "def merge_intervals(intervals):\n" + " if not intervals: return []\n" + " xs = sorted(intervals, key=lambda x: x[0])\n" + " out = [list(xs[0])]\n" + " for s, e in xs[1:]:\n" + " if s <= out[-1][1]:\n" + " out[-1][1] = max(out[-1][1], e)\n" + " else:\n" + " out.append([s, e])\n" + " return out\n"), + "public_tests": ( + 'assert merge_intervals([[1,3],[2,6],[8,10],[15,18]]) == [[1,6],[8,10],[15,18]]\n' + 'assert merge_intervals([[1,4],[4,5]]) == [[1,5]]\n' + 'assert merge_intervals([[1,4],[0,4]]) == [[0,4]]\n'), + "tests": ( + 'assert merge_intervals([[1,3],[2,6],[8,10],[15,18]]) == [[1,6],[8,10],[15,18]]\n' + 'assert merge_intervals([[1,4],[4,5]]) == [[1,5]]\n' + 'assert merge_intervals([[1,4],[0,4]]) == [[0,4]]\n' + 'assert merge_intervals([]) == []\n' + 'assert merge_intervals([[1,4],[2,3]]) == [[1,4]]\n' + 'assert merge_intervals([[1,4],[5,6]]) == [[1,4],[5,6]]\n'), + }, + { + "id": "spiral_order", "entry_point": "spiral_order", + "prompt": ('def spiral_order(matrix):\n' + ' """Return all elements of the m x n matrix in clockwise spiral order\n' + ' starting top-left. e.g. [[1,2,3],[4,5,6],[7,8,9]] ->\n' + ' [1,2,3,6,9,8,7,4,5]. Handle non-square and empty matrices."""'), + "reference": ( + "def spiral_order(matrix):\n" + " if not matrix or not matrix[0]: return []\n" + " res = []\n" + " top, bot = 0, len(matrix) - 1\n" + " left, right = 0, len(matrix[0]) - 1\n" + " while top <= bot and left <= right:\n" + " for c in range(left, right + 1): res.append(matrix[top][c])\n" + " top += 1\n" + " for r in range(top, bot + 1): res.append(matrix[r][right])\n" + " right -= 1\n" + " if top <= bot:\n" + " for c in range(right, left - 1, -1): res.append(matrix[bot][c])\n" + " bot -= 1\n" + " if left <= right:\n" + " for r in range(bot, top - 1, -1): res.append(matrix[r][left])\n" + " left += 1\n" + " return res\n"), + "public_tests": ( + 'assert spiral_order([[1,2,3],[4,5,6],[7,8,9]]) == [1,2,3,6,9,8,7,4,5]\n' + 'assert spiral_order([[1,2,3,4],[5,6,7,8],[9,10,11,12]]) == [1,2,3,4,8,12,11,10,9,5,6,7]\n'), + "tests": ( + 'assert spiral_order([[1,2,3],[4,5,6],[7,8,9]]) == [1,2,3,6,9,8,7,4,5]\n' + 'assert spiral_order([[1,2,3,4],[5,6,7,8],[9,10,11,12]]) == [1,2,3,4,8,12,11,10,9,5,6,7]\n' + 'assert spiral_order([[1,2,3]]) == [1,2,3]\n' + 'assert spiral_order([[1],[2],[3]]) == [1,2,3]\n' + 'assert spiral_order([[1]]) == [1]\n' + 'assert spiral_order([]) == []\n'), + }, + { + "id": "longest_unique", "entry_point": "length_of_longest_substring", + "prompt": ('def length_of_longest_substring(s):\n' + ' """Return the length of the longest substring of s containing no\n' + " repeating characters. 'abcabcbb' -> 3, 'bbbbb' -> 1, 'pwwkew' -> 3.\n" + ' The sliding window start must never move backwards."""'), + "reference": ( + "def length_of_longest_substring(s):\n" + " seen = {}\n" + " start = best = 0\n" + " for i, ch in enumerate(s):\n" + " if ch in seen and seen[ch] >= start:\n" + " start = seen[ch] + 1\n" + " seen[ch] = i\n" + " best = max(best, i - start + 1)\n" + " return best\n"), + "public_tests": ( + 'assert length_of_longest_substring("abcabcbb") == 3\n' + 'assert length_of_longest_substring("pwwkew") == 3\n' + 'assert length_of_longest_substring("abba") == 2\n'), + "tests": ( + 'assert length_of_longest_substring("abcabcbb") == 3\n' + 'assert length_of_longest_substring("bbbbb") == 1\n' + 'assert length_of_longest_substring("pwwkew") == 3\n' + 'assert length_of_longest_substring("") == 0\n' + 'assert length_of_longest_substring(" ") == 1\n' + 'assert length_of_longest_substring("abba") == 2\n' + 'assert length_of_longest_substring("tmmzuxt") == 5\n'), + }, +] + +BY_ID = {t["id"]: t for t in TASKS} diff --git a/research/harness_search/tasks/codeharness/probe_headroom.py b/research/harness_search/tasks/codeharness/probe_headroom.py new file mode 100644 index 0000000..1ef1069 --- /dev/null +++ b/research/harness_search/tasks/codeharness/probe_headroom.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""HEADROOM PROBE for the codeharness target — the decisive cheap experiment before +committing GPU to a full harness search. + +Question: does HARNESS quality move the held-out pass rate at all on this benchmark +with a REAL frozen model? If the hand-designed baseline and a deliberately-naive +harness both score ~1.0 (or both ~0.0), there is no signal to search — codeharness +is the packing trap again, and the honest move is harder tasks / a weaker model. If +the naive harness scores well BELOW baseline, extraction/prompt headroom is real and +a search (+ traces) is worth running. + +Frozen model = Qwen3.5-9B (HF cache), greedy/deterministic so the whole probe is +reproducible. The runner is the FIXED (reward-hack-closed) oracle. GB10-only, no +rented compute; safe_cuda guards the unified-memory pool. +""" +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[4] +sys.path.insert(0, str(REPO)) # safe_cuda at repo root +sys.path.insert(0, str(Path(__file__).resolve().parent)) # benchmark, runner, baseline_harness + +import safe_cuda # noqa: E402 +safe_cuda.guard(0.85) + +import torch # noqa: E402 +from transformers import AutoTokenizer # noqa: E402 +import benchmark as bm # noqa: E402 +import runner as rn # noqa: E402 +import baseline_harness as bh # noqa: E402 + +MODEL = "Qwen/Qwen3.5-9B" +MAX_NEW = 256 + + +def _load(): + tok = AutoTokenizer.from_pretrained(MODEL) + model = None + errs = [] + from transformers import AutoModelForCausalLM + for loader_name, loader in [("AutoModelForCausalLM", AutoModelForCausalLM)]: + try: + model = loader.from_pretrained(MODEL, torch_dtype=torch.bfloat16, device_map="cuda") + print(f"loaded via {loader_name}", flush=True) + break + except Exception as e: + errs.append(f"{loader_name}: {type(e).__name__}: {e}") + if model is None: + # VLM fallback: load the text-to-text / image-text-to-text head, use text path only + for loader_name in ("AutoModelForImageTextToText", "AutoModelForVision2Seq"): + try: + import transformers + loader = getattr(transformers, loader_name) + model = loader.from_pretrained(MODEL, torch_dtype=torch.bfloat16, device_map="cuda") + print(f"loaded via {loader_name}", flush=True) + break + except Exception as e: + errs.append(f"{loader_name}: {type(e).__name__}: {e}") + if model is None: + raise RuntimeError("could not load model:\n" + "\n".join(errs)) + model.eval() + return tok, model + + +def make_agent_fn(tok, model): + @torch.no_grad() + def agent_fn(prompt): + # the harness already built the full instruction; wrap as a single user turn. + # enable_thinking=False — else Qwen3.5's reasoning block eats the whole token + # budget before any code is emitted (the diagnostic showed this = a false 0/10). + msgs = [{"role": "user", "content": prompt}] + try: + text = tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False, + enable_thinking=False) + except TypeError: + text = tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) + enc = tok(text, return_tensors="pt").to(model.device) + out = model.generate(**enc, max_new_tokens=MAX_NEW, do_sample=False, + pad_token_id=tok.eos_token_id) + gen = out[0][enc["input_ids"].shape[1]:] + return tok.decode(gen, skip_special_tokens=True) + return agent_fn + + +def main(): + t0 = time.time() + tok, model = _load() + agent_fn = make_agent_fn(tok, model) + print(f"model ready in {time.time()-t0:.0f}s\n", flush=True) + + baseline_solve = bh.make_solve(agent_fn, bh.INCUMBENT) # build_prompt + strip_fences + # deliberately-naive harness: same prompt, NO fence stripping (raw model text as code) + def naive_solve(task): + return agent_fn(bh.build_prompt(task, bh.INCUMBENT)) + + rows = [] + for t in bm.TASKS: + raw = agent_fn(bh.build_prompt(t, bh.INCUMBENT)) + base_code = bh.strip_fences(raw) + base_pass = rn.run_solution(base_code, t) + naive_pass = rn.run_solution(raw, t) + fenced = raw.lstrip().startswith("```") + rows.append((t["id"], base_pass, naive_pass, fenced, raw)) + print(f" {t['id']:14} baseline={'P' if base_pass else 'F'} " + f"naive={'P' if naive_pass else 'F'} fenced={fenced}", flush=True) + + held = [r for r in rows if r[0] in {x['id'] for x in bm.HELDOUT_TASKS}] + def rate(rs, i): return sum(1 for r in rs if r[i]) / len(rs) + print("\n--- HEAD-ROOM ---", flush=True) + print(f"ALL (n={len(rows)}): baseline={rate(rows,1):.2f} naive={rate(rows,2):.2f}", flush=True) + print(f"HELD (n={len(held)}): baseline={rate(held,1):.2f} naive={rate(held,2):.2f}", flush=True) + print(f"fenced outputs: {sum(1 for r in rows if r[3])}/{len(rows)}", flush=True) + spread = rate(rows, 1) - rate(rows, 2) + print(f"\nHEADROOM (baseline - naive) = {spread:+.2f} -> " + + ("REAL extraction headroom; a search can find it." if spread >= 0.2 + else "thin; baseline≈naive — likely the easy-task trap (need harder tasks / weaker model)."), + flush=True) + # show one fenced raw output so the failure mode is concrete + for rid, bp, npass, fenced, raw in rows: + if fenced: + print(f"\nexample raw output [{rid}] (first 240 chars):\n{raw[:240]!r}", flush=True) + break + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/harness_search/tasks/codeharness/probe_repair.py b/research/harness_search/tasks/codeharness/probe_repair.py new file mode 100644 index 0000000..bae46b3 --- /dev/null +++ b/research/harness_search/tasks/codeharness/probe_repair.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""HEADROOM CHECK on the HARD tasks: does a TRACE-USING self-repair harness beat a +no-repair (scalar-blind) one with the real frozen model? If yes, execution traces +carry signal here -> a full Meta-Harness search (trace-using vs scalar proposer) is +justified. If no, codeharness on GB10 with this model is a dead end and we stop. + +Frozen model = Qwen/Qwen3.5-9B, greedy, enable_thinking=False (else the reasoning +block eats the budget). Grading is the FIXED hack-proof runner on HIDDEN tests. +GB10-only, safe_cuda-guarded. +""" +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[3] +sys.path.insert(0, str(REPO)) +sys.path.insert(0, str(HERE)) + +import safe_cuda # noqa: E402 +safe_cuda.guard(0.85) + +import torch # noqa: E402 +from transformers import AutoTokenizer, AutoModelForCausalLM # noqa: E402 +import hard_benchmark as hb # noqa: E402 +import runner as rn # noqa: E402 +import repair_harness as rh # noqa: E402 + +MODEL = "Qwen/Qwen3.5-9B" +MAX_NEW = 512 + + +def main(): + t0 = time.time() + tok = AutoTokenizer.from_pretrained(MODEL) + model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16, + device_map="cuda").eval() + print(f"model ready in {time.time()-t0:.0f}s\n", flush=True) + + @torch.no_grad() + def agent_fn(prompt): + msgs = [{"role": "user", "content": prompt}] + try: + text = tok.apply_chat_template(msgs, add_generation_prompt=True, + tokenize=False, enable_thinking=False) + except TypeError: + text = tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) + enc = tok(text, return_tensors="pt").to(model.device) + out = model.generate(**enc, max_new_tokens=MAX_NEW, do_sample=False, + pad_token_id=tok.eos_token_id) + return tok.decode(out[0][enc["input_ids"].shape[1]:], skip_special_tokens=True) + + no_repair = rh.make_no_repair_solve(agent_fn) + self_repair = rh.make_self_repair_solve(agent_fn, max_repairs=2) + + n0 = n1 = 0 + print(f"{'task':16} {'no_repair':>10} {'self_repair':>12}", flush=True) + for t in hb.TASKS: + c0 = no_repair(t) + p0 = rn.run_solution(c0, t) # HIDDEN grade, hack-proof runner + c1 = self_repair(t) + p1 = rn.run_solution(c1, t) + n0 += p0; n1 += p1 + print(f" {t['id']:14} {'PASS' if p0 else 'fail':>10} {'PASS' if p1 else 'fail':>12}", flush=True) + + N = len(hb.TASKS) + print(f"\n--- HARD-TASK HEADROOM (n={N}) ---", flush=True) + print(f"no_repair hidden pass rate = {n0/N:.2f} ({n0}/{N})", flush=True) + print(f"self_repair hidden pass rate = {n1/N:.2f} ({n1}/{N})", flush=True) + spread = (n1 - n0) / N + print(f"\nTRACE HEADROOM (self_repair - no_repair) = {spread:+.2f} -> " + + ("REAL: traces lift the pass rate -> a full search is justified." + if (n1 - n0) >= 1 else + "none: traces don't move it here. Either the model already maxes the " + "hard tasks (raise difficulty) or first attempts are unfixable from " + "public-test traces (dead end on this model)."), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/harness_search/tasks/codeharness/repair_harness.py b/research/harness_search/tasks/codeharness/repair_harness.py new file mode 100644 index 0000000..f08f4c2 --- /dev/null +++ b/research/harness_search/tasks/codeharness/repair_harness.py @@ -0,0 +1,71 @@ +"""Two harnesses for the headroom check, differing ONLY in whether they use +execution TRACES: + + no_repair – one shot: build prompt -> agent_fn -> strip fences. (scalar-blind) + self_repair – build prompt -> agent_fn -> RUN the public tests -> on failure feed + the actual traceback back to the model and regenerate (<= max_repairs + rounds). It consumes the execution trace; the no-repair harness can't. + +Both return CODE that is then graded by the FIXED runner.run_solution on the HIDDEN +tests (which neither harness ever sees). If self_repair's hidden pass rate beats +no_repair's, traces carry signal on this benchmark -> real headroom for a search. +The repair loop runs only the task's PUBLIC tests (not secret), so reading their +failures is fair; the grade stays hack-proof because hidden grading uses the +sentinel runner, not this loop. +""" +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import baseline_harness as bh # noqa: E402 (strip_fences + build_prompt + INCUMBENT) + +TIMEOUT_S = 10 +MAX_REPAIRS = 2 + + +def run_public(code, task): + """Run `code` against the task's PUBLIC tests; return (passed, trace_tail). The + trace_tail is the last chunk of stderr (the failing assert + traceback) that the + self-repair harness feeds back to the model.""" + program = (code or "") + "\n\n" + task["public_tests"] + "\n" + with tempfile.TemporaryDirectory() as d: + f = Path(d) / "c.py" + f.write_text(program) + try: + p = subprocess.run([sys.executable, str(f)], cwd=d, timeout=TIMEOUT_S, + capture_output=True, text=True) + except subprocess.TimeoutExpired: + return False, "timeout: the code did not finish on the public tests" + except Exception as e: # pragma: no cover + return False, f"spawn error: {type(e).__name__}: {e}" + if p.returncode == 0: + return True, "" + return False, ((p.stderr or "").strip()[-800:] or f"exit {p.returncode}") + + +def _repair_prompt(task, code, trace): + return (f"Your Python solution failed a test.\n\n" + f"Problem:\n{task['prompt']}\n\n" + f"Your code:\n{code}\n\n" + f"Running the public tests produced this error:\n{trace}\n\n" + f"Return ONLY the corrected function definition — no prose, no markdown fences.") + + +def make_no_repair_solve(agent_fn): + def solve(task): + return bh.strip_fences(agent_fn(bh.build_prompt(task, bh.INCUMBENT))) + return solve + + +def make_self_repair_solve(agent_fn, max_repairs=MAX_REPAIRS): + def solve(task): + code = bh.strip_fences(agent_fn(bh.build_prompt(task, bh.INCUMBENT))) + for _ in range(max_repairs): + passed, trace = run_public(code, task) + if passed: + break + code = bh.strip_fences(agent_fn(_repair_prompt(task, code, trace))) + return code + return solve diff --git a/research/interp/cka_probe.py b/research/interp/cka_probe.py new file mode 100644 index 0000000..9af88fe --- /dev/null +++ b/research/interp/cka_probe.py @@ -0,0 +1,227 @@ +"""Representational-convergence probe (linear CKA) for the NorMuon-vs-AdamW ladder. + +Executes research/interp/prereg_2026-07-19_repconvergence.md VERBATIM: does NorMuon reach +AdamW's 420M residual-stream representation earlier? Pre-registered H1 / null + decision rule +are fixed there; this script only measures and applies §4. + +SAFETY (§C1): imports safe_cuda before torch; drives the TRUNK ONLY (`model.model(...)`, never +the lm_head) so the 151,936-vocab logits are never materialized (the documented box-crash +vector). Residual activations are fp16, capped, CPU-cached; one 596M model on GPU at a time. + +Usage: + python3 research/interp/cka_probe.py --self-test # CPU-only: CKA math + decision-rule sanity + python3 research/interp/cka_probe.py # full pre-registered probe (needs GPU + preflight) +""" +from __future__ import annotations +import argparse +import json +import pathlib +import sys + +ROOT = pathlib.Path("/home/yashb98/Downloads/BuildFromScratch") +sys.path.insert(0, str(ROOT)) # safe_cuda +import safe_cuda # noqa: E402 + +RESULTS = ROOT / "Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results" +OUT = ROOT / "research/interp" +LAYERS = [6, 13, 20, 27] # pre-reg §2 (of 28) +N_SEQ, SEQ = 64, 512 # pre-reg §2: 32,768 token positions +BOOT = 1000 # pre-reg §3 bootstrap resamples +DEVICE = "cuda" + +# Pre-registered checkpoint arms (seed 0 primary; extra seeds for the noise ceiling). +CKPT = { + "adamw_42M": "checkpoint_adamw_seed0.pt", + "normuon_42M": "checkpoint_normuon_seed0.pt", + "adamw_168M": "checkpoint_persist_168M_adamw_s0.pt", + "normuon_168M": "checkpoint_persist_168M_normuon_s0.pt", + "adamw_420M": "checkpoint_persist_420M_adamw_s0.pt", + "normuon_420M": "checkpoint_persist_420M_normuon_s0.pt", + "adamw_420M_s1": "checkpoint_persist_420M_adamw_s1.pt", # ceiling + "normuon_420M_s1": "checkpoint_persist_420M_normuon_s1.pt", # ceiling +} + + +def linear_cka(X, Y): + """Feature-space linear CKA: ||Xc^T Yc||_F^2 / (||Xc^T Xc||_F ||Yc^T Yc||_F). + X,Y are [N,d], row-aligned (same token positions). fp16/bf16 are upcast to fp32; fp32/fp64 + inputs keep their dtype (self-test passes fp64 for exactness, the probe passes fp32 on GPU).""" + import torch + if X.dtype in (torch.float16, torch.bfloat16): + X = X.float() + if Y.dtype in (torch.float16, torch.bfloat16): + Y = Y.float() + X = X - X.mean(0, keepdim=True) + Y = Y - Y.mean(0, keepdim=True) + xty = X.t() @ Y + denom = (X.t() @ X).norm() * (Y.t() @ Y).norm() + return float((xty.norm() ** 2) / denom) if denom > 0 else 0.0 + + +def bootstrap_delta_ci(acts, key_nb, key_ab, key_target, *, seed=0, B=BOOT, alpha=0.05): + """Paired bootstrap CI on Δ = CKA(NorMuon@B, AdamW@420M) − CKA(AdamW@B, AdamW@420M), + resampling token positions (rows) identically across the three aligned matrices.""" + import torch + g = torch.Generator().manual_seed(seed) + A_nb, A_ab, A_t = acts[key_nb], acts[key_ab], acts[key_target] + n = A_t.shape[0] + point = linear_cka(A_nb, A_t) - linear_cka(A_ab, A_t) + deltas = [] + for _ in range(B): + idx = torch.randint(0, n, (n,), generator=g).to(A_t.device) + deltas.append(linear_cka(A_nb[idx], A_t[idx]) - linear_cka(A_ab[idx], A_t[idx])) + deltas.sort() + lo = deltas[int(alpha / 2 * B)] + hi = deltas[int((1 - alpha / 2) * B)] + return point, lo, hi + + +# ─────────────────────────── GPU path ─────────────────────────── +def _load_model(name): + import torch + sys.path.insert(0, str(ROOT / "Qwen3-0.6B")) + from model import Qwen3Config, Qwen3ForCausalLM + cfg = Qwen3Config() + model = Qwen3ForCausalLM(cfg) + if name != "RANDOM": + sd = torch.load(RESULTS / CKPT[name], map_location="cpu", weights_only=False)["model"] + model.load_state_dict(sd, strict=True) + return model.to(DEVICE).eval() + + +def _extract(name, seqs, batch=4): + """Residuals at each LAYER for one checkpoint. Trunk-only forward (no lm_head).""" + import torch + model = _load_model(name) + buf = {L: [] for L in LAYERS} + + def mk(L): + def hook(_m, _i, out): + # fp32, NOT fp16: NorMuon residual-stream magnitudes reach ~1e6 (>fp16 max 65504), + # which would overflow to inf and silently collapse CKA to 0. Verified 2026-07-19. + h = (out[0] if isinstance(out, (tuple, list)) else out).detach().float() + buf[L].append(h.reshape(-1, h.shape[-1]).cpu()) + return hook + + handles = [model.model.layers[L].register_forward_hook(mk(L)) for L in LAYERS] + try: + with torch.no_grad(): + for b in range(0, seqs.shape[0], batch): + model.model(input_ids=seqs[b:b + batch].to(DEVICE)) # TRUNK ONLY — no logits + finally: + for h in handles: + h.remove() + del model + torch.cuda.empty_cache() + return {L: torch.cat(buf[L], 0) for L in LAYERS} # [N,d] fp32 CPU + + +def run_probe(): + safe_cuda.guard(0.85) + import torch + from transformers import AutoTokenizer + from datasets import load_dataset + + tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base") + # Load ONLY wikitext-2 val — identical corpus + tokenization to text-lm-v2's wikitext2_val + # (score_cohort.load_corpora lines 54/65) — without its code corpus, which we do not use. + wt = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation") + wt_text = "\n\n".join(t for t in wt["text"] if t.strip()) + ids = tok(wt_text, return_tensors="pt").input_ids[0] + assert ids.numel() >= N_SEQ * SEQ, f"wikitext2_val has {ids.numel()} toks < {N_SEQ*SEQ}" + seqs = ids[: N_SEQ * SEQ].reshape(N_SEQ, SEQ) # [64,512], fixed deterministic slice + + # acts[name][L] = [N,d] residuals. Process one checkpoint at a time (memory-safe). + names = list(CKPT) + ["RANDOM"] + acts = {} + for nm in names: + acts[nm] = _extract(nm, seqs) + # INTEGRITY GATE (anti-laundering): refuse to score non-finite activations. A silent + # inf/nan (e.g. an fp16 overflow) collapses CKA to 0 and fakes a null — the exact class + # of numerical confound this repo exists to catch. + for L in LAYERS: + mx = float(acts[nm][L].abs().max()) + assert torch.isfinite(acts[nm][L]).all(), \ + f"non-finite activations in {nm} L{L} (max|act|={mx:.1f}) — numerical bug, refusing to score" + print(f" extracted {nm}: layers {LAYERS}, N={acts[nm][LAYERS[0]].shape[0]}, " + f"max|act|={max(float(acts[nm][L].abs().max()) for L in LAYERS):.0f}", flush=True) + + per_layer = {} + for L in LAYERS: + # Move this layer's matrices to GPU once (fp32→double in linear_cka); the 1000-iter + # bootstrap on [32768,1024] is PFLOP-scale and impractical on CPU. + a = {nm: acts[nm][L].to(DEVICE).float() for nm in names} + ceiling_adamw = linear_cka(a["adamw_420M"], a["adamw_420M_s1"]) + ceiling_normuon = linear_cka(a["normuon_420M"], a["normuon_420M_s1"]) + floor = linear_cka(a["RANDOM"], a["adamw_420M"]) + # INTEGRITY GATE: two seeds of the SAME config must be highly similar; a near-0 ceiling + # means a degenerate/numerical failure, not a real measurement — refuse to emit a verdict. + assert ceiling_adamw > 0.3 and ceiling_normuon > 0.3, \ + f"degenerate same-config ceiling at L{L} (adamw={ceiling_adamw:.3f} normuon={ceiling_normuon:.3f}) — numerical bug" + band = 1.0 - ceiling_adamw # pre-reg §4 across-seed band (AdamW@420M) + budgets = {} + for B in ("42M", "168M"): + pt, lo, hi = bootstrap_delta_ci(a, f"normuon_{B}", f"adamw_{B}", "adamw_420M") + cka_n = linear_cka(a[f"normuon_{B}"], a["adamw_420M"]) + cka_a = linear_cka(a[f"adamw_{B}"], a["adamw_420M"]) + confirmed = (lo > 0) and (pt > band) # pre-reg §4 decision rule + budgets[B] = {"cka_normuon_vs_adamw420": cka_n, "cka_adamw_vs_adamw420": cka_a, + "delta": pt, "ci95": [lo, hi], "exceeds_band": pt > band, + "ci_disjoint_pos": lo > 0, "H1_confirmed": confirmed} + per_layer[L] = {"ceiling_adamw_s0s1": ceiling_adamw, "ceiling_normuon_s0s1": ceiling_normuon, + "floor_random_vs_adamw420": floor, "across_seed_band": band, "budgets": budgets} + del a + torch.cuda.empty_cache() + + # Headline (pre-reg §4): H1 needs ≥2 of 4 layers confirmed for the SAME budget. + headline = "null" + for B in ("42M", "168M"): + nconf = sum(per_layer[L]["budgets"][B]["H1_confirmed"] for L in LAYERS) + if nconf >= 2: + headline = f"H1-confirmed@{B} ({nconf}/4 layers)" + verdict = { + "prereg": "research/interp/prereg_2026-07-19_repconvergence.md", + "lifecycle_stage": "interpretability", "metric": "linear_cka", "layers": LAYERS, + "eval_batch": {"corpus": "wikitext2_val", "n_seq": N_SEQ, "seq": SEQ, "tokens": N_SEQ * SEQ}, + "headline": headline, "per_layer": per_layer, + "note": ("A null (headline='null') is the pre-registered PASSING deliverable at this scale; " + "CKA is a similarity summary, not a causal proof."), + } + (OUT / "cka_verdict.json").write_text(json.dumps(verdict, indent=2)) + print(json.dumps({"headline": headline, + "per_layer_ceiling": {L: round(per_layer[L]["ceiling_adamw_s0s1"], 4) for L in LAYERS}, + "per_layer_floor": {L: round(per_layer[L]["floor_random_vs_adamw420"], 4) for L in LAYERS}}, + indent=2)) + return verdict + + +# ─────────────────────────── CPU self-test ─────────────────────────── +def _self_test(): + import torch + torch.manual_seed(0) + X = torch.randn(500, 64).double() # fp64 → exact-tolerance asserts below + assert abs(linear_cka(X, X) - 1.0) < 1e-9, "CKA(X,X) must be 1" + assert abs(linear_cka(X, X * 3.0 + 5.0) - 1.0) < 1e-6, "CKA is invariant to isotropic scale+shift" + Y = torch.randn(500, 64).double() + assert linear_cka(X, Y) < 0.15, f"CKA of independent gaussians should be low, got {linear_cka(X,Y)}" + R = torch.linalg.qr(torch.randn(64, 64).double())[0] # orthogonal rotation + assert abs(linear_cka(X, X @ R) - 1.0) < 1e-6, "linear CKA invariant to orthogonal transform" + # decision-rule plumbing: a clear positive delta with a tight CI confirms; a straddling CI nulls + acts = {"n": X, "a": Y, "t": X.clone()} # CKA(n,t)=1 >> CKA(a,t)≈0 → big positive Δ + pt, lo, hi = bootstrap_delta_ci(acts, "n", "a", "t", B=200) + assert lo > 0 and pt > 0.5, f"clear case must give disjoint-positive Δ, got {pt} [{lo},{hi}]" + acts2 = {"n": Y, "a": Y.clone(), "t": X} # both ≈0 vs t → Δ≈0, CI straddles + pt2, lo2, hi2 = bootstrap_delta_ci(acts2, "n", "a", "t", B=200) + assert lo2 <= 0 <= hi2, f"null case CI must straddle 0, got {pt2} [{lo2},{hi2}]" + print("cka_probe self-test PASS — CKA identity/scale/rotation invariance hold; " + f"decision rule: clear Δ={pt:.3f}[{lo:.3f},{hi:.3f}] confirms, null Δ={pt2:.3f}[{lo2:.3f},{hi2:.3f}] straddles") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args() + if a.self_test: + _self_test() + else: + run_probe() diff --git a/research/kernel/cce_linear_ce.py b/research/kernel/cce_linear_ce.py new file mode 100644 index 0000000..a473d86 --- /dev/null +++ b/research/kernel/cce_linear_ce.py @@ -0,0 +1,341 @@ +"""Chunked (vocab-blocked) fused linear cross-entropy — the pure-torch CCE reference, +plus the framework-agnostic `triton_linear_cross_entropy` entry point that dispatches +to the Triton kernel on CUDA and FALLS BACK to this chunked-torch reference on CPU. + +Deliverable #1 of the /kernel-dev CCE lift (SPEC_cce_fused_linear_ce.md, +CCE = "Cut Your Losses in Large-Vocabulary LMs", arXiv 2411.09009). It computes + + loss = mean_i ( logsumexp_v(H[i] @ W[v]) - (H[i] @ W[y_i]) ) + +together with the gradients d_hidden (dH) and d_lm_head_weight (dW), WITHOUT ever +materializing the full (N, V) logits tensor. For Qwen3-0.6B (V = 151,936, D = 1024, +N = 16,384 per micro-batch) that (N, V) fp32 tensor is ~9.96 GB; this reference +never allocates it — the largest intermediate along the vocab axis is one chunk of +`chunk_size` columns, i.e. (N, chunk_size). + +WHY A PURE-TORCH REFERENCE (and not "just the Triton kernel") +------------------------------------------------------------ +Two reasons, both load-bearing: + 1. It is the CPU-testable ORACLE-adjacent implementation: it runs and is + numerically correct on CPU at tiny sizes (V=512, N=64, D=32) so the whole + mechanism — online log-sum-exp, the indexed target logit, the block-wise + softmax-minus-onehot backward, the eps gradient filter — is unit-tested TODAY + without a GPU. See research/tests/test_cce_linear_ce.py. + 2. It is itself a genuine memory-saving implementation. The vocab loop means peak + activation for the CE term is O(N * chunk_size) instead of O(N * V). On a box + where over-allocation OOM-kills the whole machine (§C1), that alone is useful + even before the Triton kernel is rooflined off-box. + +CORRECTNESS ORACLE (the HARD gate — SPEC §"Correctness oracle"): + ref = torch.nn.functional.cross_entropy(H.float() @ W.float().T, y) + forward loss: atol 1e-3 + backward dH/dW: rtol/atol 1e-2 (bf16, loosened for the eps filter) +NOT bit-exact by design: the eps=2**-12 gradient filter drops sub-bf16-resolution +softmax entries (the CCE key trick), which perturbs the gradient below the bf16 +noise floor. + +The math here mirrors the Triton kernel (cce_triton.py) line-for-line so the two can +be diffed during hand review. NOTE (honest scope): this reference runs every big GEMM +in fp32; the Triton kernel runs the LOGIT recompute with bf16 tensor-core operands +(fp32 accumulate) and stores bf16 dH/dW. The CPU suite covers the fp32 path AND a +bf16-rounded emulation of the kernel path, but the *actual* kernel numerics are gated +only off-box by cce_triton.gate_against_reference(). +""" +from __future__ import annotations + +import importlib + +import torch + +# --------------------------------------------------------------------------- +# eps=2**-12 gradient-filter threshold (CCE, arXiv 2411.09009 §4). +# +# HONEST bf16 rationale (the earlier comment here was numerically wrong): bf16 has +# 7 EXPLICIT mantissa bits, so ULP(1.0) = 2**-7 and the round-to-1.0 boundary is +# ~2**-8 (half a ULP; (1.0 + 2**-8) rounds back to 1.0, (1.0 + 2**-7) does not). +# 2**-12 sits ~32x BELOW that boundary, so it is a deliberately CONSERVATIVE +# SUB-ULP filter: a softmax entry below it, sitting next to the O(1) onehot term +# in the gradient, changes nothing a bf16-resolution accumulate could represent. +# It is NOT "the smallest non-truncated bf16 magnitude" (that is ~2**-7); the value +# is a safe filter floor, and the CCE speedup comes from *skipping* the resulting +# provably-zero blocks, not from bf16 representability per se. +# --------------------------------------------------------------------------- +EPS_BF16 = 2.0 ** -12 # == 0.000244140625 (exactly representable in bf16) + +# Populated on every forward() call — a self-reported audit trail the unit test +# cross-checks against an INDEPENDENT torch.matmul shape guard. Records the widest +# vocab-axis tile ever formed so a regression that accidentally builds (N, V) is +# caught even if the independent guard were ever removed. +LAST_STATS: dict = {} + + +def _validate_labels(y, V, ignore_index): + """Fail LOUDLY on out-of-range labels, exactly as F.cross_entropy raises an + IndexError. A fused CE that silently absorbs y>=V or stray-negative y (its + onehot never subtracted, its target logit never gathered) would hide a + data-pipeline bug that stock CE surfaces immediately. Cheap on (N,) ints.""" + valid = (y != ignore_index) + bad = valid & ((y < 0) | (y >= V)) + if bool(bad.any()): + n_bad = int(bad.sum()) + raise ValueError( + f"{n_bad} label(s) out of range [0, {V}) with ignore_index={ignore_index} " + f"(first bad value {int(y[bad][0])}); F.cross_entropy would raise IndexError.") + + +def _online_lse_and_target(H32, W, y, chunk_size, ignore_index): + """FORWARD core. One streaming pass over the vocab in blocks of `chunk_size`. + + Returns (lse, zy, max_tile_cols): + lse (N,) fp32 : the exact per-row log-sum-exp of the full-vocab logits, + accumulated with the numerically-stable online (running max + m + running sum s) recurrence — never storing all V logits. + zy (N,) fp32 : the indexed correct-token logit H[i] @ W[y_i] (0 for rows + whose label == ignore_index; their loss is masked later). + max_tile_cols : the widest vocab tile formed (== chunk_size except the + remainder block) — proves (N, V) is never allocated. + """ + N, D = H32.shape + V = W.shape[0] + device = H32.device + + # Online log-sum-exp accumulators, fp32 (SPEC: "fp32 accumulation for the LSE"). + m = torch.full((N,), float("-inf"), dtype=torch.float32, device=device) # running max + s = torch.zeros((N,), dtype=torch.float32, device=device) # running sum of exp(logit - m) + zy = torch.zeros((N,), dtype=torch.float32, device=device) # gathered correct-token logit + + row_idx = torch.arange(N, device=device) + max_tile_cols = 0 + + for start in range(0, V, chunk_size): + end = min(start + chunk_size, V) + cols = end - start + max_tile_cols = max(max_tile_cols, cols) + + Wc = W[start:end].float() # (C, D) fp32 view->copy of one vocab block + # torch.matmul (NOT the @ operator) is used deliberately for EVERY big GEMM so + # the unit test's global torch.matmul shape guard can observe them and assert + # none is (N, V). This is a load-bearing contract — do not refactor to @. + logits = torch.matmul(H32, Wc.t()) # (N, C) fp32 <-- ONLY (N, C), never (N, V) + + # --- streaming, numerically-stable log-sum-exp update --- + cmax = logits.amax(dim=1) # (N,) max over this block + new_m = torch.maximum(m, cmax) # new running max + # s <- s * exp(m - new_m) + sum_c exp(logit_c - new_m). exp(-inf)=0 handles the + # first block (m starts at -inf) with no NaN because new_m is finite there. + s = s * torch.exp(m - new_m) + torch.exp(logits - new_m.unsqueeze(1)).sum(dim=1) + m = new_m + + # --- gather the correct-token logit for rows whose label lands in this block --- + in_blk = (y >= start) & (y < end) # (N,) bool; false for ignore_index=-100 + if bool(in_blk.any()): + rows = row_idx[in_blk] + local = y[in_blk] - start # column within this block + zy[rows] = logits[rows, local] + + lse = m + torch.log(s) # (N,) exact full-vocab logsumexp + return lse, zy, max_tile_cols + + +def _blocked_backward(H, W, y, lse, chunk_size, ignore_index, row_scale, grad_filter_eps): + """BACKWARD core. Recompute the SAME blocked logits (memory-optimal recompute, + not caching) and accumulate dH, dW block by block in fp32. + + For each vocab block the local gradient wrt the logits is + dlogit = softmax(logit) - onehot(y) (the standard CE gradient) + scaled per row by row_scale (the reduction/grad_output factor, 0 for + ignore_index rows). Then, exactly as the fused kernel does: + dH += dlogit @ W_block (N,C)@(C,D) -> (N,D) + dW_block += dlogit.T @ H (C,N)@(N,D) -> (C,D) + + GRADIENT FILTER (CCE): softmax entries below grad_filter_eps are zeroed before + forming dlogit. The correct-token -1 (onehot) term is subtracted AFTER the + filter, so the exact target gradient is always preserved even if its own + softmax prob happened to fall below eps. + """ + H32 = H.float() + N, D = H32.shape + V = W.shape[0] + device = H32.device + + dH32 = torch.zeros((N, D), dtype=torch.float32, device=device) # fp32 accumulate (SPEC) + dW32 = torch.zeros((V, D), dtype=torch.float32, device=device) # fp32 accumulate (SPEC) + row_idx = torch.arange(N, device=device) + + for start in range(0, V, chunk_size): + end = min(start + chunk_size, V) + + Wc = W[start:end].float() # (C, D) + logits = torch.matmul(H32, Wc.t()) # (N, C) fp32 — recomputed, never (N, V) + + # softmax via the saved (exact) lse: p = exp(logit - lse) + p = torch.exp(logits - lse.unsqueeze(1)) # (N, C) in [0, 1] + + # --- eps gradient filter: drop softmax entries below the bf16 floor --- + if grad_filter_eps and grad_filter_eps > 0.0: + p = torch.where(p >= grad_filter_eps, p, torch.zeros((), dtype=p.dtype, device=device)) + + # --- subtract the onehot (correct token) AFTER filtering: exact target term --- + in_blk = (y >= start) & (y < end) + if bool(in_blk.any()): + rows = row_idx[in_blk] + local = y[in_blk] - start + p[rows, local] -= 1.0 + + # per-row scale: grad_output * (reduction factor); 0 for ignore_index rows, + # so their dlogit row is 0 and contributes nothing to dH/dW. + dlogit = p * row_scale.unsqueeze(1) # (N, C) fp32 + + dH32 += torch.matmul(dlogit, Wc) # (N, C) @ (C, D) -> (N, D) + dW32[start:end] += torch.matmul(dlogit.t(), H32) # (C, N) @ (N, D) -> (C, D) + + return dH32.to(H.dtype), dW32.to(W.dtype) + + +class _LinearCrossEntropy(torch.autograd.Function): + """autograd.Function wiring the blocked forward/backward. Only H and W receive + gradients; y and the scalar config args return None.""" + + @staticmethod + def forward(ctx, H, W, y, chunk_size, ignore_index, reduction, grad_filter_eps): + if H.dim() != 2 or W.dim() != 2 or H.shape[1] != W.shape[1]: + raise ValueError(f"expected H (N,D), W (V,D) with matching D; got {tuple(H.shape)}, {tuple(W.shape)}") + if y.dim() != 1 or y.shape[0] != H.shape[0]: + raise ValueError(f"expected y (N,) matching H rows; got {tuple(y.shape)}") + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + V = W.shape[0] + _validate_labels(y, V, ignore_index) + + H32 = H.float() + lse, zy, max_tile_cols = _online_lse_and_target(H32, W, y, chunk_size, ignore_index) + + valid = (y != ignore_index) # (N,) bool + n_valid = int(valid.sum().item()) + per_row = lse - zy # (N,) fp32 per-row CE + per_row = torch.where(valid, per_row, torch.zeros((), dtype=per_row.dtype, device=per_row.device)) + + if reduction == "mean": + # Match F.cross_entropy exactly: an all-ignored batch (n_valid==0) is 0/0 -> + # NaN, NOT a silent 0.0 that would hide a fully-masked (labeling-bug) batch. + denom = torch.tensor(float(n_valid), dtype=per_row.dtype, device=per_row.device) + loss = per_row.sum() / denom + elif reduction == "sum": + loss = per_row.sum() + elif reduction == "none": + loss = per_row + else: + raise ValueError(f"reduction must be mean|sum|none, got {reduction!r}") + + ctx.save_for_backward(H, W, y, lse) + ctx.chunk_size = chunk_size + ctx.ignore_index = ignore_index + ctx.reduction = reduction + ctx.grad_filter_eps = grad_filter_eps + ctx.n_valid = n_valid + + LAST_STATS.clear() + LAST_STATS.update( + N=int(H.shape[0]), D=int(H.shape[1]), V=int(W.shape[0]), + chunk_size=int(chunk_size), n_chunks=(int(W.shape[0]) + chunk_size - 1) // chunk_size, + max_logits_tile_cols=int(max_tile_cols), n_valid=n_valid, reduction=reduction, + ) + return loss + + @staticmethod + def backward(ctx, grad_output): + H, W, y, lse = ctx.saved_tensors + valid = (y != ctx.ignore_index) + + # Build the per-row scale = grad_output * d(reduction)/d(per_row_i), 0 for ignored rows. + if ctx.reduction == "mean": + base = grad_output / max(ctx.n_valid, 1) # scalar tensor + row_scale = torch.where(valid, base.to(torch.float32).expand(y.shape[0]), + torch.zeros((), dtype=torch.float32, device=y.device)) + elif ctx.reduction == "sum": + row_scale = torch.where(valid, grad_output.to(torch.float32).expand(y.shape[0]), + torch.zeros((), dtype=torch.float32, device=y.device)) + else: # "none": grad_output is (N,) + row_scale = torch.where(valid, grad_output.to(torch.float32), + torch.zeros((), dtype=torch.float32, device=y.device)) + + dH, dW = _blocked_backward( + H, W, y, lse, ctx.chunk_size, ctx.ignore_index, row_scale, ctx.grad_filter_eps) + # forward signature: (H, W, y, chunk_size, ignore_index, reduction, grad_filter_eps) + return dH, dW, None, None, None, None, None + + +def linear_cross_entropy( + H: torch.Tensor, + W: torch.Tensor, + y: torch.Tensor, + chunk_size: int = 8192, + ignore_index: int = -100, + reduction: str = "mean", + grad_filter_eps: float = EPS_BF16, +) -> torch.Tensor: + """Fused linear cross-entropy, vocab-chunked, never materializing (N, V). + + Args: + H: (N, D) hidden states (any float dtype; upcast to fp32 internally). + W: (V, D) lm_head weight (tied to embed). Its .grad slot receives dW. + y: (N,) int64 target token ids. ignore_index rows are dropped from the mean. + chunk_size: vocab block width. Peak CE activation is O(N * chunk_size). + Correctness is independent of chunk_size (streaming LSE is exact); + it is purely a memory/perf knob. Production: 8192 (19 blocks over 151,936). + ignore_index: label value excluded from loss + gradient (default -100, as + torch.nn.functional.cross_entropy). + reduction: 'mean' | 'sum' | 'none'. + grad_filter_eps: softmax entries below this are zeroed in the backward + (CCE trick). Pass 0.0 to disable the filter (exact CE gradient). + + Returns: + Scalar loss (mean/sum) or (N,) per-row loss (none). + """ + return _LinearCrossEntropy.apply( + H, W, y, chunk_size, ignore_index, reduction, grad_filter_eps) + + +# ====================================================================================== +# Framework-agnostic entry point: Triton on CUDA, chunked-torch reference otherwise. +# ====================================================================================== +def _load_cce_triton(): + """Import the sibling cce_triton module by whatever import path is live (added to + sys.path as a top-level module by the tests / kernel-dev harness, or as a package + submodule). Returns the module or None if it cannot be imported.""" + for name in ("cce_triton", "research.kernel.cce_triton"): + try: + return importlib.import_module(name) + except Exception: + continue + return None + + +def triton_linear_cross_entropy( + H: torch.Tensor, + W: torch.Tensor, + y: torch.Tensor, + ignore_index: int = -100, + reduction: str = "mean", + grad_filter_eps: float = EPS_BF16, + chunk_size: int = 8192, +) -> torch.Tensor: + """The production entry point. Dispatches to the fused Triton kernel when Triton + is importable AND all inputs are on CUDA; otherwise transparently FALLS BACK to + the chunked-torch reference above (so this function imports and runs on a + triton-less / CPU-only box — e.g. in CI). + + The Triton path has NO chunk_size knob (blocking is set by the kernel launch + tiles); chunk_size only steers the CPU/torch fallback. Semantics are identical + across both paths (same math, same eps filter, same reduction policy).""" + on_cuda = bool(H.is_cuda and W.is_cuda and y.is_cuda) + if on_cuda: + mod = _load_cce_triton() + if mod is not None and getattr(mod, "HAS_TRITON", False): + return mod.triton_linear_cross_entropy( + H, W, y, ignore_index=ignore_index, reduction=reduction, + grad_filter_eps=grad_filter_eps) + # CPU / triton-less fallback: the unit-tested chunked reference. + return linear_cross_entropy( + H, W, y, chunk_size=chunk_size, ignore_index=ignore_index, + reduction=reduction, grad_filter_eps=grad_filter_eps) diff --git a/research/kernel/cce_torch.py b/research/kernel/cce_torch.py new file mode 100644 index 0000000..6e3dfd0 --- /dev/null +++ b/research/kernel/cce_torch.py @@ -0,0 +1,259 @@ +"""Chunked (vocab-blocked) fused linear cross-entropy — the pure-torch CCE reference. + +This is deliverable #1 of the /kernel-dev CCE lift (SPEC_cce_fused_linear_ce.md, +CCE = "Cut Your Losses in Large-Vocabulary LMs", arXiv 2411.09009). It computes + + loss = mean_i ( logsumexp_v(H[i] @ W[v]) - (H[i] @ W[y_i]) ) + +together with the gradients d_hidden (dH) and d_lm_head_weight (dW), WITHOUT ever +materializing the full (N, V) logits tensor. For Qwen3-0.6B (V = 151,936, D = 1024, +N = 16,384 per micro-batch) that (N, V) fp32 tensor is ~9.96 GB; this reference +never allocates it — the largest intermediate along the vocab axis is one chunk of +`chunk_size` columns, i.e. (N, chunk_size). + +WHY A PURE-TORCH REFERENCE (and not "just the Triton kernel") +------------------------------------------------------------ +Two reasons, both load-bearing: + 1. It is the CPU-testable ORACLE-adjacent implementation: it runs and is + numerically correct on CPU at tiny sizes (V=512, N=64, D=32) so the whole + mechanism — online log-sum-exp, the indexed target logit, the block-wise + softmax-minus-onehot backward, the eps gradient filter — is unit-tested TODAY + without a GPU. See research/tests/test_cce_linear_ce.py. + 2. It is itself a genuine memory-saving implementation. The vocab loop means peak + activation for the CE term is O(N * chunk_size) instead of O(N * V). On a box + where over-allocation OOM-kills the whole machine (§C1), that alone is useful + even before the Triton kernel is rooflined off-box. + +CORRECTNESS ORACLE (the HARD gate — SPEC §"Correctness oracle"): + ref = torch.nn.functional.cross_entropy(H.float() @ W.float().T, y) + forward loss: atol 1e-3 + backward dH/dW: rtol/atol 1e-2 (bf16, loosened for the eps filter) +NOT bit-exact by design: the eps=2**-12 gradient filter drops sub-bf16-precision +softmax entries (the CCE key trick), which perturbs the gradient below the bf16 +noise floor. + +Design mirrors the mechanism the Triton kernel (cce_triton.py) implements, so the +two can be diffed line-for-line during hand review. +""" +from __future__ import annotations + +import torch + +# The smallest bf16 value that is NOT truncated to zero when added to a number of +# order 1 is 2**-12 (bf16 has 8 mantissa bits -> ~2 decimal digits). CCE filters +# softmax entries below this out of the backward: they are ~0.02% of the gradient +# mass and contribute nothing a bf16 accumulate can even represent. (arXiv 2411.09009 §4) +EPS_BF16 = 2.0 ** -12 # == 0.000244140625 + +# Populated on every forward() call — a self-reported audit trail the unit test +# cross-checks against an INDEPENDENT torch.matmul shape guard. Records the widest +# vocab-axis tile ever formed so a regression that accidentally builds (N, V) is +# caught even if the independent guard were ever removed. +LAST_STATS: dict = {} + + +def _online_lse_and_target(H32, W, y, chunk_size, ignore_index): + """FORWARD core. One streaming pass over the vocab in blocks of `chunk_size`. + + Returns (lse, zy, max_tile_cols): + lse (N,) fp32 : the exact per-row log-sum-exp of the full-vocab logits, + accumulated with the numerically-stable online (running max + m + running sum s) recurrence — never storing all V logits. + zy (N,) fp32 : the indexed correct-token logit H[i] @ W[y_i] (0 for rows + whose label == ignore_index; their loss is masked later). + max_tile_cols : the widest vocab tile formed (== chunk_size except the + remainder block) — proves (N, V) is never allocated. + """ + N, D = H32.shape + V = W.shape[0] + device = H32.device + + # Online log-sum-exp accumulators, fp32 (SPEC: "fp32 accumulation for the LSE"). + m = torch.full((N,), float("-inf"), dtype=torch.float32, device=device) # running max + s = torch.zeros((N,), dtype=torch.float32, device=device) # running sum of exp(logit - m) + zy = torch.zeros((N,), dtype=torch.float32, device=device) # gathered correct-token logit + + row_idx = torch.arange(N, device=device) + max_tile_cols = 0 + + for start in range(0, V, chunk_size): + end = min(start + chunk_size, V) + cols = end - start + max_tile_cols = max(max_tile_cols, cols) + + Wc = W[start:end].float() # (C, D) fp32 view->copy of one vocab block + # torch.matmul (NOT the @ operator) is used deliberately so the unit test's + # global matmul-shape guard can observe every GEMM and assert none is (N, V). + logits = torch.matmul(H32, Wc.t()) # (N, C) fp32 <-- ONLY (N, C), never (N, V) + + # --- streaming, numerically-stable log-sum-exp update --- + cmax = logits.amax(dim=1) # (N,) max over this block + new_m = torch.maximum(m, cmax) # new running max + # s <- s * exp(m - new_m) + sum_c exp(logit_c - new_m). exp(-inf)=0 handles the + # first block (m starts at -inf) with no NaN because new_m is finite there. + s = s * torch.exp(m - new_m) + torch.exp(logits - new_m.unsqueeze(1)).sum(dim=1) + m = new_m + + # --- gather the correct-token logit for rows whose label lands in this block --- + in_blk = (y >= start) & (y < end) # (N,) bool; false for ignore_index=-100 + if bool(in_blk.any()): + rows = row_idx[in_blk] + local = y[in_blk] - start # column within this block + zy[rows] = logits[rows, local] + + lse = m + torch.log(s) # (N,) exact full-vocab logsumexp + return lse, zy, max_tile_cols + + +def _blocked_backward(H, W, y, lse, chunk_size, ignore_index, row_scale, grad_filter_eps): + """BACKWARD core. Recompute the SAME blocked logits (memory-optimal recompute, + not caching) and accumulate dH, dW block by block in fp32. + + For each vocab block the local gradient wrt the logits is + dlogit = softmax(logit) - onehot(y) (the standard CE gradient) + scaled per row by row_scale (the reduction/grad_output factor, 0 for + ignore_index rows). Then, exactly as the fused kernel does: + dH += dlogit @ W_block (N,C)@(C,D) -> (N,D) + dW_block += dlogit.T @ H (C,N)@(N,D) -> (C,D) + + GRADIENT FILTER (CCE): softmax entries below grad_filter_eps are zeroed before + forming dlogit. The correct-token -1 (onehot) term is subtracted AFTER the + filter, so the exact target gradient is always preserved even if its own + softmax prob happened to fall below eps. + """ + H32 = H.float() + N, D = H32.shape + V = W.shape[0] + device = H32.device + + dH32 = torch.zeros((N, D), dtype=torch.float32, device=device) # fp32 accumulate (SPEC) + dW32 = torch.zeros((V, D), dtype=torch.float32, device=device) # fp32 accumulate (SPEC) + row_idx = torch.arange(N, device=device) + + for start in range(0, V, chunk_size): + end = min(start + chunk_size, V) + + Wc = W[start:end].float() # (C, D) + logits = torch.matmul(H32, Wc.t()) # (N, C) fp32 — recomputed, never (N, V) + + # softmax via the saved (exact) lse: p = exp(logit - lse) + p = torch.exp(logits - lse.unsqueeze(1)) # (N, C) in [0, 1] + + # --- eps gradient filter: drop softmax entries below the bf16 floor --- + if grad_filter_eps and grad_filter_eps > 0.0: + p = torch.where(p >= grad_filter_eps, p, torch.zeros((), dtype=p.dtype, device=device)) + + # --- subtract the onehot (correct token) AFTER filtering: exact target term --- + in_blk = (y >= start) & (y < end) + if bool(in_blk.any()): + rows = row_idx[in_blk] + local = y[in_blk] - start + p[rows, local] -= 1.0 + + # per-row scale: grad_output * (reduction factor); 0 for ignore_index rows, + # so their dlogit row is 0 and contributes nothing to dH/dW. + dlogit = p * row_scale.unsqueeze(1) # (N, C) fp32 + + dH32 += torch.matmul(dlogit, Wc) # (N, C) @ (C, D) -> (N, D) + dW32[start:end] += torch.matmul(dlogit.t(), H32) # (C, N) @ (N, D) -> (C, D) + + return dH32.to(H.dtype), dW32.to(W.dtype) + + +class _LinearCrossEntropy(torch.autograd.Function): + """autograd.Function wiring the blocked forward/backward. Only H and W receive + gradients; y and the scalar config args return None.""" + + @staticmethod + def forward(ctx, H, W, y, chunk_size, ignore_index, reduction, grad_filter_eps): + if H.dim() != 2 or W.dim() != 2 or H.shape[1] != W.shape[1]: + raise ValueError(f"expected H (N,D), W (V,D) with matching D; got {tuple(H.shape)}, {tuple(W.shape)}") + if y.dim() != 1 or y.shape[0] != H.shape[0]: + raise ValueError(f"expected y (N,) matching H rows; got {tuple(y.shape)}") + if chunk_size <= 0: + raise ValueError("chunk_size must be positive") + + H32 = H.float() + lse, zy, max_tile_cols = _online_lse_and_target(H32, W, y, chunk_size, ignore_index) + + valid = (y != ignore_index) # (N,) bool + n_valid = int(valid.sum().item()) + per_row = lse - zy # (N,) fp32 per-row CE + per_row = torch.where(valid, per_row, torch.zeros((), dtype=per_row.dtype, device=per_row.device)) + + if reduction == "mean": + loss = per_row.sum() / max(n_valid, 1) + elif reduction == "sum": + loss = per_row.sum() + elif reduction == "none": + loss = per_row + else: + raise ValueError(f"reduction must be mean|sum|none, got {reduction!r}") + + ctx.save_for_backward(H, W, y, lse) + ctx.chunk_size = chunk_size + ctx.ignore_index = ignore_index + ctx.reduction = reduction + ctx.grad_filter_eps = grad_filter_eps + ctx.n_valid = n_valid + + LAST_STATS.clear() + LAST_STATS.update( + N=int(H.shape[0]), D=int(H.shape[1]), V=int(W.shape[0]), + chunk_size=int(chunk_size), n_chunks=(int(W.shape[0]) + chunk_size - 1) // chunk_size, + max_logits_tile_cols=int(max_tile_cols), n_valid=n_valid, reduction=reduction, + ) + return loss + + @staticmethod + def backward(ctx, grad_output): + H, W, y, lse = ctx.saved_tensors + valid = (y != ctx.ignore_index) + + # Build the per-row scale = grad_output * d(reduction)/d(per_row_i), 0 for ignored rows. + if ctx.reduction == "mean": + base = grad_output / max(ctx.n_valid, 1) # scalar tensor + row_scale = torch.where(valid, base.to(torch.float32).expand(y.shape[0]), + torch.zeros((), dtype=torch.float32, device=y.device)) + elif ctx.reduction == "sum": + row_scale = torch.where(valid, grad_output.to(torch.float32).expand(y.shape[0]), + torch.zeros((), dtype=torch.float32, device=y.device)) + else: # "none": grad_output is (N,) + row_scale = torch.where(valid, grad_output.to(torch.float32), + torch.zeros((), dtype=torch.float32, device=y.device)) + + dH, dW = _blocked_backward( + H, W, y, lse, ctx.chunk_size, ctx.ignore_index, row_scale, ctx.grad_filter_eps) + # forward signature: (H, W, y, chunk_size, ignore_index, reduction, grad_filter_eps) + return dH, dW, None, None, None, None, None + + +def linear_cross_entropy( + H: torch.Tensor, + W: torch.Tensor, + y: torch.Tensor, + chunk_size: int = 8192, + ignore_index: int = -100, + reduction: str = "mean", + grad_filter_eps: float = EPS_BF16, +) -> torch.Tensor: + """Fused linear cross-entropy, vocab-chunked, never materializing (N, V). + + Args: + H: (N, D) hidden states (any float dtype; upcast to fp32 internally). + W: (V, D) lm_head weight (tied to embed). Its .grad slot receives dW. + y: (N,) int64 target token ids. ignore_index rows are dropped from the mean. + chunk_size: vocab block width. Peak CE activation is O(N * chunk_size). + Correctness is independent of chunk_size (streaming LSE is exact); + it is purely a memory/perf knob. Production: 8192 (19 blocks over 151,936). + ignore_index: label value excluded from loss + gradient (default -100, as + torch.nn.functional.cross_entropy). + reduction: 'mean' | 'sum' | 'none'. + grad_filter_eps: softmax entries below this are zeroed in the backward + (CCE trick). Pass 0.0 to disable the filter (exact CE gradient). + + Returns: + Scalar loss (mean/sum) or (N,) per-row loss (none). + """ + return _LinearCrossEntropy.apply( + H, W, y, chunk_size, ignore_index, reduction, grad_filter_eps) diff --git a/research/kernel/cce_triton.py b/research/kernel/cce_triton.py new file mode 100644 index 0000000..0f41831 --- /dev/null +++ b/research/kernel/cce_triton.py @@ -0,0 +1,545 @@ +"""Fused linear cross-entropy — Triton kernels (the perf target of the CCE lift). + +Deliverable #2 of SPEC_cce_fused_linear_ce.md. This is the fused forward + backward +Triton implementation of the SAME mechanism as the pure-torch reference in +cce_linear_ce.py. It is written to be reviewed LINE-FOR-LINE against that reference +(the mechanism is identical; only the loop nest and memory hierarchy differ). + +>>> IMPORTANT — NOT YET GPU-VALIDATED <<< +The GB10 GPU is occupied for hours, so these kernels have NOT been compiled or run on +hardware yet. They are provided for HAND REVIEW of the numerics (LSE stability, the +softmax-minus-onehot backward, the eps filter, block/tile indexing, dtype and fp32 +accumulation). The correctness GATE (§C21) is gate_against_reference() below — run it +on a profileable GPU BEFORE ANY benchmark. A kernel that fails the gate is DISCARDED, +never rooflined. + +WHAT THE CPU SUITE DOES *NOT* COVER (be honest): research/tests/test_cce_linear_ce.py +gates the fp32 torch reference and a bf16-ROUNDED CPU emulation of this kernel's path, +but it cannot run these Triton kernels. The bf16 tensor-core logit recompute, the exact +tile scheduling, register/SRAM occupancy, and the D-tiled accumulation below are proven +ONLY by the off-box gate. CPU-green != kernel-correct. + +VOCAB REMAINDER (correcting an earlier false note): V = 151,936 = 2**7 * 1187, so it IS +divisible by 16/32/64/128. At the default divisor tiles (BLOCK_V in {64,128}) the +`mask_v = offs_v < V` remainder branch is INACTIVE (always true) and is dead code in +production. It exists only so an autotuner may legally pick a non-divisor BLOCK_V (e.g. +256 -> remainder 128); if it ever does, that masked path must be exercised on GPU first. + +DESIGN (three kernels, no atomics) +---------------------------------- + fwd : grid over row-blocks. Streaming online LSE over vocab-blocks in SRAM + (fp32 running max m + running sum s), plus the indexed target logit. + Writes per-row LSE (N,) and ZY (N,). The scalar loss is reduced on the + host from those two (N,) vectors — cheap, and still never (N, V). + bwd_dH : grid over row-blocks. Each program OWNS the dH rows for its block (no + atomics). D is TILED (outer loop over BLOCK_D output columns); the logit + recompute uses the SAME inner BLOCK_D contraction + bf16 operands as the + forward, so exp(logit - lse) reconstructs a softmax consistent with the + forward's LSE. Never holds a (·, D) tile whole. + bwd_dW : grid over vocab-blocks. Each program OWNS the dW rows for its block; same + D-tiled structure. + +Resource fix (was a blocker): the previous version held (BLOCK_N, D)/(BLOCK_V, D) fp32 +accumulators and whole-D operand tiles at D=1024 (128-256 KB) — a near-certain +out-of-resource / heavy-spill on a single SM. Both backward kernels now tile D so the +widest live tile is (BLOCK_*, BLOCK_D). The cost is that the logits are recomputed once +per output-D block (D/BLOCK_D times); that is a memory-safe DEFAULT the off-box tuner +can trade back for speed (larger BLOCK_D, or a cached-dlogit variant) once it compiles. + +Numeric-consistency fix: forward and both backward kernels contract the hidden dim in +the SAME BLOCK_D order with the SAME bf16 operands, so the backward-recomputed logits +match the forward's (bf16 matmul is non-associative — mismatched tiling would make the +per-row softmax not sum to 1 against the saved LSE). + +Precision fix: the ACCUMULATE GEMMs (dlogit @ W, dlogit.T @ H) keep dlogit in fp32 and +cast the W/H operand to fp32, matching the fp32 reference (the old kernel downcast +dlogit to bf16, injecting ~0.4% error). The off-box engineer may switch these to bf16 +tensor-core for speed IF the gate still passes at fp32-cast. + +Reference template: apple/ml-cross-entropy (MIT). CCE: arXiv 2411.09009. +""" +from __future__ import annotations + +import torch + +try: + import triton + import triton.language as tl + HAS_TRITON = True +except Exception: # pragma: no cover - CPU-only / triton-less box + triton = None + tl = None + HAS_TRITON = False + +# CCE gradient-filter threshold. See cce_linear_ce.EPS_BF16 for the (corrected) bf16 +# rationale: 2**-12 is a CONSERVATIVE SUB-ULP floor (bf16 ULP(1.0) = 2**-7, round-to-1 +# boundary ~2**-8), NOT "the smallest non-truncated bf16 magnitude". Accumulation here +# is fp32, so the filter's justification is the SPEC's <0.02% dropped-mass-for-peaked- +# softmax + the loosened 1e-2 tolerance + the block-skip sparsity — valid for peaked +# (trained-model) softmax, still an open question at pretraining scale. +EPS_BF16 = 2.0 ** -12 + + +# ====================================================================================== +# Kernels (only defined when Triton is importable; otherwise the wrappers raise clearly) +# ====================================================================================== +if HAS_TRITON: + + @triton.jit + def _cce_fwd_kernel( + H_ptr, W_ptr, Y_ptr, LSE_ptr, ZY_ptr, + N, V, D, + stride_hn, stride_hd, + stride_wv, stride_wd, + IGNORE_INDEX: tl.constexpr, + BLOCK_N: tl.constexpr, BLOCK_V: tl.constexpr, BLOCK_D: tl.constexpr, + ): + """Forward: per-row online log-sum-exp over the full vocab + indexed target logit. + + One program handles BLOCK_N rows. It streams over the vocab in BLOCK_V-wide + blocks, and within each vocab block contracts the full hidden dim D in BLOCK_D + steps. All reductions are fp32. Never materializes (N, V): the widest live logit + tile is (BLOCK_N, BLOCK_V).""" + pid_n = tl.program_id(0) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) # (BN,) + mask_n = offs_n < N + + # target ids as int64 (labels are int64); ignore rows read as IGNORE_INDEX so + # they can never match a (non-negative) vocab column. + y = tl.load(Y_ptr + offs_n, mask=mask_n, other=IGNORE_INDEX).to(tl.int64) # (BN,) + + m = tl.full((BLOCK_N,), -float("inf"), tl.float32) # running max + s = tl.zeros((BLOCK_N,), tl.float32) # running sum of exp(logit - m) + zy = tl.zeros((BLOCK_N,), tl.float32) # gathered target logit + + for v0 in range(0, V, BLOCK_V): + offs_v = v0 + tl.arange(0, BLOCK_V) # (BV,) int32 + mask_v = offs_v < V # REMAINDER MASK (inactive at divisor tiles) + offs_v64 = offs_v.to(tl.int64) # int64 for the target compare + + # ---- logits tile (BN, BV) fp32, contract D in BLOCK_D steps (bf16 operands) ---- + acc = tl.zeros((BLOCK_N, BLOCK_V), tl.float32) + for d0 in range(0, D, BLOCK_D): + cur_d = d0 + tl.arange(0, BLOCK_D) + mask_d = cur_d < D + h = tl.load( + H_ptr + offs_n[:, None] * stride_hn + cur_d[None, :] * stride_hd, + mask=mask_n[:, None] & mask_d[None, :], other=0.0, + ) # (BN, BD) bf16 + w = tl.load( + W_ptr + offs_v[:, None] * stride_wv + cur_d[None, :] * stride_wd, + mask=mask_v[:, None] & mask_d[None, :], other=0.0, + ) # (BV, BD) bf16 + acc += tl.dot(h, tl.trans(w)) # (BN, BV) += (BN,BD)@(BD,BV), fp32 acc + + # mask padded vocab columns to -inf so they never enter max/sum/gather + acc = tl.where(mask_v[None, :], acc, -float("inf")) + + # ---- streaming stable LSE update ---- + cmax = tl.max(acc, axis=1) # (BN,) + new_m = tl.maximum(m, cmax) + p = tl.exp(acc - new_m[:, None]) # (BN, BV); masked cols -> exp(-inf)=0 + s = s * tl.exp(m - new_m) + tl.sum(p, axis=1) # exp(-inf)=0 on the first block + m = new_m + + # ---- gather the correct-token logit for rows whose label is in this block ---- + in_blk = (y[:, None] == offs_v64[None, :]) & mask_v[None, :] # (BN, BV) bool + zy += tl.sum(tl.where(in_blk, acc, 0.0), axis=1) # exactly one hit per matching row + + lse = m + tl.log(s) # (BN,) + tl.store(LSE_ptr + offs_n, lse, mask=mask_n) + tl.store(ZY_ptr + offs_n, zy, mask=mask_n) + + @triton.jit + def _cce_recompute_logits( + H_ptr, W_ptr, offs_n, mask_n, offs_v, mask_v, + stride_hn, stride_hd, stride_wv, stride_wd, + D, BLOCK_N: tl.constexpr, BLOCK_V: tl.constexpr, BLOCK_D: tl.constexpr, + ): + """Shared helper: recompute one (BN, BV) logits tile with the EXACT same + BLOCK_D contraction order + bf16 operands as the forward, so the backward's + softmax is consistent with the saved forward LSE.""" + acc = tl.zeros((BLOCK_N, BLOCK_V), tl.float32) + for d0 in range(0, D, BLOCK_D): + cur_d = d0 + tl.arange(0, BLOCK_D) + mask_d = cur_d < D + h = tl.load( + H_ptr + offs_n[:, None] * stride_hn + cur_d[None, :] * stride_hd, + mask=mask_n[:, None] & mask_d[None, :], other=0.0, + ) + w = tl.load( + W_ptr + offs_v[:, None] * stride_wv + cur_d[None, :] * stride_wd, + mask=mask_v[:, None] & mask_d[None, :], other=0.0, + ) + acc += tl.dot(h, tl.trans(w)) + return acc + + @triton.jit + def _cce_bwd_dh_kernel( + H_ptr, W_ptr, Y_ptr, LSE_ptr, SCALE_ptr, DH_ptr, + N, V, D, + stride_hn, stride_hd, + stride_wv, stride_wd, + stride_dhn, stride_dhd, + EPS: tl.constexpr, IGNORE_INDEX: tl.constexpr, + BLOCK_N: tl.constexpr, BLOCK_V: tl.constexpr, BLOCK_D: tl.constexpr, + TILE_SKIP: tl.constexpr, + ): + """Backward dH. Grid over row-blocks; each program owns dH rows [pid_n] (no + atomics). D is TILED via an OUTER loop over BLOCK_D output columns, so the + accumulator is only (BLOCK_N, BLOCK_D) — never (BLOCK_N, D). + + dH[:, d_out] = sum_v dlogit[:, v] * W[v, d_out], dlogit = filter(softmax) - onehot.""" + pid_n = tl.program_id(0) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) # (BN,) + mask_n = offs_n < N + + y = tl.load(Y_ptr + offs_n, mask=mask_n, other=IGNORE_INDEX).to(tl.int64) # (BN,) + lse = tl.load(LSE_ptr + offs_n, mask=mask_n, other=0.0) # (BN,) fp32 + scale = tl.load(SCALE_ptr + offs_n, mask=mask_n, other=0.0) # (BN,) fp32 per-row grad scale + + for d_out in range(0, D, BLOCK_D): # OUTER over output D + offs_do = d_out + tl.arange(0, BLOCK_D) + mask_do = offs_do < D + acc = tl.zeros((BLOCK_N, BLOCK_D), tl.float32) # small (BN, BD) accumulator + + for v0 in range(0, V, BLOCK_V): + offs_v = v0 + tl.arange(0, BLOCK_V) + mask_v = offs_v < V # REMAINDER MASK + offs_v64 = offs_v.to(tl.int64) + + logits = _cce_recompute_logits( + H_ptr, W_ptr, offs_n, mask_n, offs_v, mask_v, + stride_hn, stride_hd, stride_wv, stride_wd, + D, BLOCK_N, BLOCK_V, BLOCK_D) + logits = tl.where(mask_v[None, :], logits, -float("inf")) + + p = tl.exp(logits - lse[:, None]) # softmax (BN, BV) in [0,1] + p = tl.where(p >= EPS, p, 0.0) # eps gradient filter (elementwise, exact) + + in_blk = (y[:, None] == offs_v64[None, :]) & mask_v[None, :] + dlogit = tl.where(in_blk, p - 1.0, p) # subtract onehot AFTER filter + dlogit = dlogit * scale[:, None] + dlogit = tl.where(mask_v[None, :], dlogit, 0.0) # zero padded cols + + # W tile for the output-D columns only (BV, BD) — never (BV, D) + w_out = tl.load( + W_ptr + offs_v[:, None] * stride_wv + offs_do[None, :] * stride_wd, + mask=mask_v[:, None] & mask_do[None, :], other=0.0, + ) + # fp32 accumulate GEMM (operand cast to fp32 to match the fp32 reference). + # TILE_SKIP is a compile-time toggle for the CCE block-skip: when a tile's + # dlogit is provably all-zero (fully filtered, no target) the accumulate is + # a no-op and can be skipped. Data-dependent control flow is version- + # sensitive in Triton, so it DEFAULTS OFF (always-accumulate = the + # correct, always-compilable fallback); the elementwise filter above + # already guarantees correctness. Flip on off-box after confirming it + # compiles + measuring the real skip rate on a PEAKED (trained) softmax. + if TILE_SKIP: + if tl.max(tl.abs(dlogit)) > 0.0: + acc += tl.dot(dlogit, w_out.to(tl.float32)) + else: + acc += tl.dot(dlogit, w_out.to(tl.float32)) + + tl.store( + DH_ptr + offs_n[:, None] * stride_dhn + offs_do[None, :] * stride_dhd, + acc, mask=mask_n[:, None] & mask_do[None, :], + ) + + @triton.jit + def _cce_bwd_dw_kernel( + H_ptr, W_ptr, Y_ptr, LSE_ptr, SCALE_ptr, DW_ptr, + N, V, D, + stride_hn, stride_hd, + stride_wv, stride_wd, + stride_dwv, stride_dwd, + EPS: tl.constexpr, IGNORE_INDEX: tl.constexpr, + BLOCK_N: tl.constexpr, BLOCK_V: tl.constexpr, BLOCK_D: tl.constexpr, + TILE_SKIP: tl.constexpr, + ): + """Backward dW. Grid over vocab-blocks; each program owns dW rows [pid_v] (no + atomics). D is TILED via an OUTER loop over BLOCK_D output columns, so the + accumulator is only (BLOCK_V, BLOCK_D) — never (BLOCK_V, D). + + dW[:, d_out] = sum_n dlogit[n, :].T * H[n, d_out].""" + pid_v = tl.program_id(0) + offs_v = pid_v * BLOCK_V + tl.arange(0, BLOCK_V) # (BV,) + mask_v = offs_v < V # REMAINDER MASK + offs_v64 = offs_v.to(tl.int64) + + for d_out in range(0, D, BLOCK_D): # OUTER over output D + offs_do = d_out + tl.arange(0, BLOCK_D) + mask_do = offs_do < D + acc = tl.zeros((BLOCK_V, BLOCK_D), tl.float32) # small (BV, BD) accumulator + + for n0 in range(0, N, BLOCK_N): + offs_n = n0 + tl.arange(0, BLOCK_N) + mask_n = offs_n < N + + y = tl.load(Y_ptr + offs_n, mask=mask_n, other=IGNORE_INDEX).to(tl.int64) + lse = tl.load(LSE_ptr + offs_n, mask=mask_n, other=0.0) + scale = tl.load(SCALE_ptr + offs_n, mask=mask_n, other=0.0) + + logits = _cce_recompute_logits( + H_ptr, W_ptr, offs_n, mask_n, offs_v, mask_v, + stride_hn, stride_hd, stride_wv, stride_wd, + D, BLOCK_N, BLOCK_V, BLOCK_D) + logits = tl.where(mask_v[None, :], logits, -float("inf")) + + p = tl.exp(logits - lse[:, None]) # (BN, BV) + p = tl.where(p >= EPS, p, 0.0) # eps filter + + in_blk = (y[:, None] == offs_v64[None, :]) & mask_v[None, :] + dlogit = tl.where(in_blk, p - 1.0, p) + dlogit = dlogit * scale[:, None] + dlogit = tl.where(mask_n[:, None] & mask_v[None, :], dlogit, 0.0) + + h_out = tl.load( + H_ptr + offs_n[:, None] * stride_hn + offs_do[None, :] * stride_hd, + mask=mask_n[:, None] & mask_do[None, :], other=0.0, + ) + if TILE_SKIP: + if tl.max(tl.abs(dlogit)) > 0.0: + acc += tl.dot(tl.trans(dlogit), h_out.to(tl.float32)) + else: + acc += tl.dot(tl.trans(dlogit), h_out.to(tl.float32)) + + tl.store( + DW_ptr + offs_v[:, None] * stride_dwv + offs_do[None, :] * stride_dwd, + acc, mask=mask_v[:, None] & mask_do[None, :], + ) + + +# ====================================================================================== +# Python wrappers / autograd glue +# ====================================================================================== +def _require_triton_cuda(H, W, y): + if not HAS_TRITON: + raise RuntimeError( + "Triton is not importable — the Triton CCE kernel needs a CUDA box with " + "triton installed. Use cce_linear_ce.linear_cross_entropy for the CPU reference.") + if not (H.is_cuda and W.is_cuda and y.is_cuda): + raise RuntimeError("Triton CCE kernel requires all inputs on CUDA.") + + +# Default launch tiles. PLACEHOLDERS for an off-box autotune sweep — chosen so that no +# (·, D) tile is ever held whole (D is tiled by BLOCK_D). Not tuned for occupancy. +# BLOCK_V is kept a divisor of V=151936 (64) so the remainder mask stays inactive. +_FWD_TILES = dict(BLOCK_N=64, BLOCK_V=64, BLOCK_D=128) +_DH_TILES = dict(BLOCK_N=32, BLOCK_V=64, BLOCK_D=128) +_DW_TILES = dict(BLOCK_N=32, BLOCK_V=64, BLOCK_D=128) +# CCE block-skip. OFF by default (always-accumulate = correct + always compiles). +# Turn on off-box only after confirming the data-dependent branch compiles on the +# target Triton/GB10 AND measuring the real skip rate on a peaked softmax. +_TILE_SKIP = False + + +def _validate_labels(y, V, ignore_index): + valid = (y != ignore_index) + bad = valid & ((y < 0) | (y >= V)) + if bool(bad.any()): + raise ValueError( + f"{int(bad.sum())} label(s) out of range [0, {V}) with ignore_index={ignore_index}; " + f"F.cross_entropy would raise IndexError.") + + +class _TritonLinearCrossEntropy(torch.autograd.Function): + @staticmethod + def forward(ctx, H, W, y, ignore_index, reduction, grad_filter_eps): + _require_triton_cuda(H, W, y) + N, D = H.shape + V = W.shape[0] + _validate_labels(y, V, ignore_index) + + lse = torch.empty((N,), dtype=torch.float32, device=H.device) + zy = torch.empty((N,), dtype=torch.float32, device=H.device) + + grid = (triton.cdiv(N, _FWD_TILES["BLOCK_N"]),) + _cce_fwd_kernel[grid]( + H, W, y, lse, zy, + N, V, D, + H.stride(0), H.stride(1), + W.stride(0), W.stride(1), + IGNORE_INDEX=ignore_index, **_FWD_TILES, + ) + + valid = (y != ignore_index) + n_valid = int(valid.sum().item()) + per_row = torch.where(valid, lse - zy, torch.zeros((), dtype=torch.float32, device=H.device)) + if reduction == "mean": + # 0/0 -> NaN for an all-ignored batch, matching F.cross_entropy. + denom = torch.tensor(float(n_valid), dtype=torch.float32, device=H.device) + loss = per_row.sum() / denom + elif reduction == "sum": + loss = per_row.sum() + elif reduction == "none": + loss = per_row + else: + raise ValueError(f"reduction must be mean|sum|none, got {reduction!r}") + + ctx.save_for_backward(H, W, y, lse) + ctx.ignore_index = ignore_index + ctx.reduction = reduction + ctx.grad_filter_eps = grad_filter_eps + ctx.n_valid = n_valid + return loss + + @staticmethod + def backward(ctx, grad_output): + H, W, y, lse = ctx.saved_tensors + N, D = H.shape + V = W.shape[0] + valid = (y != ctx.ignore_index) + + # per-row grad scale (same policy as the torch reference) + if ctx.reduction == "mean": + base = (grad_output / max(ctx.n_valid, 1)).to(torch.float32) + row_scale = torch.where(valid, base.expand(N), torch.zeros((), dtype=torch.float32, device=H.device)) + elif ctx.reduction == "sum": + row_scale = torch.where(valid, grad_output.to(torch.float32).expand(N), + torch.zeros((), dtype=torch.float32, device=H.device)) + else: # none + row_scale = torch.where(valid, grad_output.to(torch.float32), + torch.zeros((), dtype=torch.float32, device=H.device)) + row_scale = row_scale.contiguous() + + dH = torch.zeros_like(H) + dW = torch.zeros_like(W) + eps = ctx.grad_filter_eps if ctx.grad_filter_eps else 0.0 + + grid_dh = (triton.cdiv(N, _DH_TILES["BLOCK_N"]),) + _cce_bwd_dh_kernel[grid_dh]( + H, W, y, lse, row_scale, dH, + N, V, D, + H.stride(0), H.stride(1), + W.stride(0), W.stride(1), + dH.stride(0), dH.stride(1), + EPS=eps, IGNORE_INDEX=ctx.ignore_index, TILE_SKIP=_TILE_SKIP, **_DH_TILES, + ) + + grid_dw = (triton.cdiv(V, _DW_TILES["BLOCK_V"]),) + _cce_bwd_dw_kernel[grid_dw]( + H, W, y, lse, row_scale, dW, + N, V, D, + H.stride(0), H.stride(1), + W.stride(0), W.stride(1), + dW.stride(0), dW.stride(1), + EPS=eps, IGNORE_INDEX=ctx.ignore_index, TILE_SKIP=_TILE_SKIP, **_DW_TILES, + ) + # forward signature: (H, W, y, ignore_index, reduction, grad_filter_eps) -> 6 inputs, + # so backward MUST return exactly 6 grads (dH, dW, then one None per config arg). + return dH, dW, None, None, None, None + + +def triton_linear_cross_entropy( + H: torch.Tensor, + W: torch.Tensor, + y: torch.Tensor, + ignore_index: int = -100, + reduction: str = "mean", + grad_filter_eps: float = EPS_BF16, +) -> torch.Tensor: + """Triton fused linear cross-entropy (CUDA only). Same signature/semantics as + cce_linear_ce.linear_cross_entropy but without the chunk_size knob (blocking is set + by the kernel launch tiles). See module docstring for the correctness gate. For a + CPU-safe entry that falls back to the reference, use + cce_linear_ce.triton_linear_cross_entropy.""" + return _TritonLinearCrossEntropy.apply(H, W, y, ignore_index, reduction, grad_filter_eps) + + +# ====================================================================================== +# §C21 HARD correctness gate (OFF-BOX, needs CUDA). MUST pass before any roofline. +# ====================================================================================== +def gate_against_reference(N=1024, D=1024, V=8192, dtype=None, device="cuda", + atol_loss=1e-3, rtol_grad=2e-2, seed=0): + """OFF-BOX correctness gate (§C21). Compares the Triton kernel to the model's own + fp32 unfused CE and returns a verdict dict. MUST pass before any roofline. + + THREE fixes vs the original vacuous gate: + 1. PEAKED inputs (0.5 scale, matching the meaningful CPU-test regime) so the + softmax concentrates and the eps filter actually DROPS mass — not the + near-uniform regime where the filter was a no-op and every check was benign. + 2. RELATIVE grad tolerance: atol is tied to the largest reference-grad magnitude + (rtol_grad * |ref|.max()), because under mean-reduction the gradients are + O(1/N) ~ 1e-4 and a FIXED atol=1e-2 was ~100x larger than the signal — the old + gate greened an all-zero backward. The check is now scaled to the data. + 3. NEGATIVE CONTROLS: the gate only PASSES if a zero-gradient AND a half-gradient + candidate both FAIL the same comparison. If they don't, the tolerance is + vacuous and the gate returns passed=False (fail closed) regardless of the + real candidate — so a broken/absent backward can never be certified. + + Also reports the oracle-INDEPENDENT eps bound: filter-on vs filter-off FUSED grads. + Not run in CI (needs CUDA + the busy GB10/rented GPU). Mirrors SPEC tolerances.""" + import torch.nn.functional as F + if dtype is None: + dtype = torch.bfloat16 + g = torch.Generator(device=device).manual_seed(seed) + # Peaked: 0.5 scale -> logit std ~ 0.25*sqrt(D), a concentrated softmax over V. + H = torch.randn(N, D, device=device, dtype=dtype, generator=g) * 0.5 + W = torch.randn(V, D, device=device, dtype=dtype, generator=g) * 0.5 + y = torch.randint(0, V, (N,), device=device, generator=g) + + # --- reference: fp32 unfused CE (the HARD oracle) --- + Href = H.detach().clone().float().requires_grad_(True) + Wref = W.detach().clone().float().requires_grad_(True) + ref_loss = F.cross_entropy(torch.matmul(Href, Wref.t()), y) + ref_loss.backward() + ref_dH, ref_dW = Href.grad, Wref.grad + + # --- candidate: Triton fused (default eps filter ON) --- + Hc = H.detach().clone().requires_grad_(True) + Wc = W.detach().clone().requires_grad_(True) + cand_loss = triton_linear_cross_entropy(Hc, Wc, y) + cand_loss.backward() + cand_dH, cand_dW = Hc.grad.float(), Wc.grad.float() + + # relative, data-scaled tolerances (fix #2) + dH_atol = rtol_grad * float(ref_dH.abs().max()) + dW_atol = rtol_grad * float(ref_dW.abs().max()) + + def _passes(cand, ref, atol): + return bool(((cand - ref).abs() <= atol + rtol_grad * ref.abs()).all()) + + loss_ok = abs(float(cand_loss) - float(ref_loss)) <= atol_loss + dH_ok = _passes(cand_dH, ref_dH, dH_atol) + dW_ok = _passes(cand_dW, ref_dW, dW_atol) + + # negative controls (fix #3): zero AND half grads MUST fail, else the gate is vacuous. + zero_fails = (not _passes(torch.zeros_like(ref_dH), ref_dH, dH_atol)) and \ + (not _passes(torch.zeros_like(ref_dW), ref_dW, dW_atol)) + half_fails = (not _passes(0.5 * ref_dH, ref_dH, dH_atol)) and \ + (not _passes(0.5 * ref_dW, ref_dW, dW_atol)) + non_vacuous = zero_fails and half_fails + + # oracle-independent eps bound: filter-off vs filter-on FUSED grads. + Hc0 = H.detach().clone().requires_grad_(True) + Wc0 = W.detach().clone().requires_grad_(True) + loss0 = triton_linear_cross_entropy(Hc0, Wc0, y, grad_filter_eps=0.0) + loss0.backward() + eps_dH_gap = float((Hc.grad.float() - Hc0.grad.float()).abs().max()) + eps_dW_gap = float((Wc.grad.float() - Wc0.grad.float()).abs().max()) + + passed = bool(loss_ok and dH_ok and dW_ok and non_vacuous) + return { + "passed": passed, + "loss_ref": float(ref_loss), "loss_cand": float(cand_loss), + "d_loss": abs(float(ref_loss) - float(cand_loss)), "loss_ok": loss_ok, + "dH_max_err": float((cand_dH - ref_dH).abs().max()), "dH_ok": dH_ok, "dH_atol": dH_atol, + "dW_max_err": float((cand_dW - ref_dW).abs().max()), "dW_ok": dW_ok, "dW_atol": dW_atol, + "ref_dH_absmax": float(ref_dH.abs().max()), "ref_dW_absmax": float(ref_dW.abs().max()), + "negative_control_zero_fails": zero_fails, + "negative_control_half_fails": half_fails, + "non_vacuous": non_vacuous, + "eps_filter_dH_gap": eps_dH_gap, "eps_filter_dW_gap": eps_dW_gap, + "note": ("passed=True requires the real candidate to match AND the zero/half " + "negative controls to FAIL (proving the tolerance is not vacuous)."), + } + + +if __name__ == "__main__": # pragma: no cover - off-box smoke + if HAS_TRITON and torch.cuda.is_available(): + import json + print(json.dumps(gate_against_reference(N=1024), indent=2, default=str)) + else: + print("Triton/CUDA unavailable — cannot run the Triton CCE gate here. " + "Use cce_linear_ce.linear_cross_entropy on CPU for the unit-tested reference.") diff --git a/research/kernel/gb10_cce_bench.py b/research/kernel/gb10_cce_bench.py new file mode 100644 index 0000000..4b2c393 --- /dev/null +++ b/research/kernel/gb10_cce_bench.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""GB10 (Blackwell) fused-CE benchmark — the same 'killing the logit tensor' evals as the Kaggle +notebook, but on real deployment hardware where CCE's 96 KB-shared-memory kernel can actually run. +safe_cuda-guarded; the anneal is paused while this runs (one GPU job at a time, §C4.5).""" +import sys, pathlib, json, gc +import numpy as np +ROOT = pathlib.Path("/home/yashb98/Downloads/BuildFromScratch") +sys.path.insert(0, str(ROOT)) +import safe_cuda # noqa: F401 — caps the process before torch touches CUDA +import torch, torch.nn.functional as F +import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt +import pandas as pd + +safe_cuda.guard(0.85) +dev = torch.device("cuda") +cap = torch.cuda.get_device_capability(0) +DTYPE = torch.bfloat16 if cap[0] >= 8 else torch.float16 # GB10 is Blackwell -> bf16 +V, D, SEED = 151936, 1024, 0 +OUT = ROOT / "research/kernel" +print("GPU", torch.cuda.get_device_name(0), "| cc", cap, "| dtype", DTYPE, + "| pool", round(torch.cuda.get_device_properties(0).total_memory/1e9, 1), "GB", flush=True) + +def make(N): + g = torch.Generator(device=dev).manual_seed(SEED) + h = (torch.randn(N, D, device=dev, dtype=DTYPE, generator=g) * .1).requires_grad_() + W = (torch.randn(V, D, device=dev, dtype=DTYPE, generator=g) * .02).requires_grad_() + y = torch.randint(0, V, (N,), device=dev, generator=g) + return h, W, y + +def ce_naive(h, W, y): return F.cross_entropy((h @ W.t()).float(), y) +ce_compiled = torch.compile(ce_naive) +methods = {"naive": ce_naive, "torch.compile": ce_compiled} +try: + from cut_cross_entropy import linear_cross_entropy + methods["CCE"] = lambda h, W, y: linear_cross_entropy(h, W, y); print("CCE imported", flush=True) +except Exception as e: print("CCE import failed:", repr(e)[:140], flush=True) +try: + from liger_kernel.transformers import LigerFusedLinearCrossEntropyLoss + _flce = LigerFusedLinearCrossEntropyLoss() + methods["Liger"] = lambda h, W, y: _flce(W, h, y); print("Liger imported", flush=True) +except Exception as e: print("Liger import failed:", repr(e)[:140], flush=True) + +# smoke each -> drop any that won't run on this GPU (e.g. CCE if Triton is too new for Blackwell) +for nm in list(methods): + try: + h, W, y = make(64); methods[nm](h, W, y).backward(); del h, W, y; torch.cuda.empty_cache() + print(f"[{nm}] smoke OK", flush=True) + except Exception as e: + print(f"[{nm}] smoke FAILED -> dropped: {repr(e)[:150]}", flush=True); methods.pop(nm); torch.cuda.empty_cache() +print("ACTIVE METHODS:", list(methods), flush=True) + +def isoom(e): return isinstance(e, torch.cuda.OutOfMemoryError) or "out of memory" in str(e).lower() + +# --- Eval 1: correctness vs fp32 oracle --- +def ref(h, W, y): return F.cross_entropy(h.float() @ W.float().t(), y) +def grads(fn, N): + h, W, y = make(N); L = fn(h, W, y); L.backward() + return float(L), h.grad.float().clone(), W.grad.float().clone() +Nc = 2048; Lr, ghr, gWr = grads(ref, Nc); rows = [] +for nm, fn in methods.items(): + if nm in ("naive", "torch.compile"): continue + try: + Lf, ghf, gWf = grads(fn, Nc) + rows.append([nm, round(Lr,6), round(Lf,6), abs(Lr-Lf), + (ghr-ghf).abs().max().item(), (gWr-gWf).abs().max().item()]) + except Exception as e: rows.append([nm, round(Lr,6), None, None, None, repr(e)[:50]]) + torch.cuda.empty_cache() +df_chk = pd.DataFrame(rows, columns=["method","loss_ref","loss","|d_loss|","d_hidden_max","d_weight_max"]) +df_chk.to_csv(OUT/"gb10_correctness.csv", index=False); print("CORRECTNESS\n", df_chk.to_string(index=False), flush=True) + +# --- Eval 2/3: memory + throughput sweep (GB10 has ~119 GB; safe_cuda catches OOM cleanly) --- +N_SWEEP = [4096, 16384, 32768, 65536] +def peak_mem(fn, N): + gc.collect(); torch.cuda.empty_cache(); torch.cuda.reset_peak_memory_stats() + try: + h, W, y = make(N); fn(h, W, y).backward(); torch.cuda.synchronize() + m = torch.cuda.max_memory_allocated()/1e9; del h, W, y + except Exception as e: + if not isoom(e): print("memerr", N, repr(e)[:80], flush=True) + m = float("nan") + gc.collect(); torch.cuda.empty_cache(); return m +def bench(fn, N, it=10, wu=3): + gc.collect(); torch.cuda.empty_cache() + try: + h, W, y = make(N) + for _ in range(wu): h.grad=None; W.grad=None; fn(h, W, y).backward() + torch.cuda.synchronize(); s, e = torch.cuda.Event(True), torch.cuda.Event(True); ts=[] + for _ in range(it): + h.grad=None; W.grad=None; s.record(); fn(h, W, y).backward(); e.record(); torch.cuda.synchronize() + ts.append(s.elapsed_time(e)) + t = float(np.median(ts)); del h, W, y + except Exception: t = float("nan") + gc.collect(); torch.cuda.empty_cache(); return t +mem = {nm: [peak_mem(fn, N) for N in N_SWEEP] for nm, fn in methods.items()} +tput = {nm: [bench(fn, N) for N in N_SWEEP] for nm, fn in methods.items()} +df_mem = pd.DataFrame(mem, index=N_SWEEP); df_mem.index.name="N_tokens"; df_mem.to_csv(OUT/"gb10_memory.csv") +df_t = pd.DataFrame(tput, index=N_SWEEP); df_t.index.name="N_tokens"; df_t.to_csv(OUT/"gb10_throughput.csv") +print("MEMORY (GB)\n", df_mem.round(3).to_string(), flush=True) +print("THROUGHPUT (ms)\n", df_t.round(2).to_string(), flush=True) + +# --- plots --- +for data, ylab, fn_png, title in [(mem,"peak GPU memory (GB)","gb10_mem.png","peak memory"), + (tput,"fwd+bwd time (ms)","gb10_tput.png","throughput")]: + plt.figure(figsize=(8,5)) + for nm in methods: plt.plot(N_SWEEP, data[nm], "o-", label=nm) + plt.xscale("log"); plt.xlabel("tokens N (batch×seq, log)"); plt.ylabel(ylab) + plt.title(f"GB10 Grace Blackwell — {title} vs tokens (V={V}, D={D}, bf16)") + plt.legend(); plt.grid(True, alpha=.3); plt.tight_layout(); plt.savefig(OUT/fn_png, dpi=120) + +summary = {"gpu": torch.cuda.get_device_name(0), "cc": list(cap), "dtype": str(DTYPE), + "pool_gb": round(torch.cuda.get_device_properties(0).total_memory/1e9,1), + "V": V, "D": D, "methods": list(methods), "N_sweep": N_SWEEP, + "memory_gb": {k:[None if v!=v else round(v,3) for v in mem[k]] for k in methods}, + "throughput_ms": {k:[None if v!=v else round(v,2) for v in tput[k]] for k in methods}, + "correctness": rows} +json.dump(summary, open(OUT/"gb10_summary.json","w"), indent=2, default=str) +print("SAVED gb10_* ->", OUT, "\nDONE", flush=True) diff --git a/research/kernel/pause_bench_resume.sh b/research/kernel/pause_bench_resume.sh new file mode 100755 index 0000000..b1cdcdf --- /dev/null +++ b/research/kernel/pause_bench_resume.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Pause the anneal at its step-1500 checkpoint -> run the GB10 CCE benchmark -> ALWAYS resume the +# anneal (trap EXIT) from that checkpoint. One GPU job at a time (§C4.5); resume is guaranteed. +set -u +ROOT=/home/yashb98/Downloads/BuildFromScratch +EXP=$ROOT/Qwen3-0.6B/experiments/2026-06-30_qwen3-0.6b_midtrain-anneal +KDIR=$ROOT/research/kernel +RUNLOG=$KDIR/gb10_cce_run.log + +resume_anneal() { + rm -f "$KDIR/gb10_cce_running" + cd "$EXP" + if ! pgrep -f "$EXP/run_arms.sh" >/dev/null 2>&1 && [ ! -f cohort.done ]; then + setsid nohup bash run_arms.sh >> run_arms.log 2>&1 & + echo "[$(date '+%T')] ANNEAL RESUMED — run_arms.sh relaunched (--resume mix_seed0 from ckpt)" >> "$RUNLOG" + fi +} +trap resume_anneal EXIT # guarantee the anneal comes back, success/fail/timeout + +{ +echo "===== $(date '+%F %T') GB10 CCE benchmark orchestration =====" +echo "[1/4] pip install liger + cce (--no-deps; runs while the anneal trains)" +python3 -m pip install -q --no-deps liger-kernel "git+https://github.com/apple/ml-cross-entropy.git" 2>&1 | tail -3 + +echo "[2/4] waiting for the step-1500 checkpoint (so resume loses ~0 progress)..." +for _ in $(seq 1 180); do + grep -q "ckpt @ 1500" "$EXP/run_mix_seed0.log" 2>/dev/null && { echo " -> ckpt @ 1500 saved"; break; } + sleep 10 +done + +echo "[3/4] pausing the anneal" +touch "$KDIR/gb10_cce_running" +pkill -TERM -f "$EXP/run_arms.sh" 2>/dev/null +pkill -TERM -f "train_anneal.py" 2>/dev/null +pkill -f "sentinel.py watch" 2>/dev/null +sleep 8 +echo " GPU util now: $(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader 2>/dev/null) (should be ~0)" + +echo "[4/4] running the GB10 CCE benchmark (CCE should now fit — Blackwell shared memory)" +cd "$KDIR"; timeout 1500 python3 gb10_cce_bench.py +touch "$KDIR/gb10_cce.done" +echo "[$(date '+%T')] benchmark finished -> EXIT trap will resume the anneal" +} >> "$RUNLOG" 2>&1 diff --git a/research/kernel_oracle.py b/research/kernel_oracle.py new file mode 100644 index 0000000..93a99d2 --- /dev/null +++ b/research/kernel_oracle.py @@ -0,0 +1,295 @@ +"""Correctness-first oracle for the Triton-kernel dev harness (§C21). + +The Frontier-Eng kernel lift writes a candidate kernel (e.g. a fused RMSNorm) +and claims it is faster than the model's reference op. This module is the GATE +that stands BETWEEN "I wrote a kernel" and "I am allowed to benchmark it": a +candidate kernel is run against a REFERENCE op across a sweep of shapes x dtypes +and must agree within a per-dtype tolerance (an allclose-style check). A +fast-but-wrong kernel is DISCARDED here, never rooflined, never reported — the +§C21 correctness-first discipline. A kernel that is 3x faster but disagrees in +the 4th decimal at fp16 is not a speedup, it is a bug. + +WHY THIS IS A CPU MODULE +------------------------ +The subtle, get-it-wrong-able logic of a kernel gate is NOT the GPU launch — it +is the comparison policy: which tolerance per dtype, how a NaN/Inf is treated +(a single non-finite element fails the gate, never "allclose-with-NaN==True"), +how a shape-mismatch or dtype-mismatch is surfaced (a structural failure, not a +numeric one), and how a sweep aggregates into a single PASS/FAIL verdict. All of +that is pure Python/numpy and is proven correct WITHOUT a GPU here, mirroring the +eval_stats.py / posttrain_losses.py pattern: the SKILL is not "documented but +fake" — only the actual Triton launch on a profileable GPU is the off-box part. + +To prove the gate itself works we ship two numpy reference ops (RMSNorm, SiLU) +AND, for each, a deliberately-subtly-wrong "candidate" (an epsilon dropped, a +mean used where the mean-square belongs). The oracle PASSES the correct candidate +and FAILS the wrong one — that is the test of the gate, on CPU. + +TORCH/TRITON ARE OPTIONAL. The module imports on a CPU-only box: torch is only +touched inside helpers guarded by `torch is not None` / `torch.cuda.is_available()`, +and triton is never imported at module load. `compare()` works on anything +array-like (numpy arrays, torch tensors, or plain nested lists) by normalizing to +numpy first, so the same gate logic that runs in CI also runs against real device +tensors off-box (a torch tensor is moved to CPU+numpy before comparison). +""" +from __future__ import annotations + +import math + +import numpy as np + +try: # torch is optional — the gate logic must import & run without it (§ CPU-only) + import torch # noqa: F401 +except Exception: # pragma: no cover - exercised only on a torch-less box + torch = None + + +# -------------------------------------------------------------------------- +# Per-dtype tolerances. fp32 is tight; reduced precision is looser because the +# kernel's accumulation order legitimately differs from the reference's. These +# are the allclose (rtol, atol) pair, matched to torch.testing.assert_close's +# defaults for the corresponding dtype so the CPU gate and an off-box +# torch.testing gate agree on what "close enough" means. +# -------------------------------------------------------------------------- +DTYPE_TOL = { + "float32": {"rtol": 1.3e-6, "atol": 1e-5}, + "float64": {"rtol": 1e-7, "atol": 1e-8}, + "float16": {"rtol": 1e-3, "atol": 1e-5}, + "bfloat16": {"rtol": 1.6e-2, "atol": 1e-5}, +} +DEFAULT_TOL = {"rtol": 1.3e-6, "atol": 1e-5} + + +def tolerance_for(dtype) -> dict: + """The (rtol, atol) pair for a dtype, accepting a numpy dtype, a torch dtype, + or a plain name string ('float16', 'bf16', 'fp32', ...). Unknown dtypes fall + back to the conservative fp32 tolerance (never silently loose).""" + name = _dtype_name(dtype) + return dict(DTYPE_TOL.get(name, DEFAULT_TOL)) + + +def _dtype_name(dtype) -> str: + """Normalize any dtype-ish object to a canonical numpy-style name string.""" + if dtype is None: + return "float32" + raw = str(dtype) + # 'torch.float16' -> 'float16'; 'torch.bfloat16' -> 'bfloat16' + raw = raw.split(".")[-1].strip().lower() + aliases = { + "fp32": "float32", "f32": "float32", "float": "float32", + "fp64": "float64", "f64": "float64", "double": "float64", + "fp16": "float16", "f16": "float16", "half": "float16", + "bf16": "bfloat16", + } + return aliases.get(raw, raw) + + +def _to_numpy(x): + """Coerce an array-like (numpy array, torch tensor on any device, or a nested + Python list/scalar) to a contiguous numpy float64 array WITHOUT requiring a + GPU import path at module load. A torch tensor is detached, moved to CPU, and + upcast — bf16 has no numpy dtype, so we always go through float for the + comparison (the tolerance, not the storage dtype, encodes the precision).""" + if torch is not None and isinstance(x, torch.Tensor): + x = x.detach().to("cpu", dtype=torch.float64).numpy() + arr = np.asarray(x) + if arr.dtype == object: + raise TypeError("ragged / non-numeric array-like passed to oracle") + return arr.astype(np.float64, copy=False) + + +def _allclose_report(ref, cand, rtol, atol): + """numpy-allclose with a diagnostic. Returns (passed, max_abs, max_rel, + n_bad, reason). A NaN/Inf anywhere (in either array) is a HARD FAIL — we + never let np.allclose's equal_nan path mask a kernel that produced NaNs. + The element-wise rule mirrors numpy/torch: |c-r| <= atol + rtol*|r|.""" + if ref.shape != cand.shape: + return False, math.inf, math.inf, ref.size, ( + f"shape mismatch: reference {ref.shape} vs candidate {cand.shape}") + + finite_ref = np.isfinite(ref) + finite_cand = np.isfinite(cand) + if not finite_ref.all() or not finite_cand.all(): + n_bad = int((~finite_cand).sum()) + where = "candidate" if not finite_cand.all() else "reference" + return False, math.inf, math.inf, max(n_bad, 1), ( + f"non-finite (NaN/Inf) values in {where} output — fast-but-wrong, discarded") + + abs_err = np.abs(cand - ref) + tol = atol + rtol * np.abs(ref) + bad = abs_err > tol + n_bad = int(bad.sum()) + max_abs = float(abs_err.max()) if abs_err.size else 0.0 + # relative error guarded against divide-by-zero + denom = np.abs(ref) + rel = np.where(denom > 0, abs_err / denom, 0.0) + max_rel = float(rel.max()) if rel.size else 0.0 + passed = n_bad == 0 + reason = "ok" if passed else ( + f"{n_bad}/{ref.size} elements exceed tol " + f"(max_abs={max_abs:.3e}, max_rel={max_rel:.3e}, rtol={rtol:.1e}, atol={atol:.1e})") + return passed, max_abs, max_rel, n_bad, reason + + +def compare(reference_out, candidate_out, dtype="float32"): + """Single allclose-style comparison of one candidate output vs the reference. + + reference_out / candidate_out: any array-like (numpy, torch tensor, list). + dtype: drives the tolerance (the candidate's compute precision), NOT the + comparison storage — both sides are upcast to float64 first. + + Returns a result dict: passed, dtype, rtol, atol, max_abs_err, max_rel_err, + n_mismatch, n_total, reason. A shape/dtype-structural problem and a numeric + drift are BOTH reported as passed=False but with distinguishable reasons. + """ + tol = tolerance_for(dtype) + ref = _to_numpy(reference_out) + cand = _to_numpy(candidate_out) + passed, max_abs, max_rel, n_bad, reason = _allclose_report( + ref, cand, tol["rtol"], tol["atol"]) + return { + "passed": passed, + "dtype": _dtype_name(dtype), + "rtol": tol["rtol"], "atol": tol["atol"], + "max_abs_err": max_abs, "max_rel_err": max_rel, + "n_mismatch": n_bad, "n_total": int(ref.size), + "reason": reason, + } + + +def gate(reference_fn, candidate_fn, shapes, dtypes=("float32",), + input_factory=None, seed=0): + """Run the full correctness gate: the candidate is compared against the + reference across the CROSS PRODUCT of `shapes` x `dtypes`. The verdict is + PASS iff EVERY cell passes — a kernel correct at fp32 but wrong at one shape + in bf16 is DISCARDED (§C21: a kernel must be correct everywhere it claims to + run, not on average). + + reference_fn / candidate_fn: callables taking the SAME generated input + (a tuple of numpy arrays) and an optional `dtype` keyword; each returns + an array-like output. The reference is the source of truth (the model's + own op, ported to numpy on CPU; the model.py op on GPU off-box). + shapes: iterable of shape tuples, e.g. [(8, 256), (1, 4096)]. + dtypes: iterable of dtype names ('float32', 'float16', 'bfloat16', ...). + input_factory: optional callable(shape, dtype, rng) -> tuple-of-inputs. The + default makes a single standard-normal input array of `shape` (the + common single-tensor elementwise/norm op). Both fns get the SAME inputs. + seed: RNG seed so the sweep is deterministic and reproducible. + + Returns a verdict dict: passed (bool, the gate), n_cells, n_failed, cells + (per-cell result dicts incl. shape/dtype), and a one-line `summary`. The + contract: callers MUST NOT benchmark/roofline a candidate unless + verdict['passed'] is True. + """ + shapes = [tuple(s) for s in shapes] + dtypes = [_dtype_name(d) for d in dtypes] + if not shapes: + raise ValueError("need at least one shape to sweep") + if not dtypes: + raise ValueError("need at least one dtype to sweep") + if input_factory is None: + input_factory = _default_input_factory + + cells = [] + n_failed = 0 + for di, dt in enumerate(dtypes): + for si, shape in enumerate(shapes): + # deterministic, distinct RNG per cell + rng = np.random.default_rng(seed + 1009 * di + si) + inputs = input_factory(shape, dt, rng) + try: + ref_out = _call(reference_fn, inputs, dt) + cand_out = _call(candidate_fn, inputs, dt) + cell = compare(ref_out, cand_out, dtype=dt) + except Exception as exc: # a candidate that raises is a FAIL, not a crash + cell = { + "passed": False, "dtype": dt, "rtol": float("nan"), + "atol": float("nan"), "max_abs_err": math.inf, + "max_rel_err": math.inf, "n_mismatch": -1, "n_total": -1, + "reason": f"candidate raised {type(exc).__name__}: {exc}", + } + cell["shape"] = shape + cells.append(cell) + if not cell["passed"]: + n_failed += 1 + + passed = n_failed == 0 + summary = ( + f"GATE {'PASS' if passed else 'FAIL'}: {len(cells) - n_failed}/{len(cells)} " + f"cells ok across {len(shapes)} shapes x {len(dtypes)} dtypes" + + ("" if passed else " — candidate DISCARDED (correctness-first §C21)")) + return { + "passed": passed, + "n_cells": len(cells), + "n_failed": n_failed, + "cells": cells, + "summary": summary, + } + + +def _call(fn, inputs, dtype): + """Call a reference/candidate fn with the generated inputs, passing `dtype` + only if the fn accepts it (kwarg-tolerant so simple numpy refs need no dtype + param).""" + try: + return fn(*inputs, dtype=dtype) + except TypeError: + return fn(*inputs) + + +def _default_input_factory(shape, dtype, rng): + """Default single-input generator: one standard-normal array of `shape`. + Returns a 1-tuple (so reference/candidate fns are called as fn(x)).""" + x = rng.standard_normal(size=shape).astype(np.float32) + return (x,) + + +# -------------------------------------------------------------------------- +# Reference ops (numpy) + matched candidates. These exist to PROVE the gate on +# CPU: the *_ref is the source of truth; the *_correct candidate must PASS and +# the *_wrong candidate must FAIL. Off-box, the reference_fn is the model's own +# op (model.py) and the candidate_fn wraps the Triton kernel. +# -------------------------------------------------------------------------- + +def rmsnorm_ref(x, weight=None, eps=1e-6, dtype="float32"): + """Reference RMSNorm over the last axis: x / sqrt(mean(x^2) + eps) * weight. + This is the numpy oracle the candidate Triton RMSNorm kernel is gated against.""" + x = np.asarray(x, dtype=np.float64) + ms = np.mean(x * x, axis=-1, keepdims=True) + out = x / np.sqrt(ms + eps) + if weight is not None: + out = out * np.asarray(weight, dtype=np.float64) + return out + + +def rmsnorm_candidate_correct(x, weight=None, eps=1e-6, dtype="float32"): + """A *correct* alternate implementation (different but equivalent algebra: + reciprocal-sqrt factored out). Stands in for a correct Triton kernel — the + gate must PASS this.""" + x = np.asarray(x, dtype=np.float64) + inv = 1.0 / np.sqrt(np.mean(np.square(x), axis=-1, keepdims=True) + eps) + out = x * inv + return out if weight is None else out * np.asarray(weight, dtype=np.float64) + + +def rmsnorm_candidate_wrong(x, weight=None, eps=1e-6, dtype="float32"): + """A SUBTLY WRONG RMSNorm: it normalizes by sqrt(mean(x)^2 + eps) instead of + sqrt(mean(x^2) + eps) — i.e. mean-then-square rather than square-then-mean + (a classic fused-kernel bug). Numerically close on some inputs, catastrophic + on others. The gate MUST FAIL this and DISCARD it (the whole point).""" + x = np.asarray(x, dtype=np.float64) + m = np.mean(x, axis=-1, keepdims=True) # BUG: mean before square + out = x / np.sqrt(m * m + eps) + return out if weight is None else out * np.asarray(weight, dtype=np.float64) + + +def silu_ref(x, dtype="float32"): + """Reference SiLU / swish: x * sigmoid(x).""" + x = np.asarray(x, dtype=np.float64) + return x * (1.0 / (1.0 + np.exp(-x))) + + +def silu_candidate_wrong(x, dtype="float32"): + """WRONG SiLU: plain sigmoid(x) (forgot the elementwise * x). Must FAIL.""" + x = np.asarray(x, dtype=np.float64) + return 1.0 / (1.0 + np.exp(-x)) diff --git a/research/liveness_cron.sh b/research/liveness_cron.sh new file mode 100755 index 0000000..cedcd91 --- /dev/null +++ b/research/liveness_cron.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Lightweight recovery trigger for the research loop (contracts §C6). +# Used by TWO crons: +# */30 * * * * liveness_cron.sh -> catch a silently-dead run in <=30 min +# @reboot liveness_cron.sh 180 -> auto-resume an in-flight run after a crash +# +# Cheap by design: it only runs `sentinel.py liveness` (a PID check, no Claude +# session). It launches a session ONLY when an in-flight run's PID is dead +# (sentinel exit 4), and then defers to cron_runner.sh "/research-loop resume", +# whose flock prevents overlap with the nightly fire and whose S1 logic does +# the full preflight + §C5 (incl. the §C5.0 smoke test) before any relaunch. +set -u + +REPO=/home/yashb98/Downloads/BuildFromScratch +SETTLE="${1:-0}" # seconds to wait before probing (180 on @reboot, 0 otherwise) +LOG_DIR="$REPO/research/cron_logs" +mkdir -p "$LOG_DIR" +LOG="$LOG_DIR/liveness.log" + +[ "$SETTLE" -gt 0 ] 2>/dev/null && sleep "$SETTLE" + +python3 "$REPO/sentinel.py" liveness >>"$LOG" 2>&1 +rc=$? +if [ "$rc" -eq 4 ]; then + echo "$(date -Is) liveness rc=4 (dead in-flight run) -> /research-loop resume" >>"$LOG" + "$REPO/research/cron_runner.sh" "/research-loop resume" +else + # rc 0 = idle or running fine; anything else = probe glitch (fail open). + echo "$(date -Is) liveness rc=$rc -> no action" >>"$LOG" +fi diff --git a/research/papers/qwen3-imu1-matched-compute/figures/make_figures.py b/research/papers/qwen3-imu1-matched-compute/figures/make_figures.py new file mode 100644 index 0000000..bc92fe7 --- /dev/null +++ b/research/papers/qwen3-imu1-matched-compute/figures/make_figures.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""make_figures.py — figures for the qwen3-imu1-matched-compute paper. + +Every figure is plotted from the ORIGINAL result files on disk (no hardcoded +numbers). Run from anywhere; paths are absolute. CPU only, matplotlib Agg. +Outputs PNG (300 dpi) + PDF (vector, for the LaTeX build) into this directory, +and prints the extracted endpoints so they can be cross-checked against +evidence_manifest.json. + +Sources: + faithful CSV : .../reproduce-faithful.../results/qwen3_baseline2tpp_train.csv (step,loss,...) + faithful log : .../reproduce-faithful.../results/qwen3_baseline2tpp_train.log ([eval @ N] val PPL=..) + imu log : .../reproduce-modernized.../results/qwen3_imu1_2tpp_train.log (step lines + eval lines) +""" +import re +import csv +from pathlib import Path +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +REPO = Path("/home/yashb98/Downloads/BuildFromScratch") +FAITH = REPO / "Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results" +MOD = REPO / "Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results" +OUT = Path(__file__).resolve().parent +TOK_PER_STEP = 65_536 + +plt.rcParams.update({ + "font.family": "serif", + "font.size": 9, + "axes.titlesize": 10, + "axes.labelsize": 9, + "legend.fontsize": 8, + "axes.grid": True, + "grid.alpha": 0.3, + "grid.linewidth": 0.4, + "figure.dpi": 300, +}) +FAITH_C = "#444444" # baseline: neutral grey +IMU_C = "#1f6fb4" # modernized: blue + + +def parse_evals(logpath): + """[(step, ppl)] from '[eval @ N] val PPL=X' lines.""" + pat = re.compile(r"\[eval @ (\d+)\] val PPL=([\d.]+)") + out = [] + for line in Path(logpath).read_text().splitlines(): + m = pat.search(line) + if m: + out.append((int(m.group(1)), float(m.group(2)))) + return out + + +def parse_log_ce(logpath): + """[(step, ce)] from 'step N/T ce X' lines (IMU log, every 50 steps).""" + pat = re.compile(r"step\s+(\d+)/\d+\s+ce\s+([\d.]+)") + out = [] + for line in Path(logpath).read_text().splitlines(): + m = pat.search(line) + if m: + out.append((int(m.group(1)), float(m.group(2)))) + return out + + +def parse_csv_loss(csvpath): + """[(step, loss)] from the faithful per-step CSV.""" + out = [] + with open(csvpath) as f: + for row in csv.DictReader(f): + out.append((int(row["step"]), float(row["loss"]))) + return out + + +def smooth(ys, k=51): + """Centered moving average; k odd. Keeps length via edge clamping.""" + if len(ys) < k: + return ys + half = k // 2 + out = [] + for i in range(len(ys)): + lo, hi = max(0, i - half), min(len(ys), i + half + 1) + out.append(sum(ys[lo:hi]) / (hi - lo)) + return out + + +# ---- gather ---- +faith_eval = parse_evals(FAITH / "qwen3_baseline2tpp_train.log") +imu_eval = parse_evals(MOD / "qwen3_imu1_2tpp_train.log") +faith_loss = parse_csv_loss(FAITH / "qwen3_baseline2tpp_train.csv") +imu_ce = parse_log_ce(MOD / "qwen3_imu1_2tpp_train.log") + +print("faithful eval points:", faith_eval) +print("imu eval points:", imu_eval) +print(f"faithful CSV rows: {len(faith_loss)} (final step {faith_loss[-1][0]})") +print(f"imu log CE points: {len(imu_ce)} (final step {imu_ce[-1][0]})") + +# ---- Figure 1: validation perplexity vs tokens (the headline) ---- +fig, ax = plt.subplots(figsize=(5.0, 3.4)) +fx = [s * TOK_PER_STEP / 1e9 for s, _ in faith_eval] +fy = [p for _, p in faith_eval] +ix = [s * TOK_PER_STEP / 1e9 for s, _ in imu_eval] +iy = [p for _, p in imu_eval] +ax.plot(fx, fy, "o-", color=FAITH_C, ms=3.5, lw=1.3, label="Faithful baseline (AdamW + cosine)") +ax.plot(ix, iy, "s-", color=IMU_C, ms=3.5, lw=1.3, label="Modernized (IMU-1 bundle + NorMuon, WSD)") +ax.set_yscale("log") +ax.set_xlabel("Training tokens (billions)") +ax.set_ylabel("Validation perplexity (log scale)") +ax.set_title("Validation perplexity at matched compute") +# annotate the matched-compute endpoints +ax.annotate(f"{fy[-1]:.2f}", (fx[-1], fy[-1]), textcoords="offset points", + xytext=(4, 6), color=FAITH_C, fontsize=8) +ax.annotate(f"{iy[-1]:.2f}", (ix[-1], iy[-1]), textcoords="offset points", + xytext=(4, -10), color=IMU_C, fontsize=8) +ax.legend(frameon=False, loc="upper right") +fig.tight_layout() +fig.savefig(OUT / "fig1_val_ppl.png") +fig.savefig(OUT / "fig1_val_ppl.pdf") +plt.close(fig) + +# ---- Figure 2: training cross-entropy vs step (dynamics) ---- +fig, ax = plt.subplots(figsize=(5.0, 3.4)) +fs = [s for s, _ in faith_loss] +fl = smooth([l for _, l in faith_loss], 101) +isx = [s for s, _ in imu_ce] +il = smooth([c for _, c in imu_ce], 11) +ax.plot(fs, fl, color=FAITH_C, lw=1.0, label="Faithful baseline (per-step, smoothed)") +ax.plot(isx, il, color=IMU_C, lw=1.2, label="Modernized (every 50 steps, smoothed)") +ax.set_xlabel("Optimizer step") +ax.set_ylabel("Training cross-entropy (nats)") +ax.set_title("Training loss") +ax.set_ylim(2.5, 6.0) +ax.legend(frameon=False, loc="upper right") +fig.tight_layout() +fig.savefig(OUT / "fig2_train_loss.png") +fig.savefig(OUT / "fig2_train_loss.pdf") +plt.close(fig) + +print("\nWROTE:", sorted(p.name for p in OUT.glob("fig*.p*"))) +print("ENDPOINTS for cross-check -> faithful final eval %.2f @ %.3fB tok ; imu final eval %.2f @ %.3fB tok" + % (fy[-1], fx[-1], iy[-1], ix[-1])) diff --git a/research/posttrain_losses.py b/research/posttrain_losses.py new file mode 100644 index 0000000..20d656a --- /dev/null +++ b/research/posttrain_losses.py @@ -0,0 +1,175 @@ +"""Pure post-training loss/objective math — stdlib-only, no torch, no model, no +network. The CPU-buildable, unit-testable CORE of the post-training executor +(SFT / DPO / GRPO), mirroring the eval_stats.py / eval_metrics.py pattern: the +subtle math lives here and is proven correct WITHOUT a GPU, so the SKILL is not +"documented but fake" — only the actual training loop (forward/backward on a +checkpoint) is the GPU-staged part. + +Lifecycle slot: §C13 `finetune` objective on an EXISTING checkpoint (NOT a +from-scratch build, §C4.2). Methods covered: + * SFT — standard next-token CE; helper here is the (optionally + label-smoothed) per-token NLL from a target log-prob. + * DPO — Direct Preference Optimization closed-form loss (no reward model, + no RL rollout): a function of policy & reference log-probs on a + chosen/rejected pair. + * GRPO — Group-Relative Policy Optimization: group-normalized advantages + + a PPO-style clipped surrogate + the k3 KL estimator. + +Everything operates on plain Python lists of per-example scalar log-probs / +rewards / ratios (whatever the training loop reduces a sequence to), so it is +deterministic and directly testable (test_posttrain_losses.py). +""" +from __future__ import annotations + +import math +import statistics + + +# --------------------------------------------------------------- helpers + +def log_sigmoid(x): + """Numerically-stable log(sigmoid(x)) = -softplus(-x).""" + if x >= 0: + return -math.log1p(math.exp(-x)) + return x - math.log1p(math.exp(x)) + + +# --------------------------------------------------------------- SFT + +def label_smoothed_nll(logp_target, vocab_size, smoothing=0.0): + """Per-token SFT loss from the model's log-prob of the GOLD token. + smoothing=0 -> plain negative log-likelihood (-logp_target). With label + smoothing eps, the target distribution is (1-eps) on the gold token and + eps/V uniform elsewhere, giving loss = (1-eps)*(-logp_target) + eps*mean_nll, + approximated (uniform-prior surrogate) as (1-eps)*(-logp) + eps*log(V). + Raises on an out-of-range smoothing.""" + if not (0.0 <= smoothing < 1.0): + raise ValueError("smoothing must be in [0,1)") + if vocab_size < 2: + raise ValueError("vocab_size must be >= 2") + return (1 - smoothing) * (-logp_target) + smoothing * math.log(vocab_size) + + +# --------------------------------------------------------------- DPO + +def dpo_loss(logp_pol_chosen, logp_ref_chosen, + logp_pol_rejected, logp_ref_rejected, beta=0.1): + """Direct Preference Optimization loss over a batch of preference pairs. + Inputs are equal-length lists of SEQUENCE log-probs (sum of token log-probs) + under the policy and the frozen reference, for the chosen and rejected + responses. Returns a dict: + + loss = mean of -log_sigmoid( beta * (chosen_logratio - rejected_logratio) ) + chosen_reward / rejected_reward = mean implicit reward beta*(logp_pol - logp_ref) + accuracy = fraction of pairs where chosen_reward > rejected_reward + margin = mean(chosen_reward - rejected_reward) + + No reward model, no rollouts — DPO's whole point is this closed form. + """ + n = len(logp_pol_chosen) + if not (len(logp_ref_chosen) == len(logp_pol_rejected) == len(logp_ref_rejected) == n): + raise ValueError("all four log-prob lists must be the same length") + if n == 0: + raise ValueError("need at least one preference pair") + losses, ch_r, rj_r, correct = [], [], [], 0 + for pc, rc, pr, rr in zip(logp_pol_chosen, logp_ref_chosen, + logp_pol_rejected, logp_ref_rejected): + chosen_logratio = pc - rc # log pi(y_w)/pi_ref(y_w) + rejected_logratio = pr - rr # log pi(y_l)/pi_ref(y_l) + losses.append(-log_sigmoid(beta * (chosen_logratio - rejected_logratio))) + cr, rj = beta * chosen_logratio, beta * rejected_logratio + ch_r.append(cr) + rj_r.append(rj) + if cr > rj: + correct += 1 + return { + "loss": sum(losses) / n, + "chosen_reward": sum(ch_r) / n, + "rejected_reward": sum(rj_r) / n, + "margin": sum(c - r for c, r in zip(ch_r, rj_r)) / n, + "accuracy": correct / n, + "n": n, + } + + +# --------------------------------------------------------------- GRPO + +def group_normalized_advantages(rewards, eps=1e-6): + """GRPO advantage = (r - mean(r)) / (std(r) + eps) over a sampled GROUP of + rollouts for one prompt (population std). A group whose rewards are all equal + yields all-zero advantages (std 0 -> divide by eps -> ~0).""" + rs = [float(r) for r in rewards] + if not rs: + raise ValueError("need at least one reward") + mu = statistics.fmean(rs) + sd = statistics.pstdev(rs) if len(rs) > 1 else 0.0 + return [(r - mu) / (sd + eps) for r in rs] + + +def grpo_clipped_objective(ratios, advantages, clip_eps=0.2): + """PPO/GRPO clipped surrogate to MAXIMIZE (the training loss is its + negative). Per sample: min(ratio*A, clip(ratio, 1-eps, 1+eps)*A); returns the + mean. ratio = pi_new/pi_old. The min correctly handles both advantage signs + (it caps the gain on positive A and does not reward moving further on the + clipped side for negative A).""" + if len(ratios) != len(advantages): + raise ValueError("ratios and advantages must be the same length") + if not ratios: + raise ValueError("need at least one sample") + lo, hi = 1.0 - clip_eps, 1.0 + clip_eps + terms = [] + for r, a in zip(ratios, advantages): + clipped = min(max(r, lo), hi) + terms.append(min(r * a, clipped * a)) + return sum(terms) / len(terms) + + +def kl_penalty_k3(logp_pol, logp_ref): + """Schulman's low-variance, unbiased k3 KL estimator, mean over tokens: + KL ~= mean( exp(logp_ref - logp_pol) - (logp_ref - logp_pol) - 1 ). Always + >= 0; exactly 0 when the policy equals the reference. Used as the GRPO KL + regularizer toward the frozen reference.""" + if len(logp_pol) != len(logp_ref): + raise ValueError("logp_pol and logp_ref must be the same length") + if not logp_pol: + raise ValueError("need at least one token") + out = [] + for lp, lr in zip(logp_pol, logp_ref): + d = lr - lp + out.append(math.exp(d) - d - 1.0) + return sum(out) / len(out) + + +def grpo_loss(ratios, advantages, logp_pol, logp_ref, clip_eps=0.2, beta_kl=0.0): + """The ASSEMBLED GRPO loss to MINIMIZE: the NEGATIVE clipped surrogate (the + surrogate is a reward to MAXIMIZE) plus a positive KL penalty toward the + frozen reference. A naive `minimize(grpo_clipped_objective)` ascends the WRONG + direction and wastes the whole RL run — this helper fixes the sign in one + place. Returns {loss, surrogate, kl}.""" + surrogate = grpo_clipped_objective(ratios, advantages, clip_eps) + kl = kl_penalty_k3(logp_pol, logp_ref) if beta_kl else 0.0 + return {"loss": -surrogate + beta_kl * kl, "surrogate": surrogate, "kl": kl} + + +# --------------------------------------------------------------- SFT masking + +def masked_sft_nll(per_token_logp, response_mask, smoothing=0.0, vocab_size=None): + """SFT loss averaged over RESPONSE tokens only. `per_token_logp[i]` is the + model's log-prob of the gold token at position i; `response_mask[i]` in {0,1} + selects completion tokens. Training SFT on PROMPT tokens is objective + misspecification, so masking is mandatory — `label_smoothed_nll` takes a + single scalar and delegates the masking to this caller. `vocab_size` is + required when smoothing > 0.""" + if len(per_token_logp) != len(response_mask): + raise ValueError("per_token_logp and response_mask must be the same length") + if smoothing and vocab_size is None: + raise ValueError("vocab_size required when smoothing > 0") + num, den = 0.0, 0 + for lp, m in zip(per_token_logp, response_mask): + if not m: + continue + num += label_smoothed_nll(lp, vocab_size, smoothing) if smoothing else (-lp) + den += 1 + if den == 0: + raise ValueError("response_mask selects zero tokens") + return num / den diff --git a/research/recovery/.gitignore b/research/recovery/.gitignore new file mode 100644 index 0000000..691ff4b --- /dev/null +++ b/research/recovery/.gitignore @@ -0,0 +1,3 @@ +# Recovery-run logs are runtime forensics, not source. Keep the dir, ignore contents. +* +!.gitignore diff --git a/research/render_md.py b/research/render_md.py new file mode 100644 index 0000000..91e77c8 --- /dev/null +++ b/research/render_md.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Render a Markdown file to a styled, self-contained HTML page. + +Handles: fenced/inline code (Pygments), GFM tables, a sticky TOC sidebar, and +LaTeX math via MathJax -- WITHOUT mis-rendering prose dollar amounts ($30M etc.). +Only genuine math spans are converted to \\(...\\) / \\[...\\] delimiters; MathJax +is configured to NOT scan bare `$`, so "$30M" stays literal. + +Usage: render_md.py "" "<subtitle>" +""" +import re +import sys +import html +from pygments.formatters import HtmlFormatter + +import markdown + + +def protect(text, pattern, store, tag, flags=0): + """Replace each match of `pattern` with an inert token, stashing the raw text.""" + def repl(m): + store.append(m.group(0)) + return f"zz{tag}{len(store)-1}zz" + return re.sub(pattern, repl, text, flags=flags) + + +def is_math(content): + """Heuristic: is this $...$ span genuine LaTeX math vs a prose dollar amount?""" + c = content.strip() + if not c: + return False + # currency / numeric amount like 30M, 1.5B, 2-4/GPU-hr -> NOT math + if re.fullmatch(r"[\d.,]+\s*[-–]?\s*[\d.,]*\s*(?:[MBKk]|million|billion|trillion)?(?:/[\w-]+)?", c): + return False + # genuine math indicators, or a short symbol token + if re.search(r"[\\^_{}]", c): + return True + if len(c) <= 12: + return True + return False + + +def extract_math(text): + """Pull display ($$..$$) then inline ($..$) math into \\[..\\] / \\(..\\) tokens.""" + disp, inl = [], [] + + def disp_repl(m): + disp.append(m.group(1).strip()) + return f"zzDMATHzz{len(disp)-1}zz" + text = re.sub(r"\$\$(.+?)\$\$", disp_repl, text, flags=re.DOTALL) + + # paired inline $...$ with no leading/trailing space and no $ or newline inside + def inl_repl(m): + content = m.group(1) + if not is_math(content): + return m.group(0) # leave prose dollars untouched + inl.append(content) + return f"zzIMATHzz{len(inl)-1}zz" + text = re.sub(r"\$(?=\S)([^$\n]{1,160}?)(?<=\S)\$", inl_repl, text) + return text, disp, inl + + +def main(): + src, out, title, subtitle = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] + raw = open(src, encoding="utf-8").read() + + # 1. protect code so dollars inside it are never treated as math + code_store = [] + raw = protect(raw, r"```.*?```", code_store, "CODE", flags=re.DOTALL) + raw = protect(raw, r"`[^`\n]+`", code_store, "ICODE") + + # 2. extract genuine math + raw, disp, inl = extract_math(raw) + + # 3. restore code blocks so markdown renders them + for i, block in enumerate(code_store): + raw = raw.replace(f"zzCODE{i}zz", block).replace(f"zzICODE{i}zz", block) + + md = markdown.Markdown(extensions=["extra", "codehilite", "toc", "sane_lists", "smarty"], + extension_configs={"codehilite": {"guess_lang": False}, + "toc": {"permalink": "#"}}) + body = md.convert(raw) + toc = md.toc + + # 4. restore math as MathJax delimiters + for i, m in enumerate(disp): + body = body.replace(f"zzDMATHzz{i}zz", "\\[" + m + "\\]") + for i, m in enumerate(inl): + body = body.replace(f"zzIMATHzz{i}zz", "\\(" + m + "\\)") + + pyg = HtmlFormatter(style="monokai").get_style_defs(".codehilite") + page = TEMPLATE.format(title=html.escape(title), subtitle=html.escape(subtitle), + toc=toc, body=body, pygments=pyg) + open(out, "w", encoding="utf-8").write(page) + # sanity: math delimiter balance + print(f"wrote {out} | display-math={len(disp)} inline-math={len(inl)} " + f"| balance \\[={body.count(chr(92)+'[')} \\]={body.count(chr(92)+']')} " + f"\\(={body.count(chr(92)+'(')} \\)={body.count(chr(92)+')')}") + + +TEMPLATE = r"""<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>{title} + + + + + +
+ +
+

{title}

{subtitle}
+ {body} +
+
+ +""" + + +if __name__ == "__main__": + main() diff --git a/research/scaling_ladder.py b/research/scaling_ladder.py new file mode 100644 index 0000000..743f8de --- /dev/null +++ b/research/scaling_ladder.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""research/scaling_ladder.py — infra for the SCALING-LADDER experiment (the #1 RS +lift): does an optimizer/architecture edge measured at small budget SURVIVE at +scale, or is it just an early-training speedup that converges away? + +Two halves, both pure stdlib (no numpy/scipy, no torch, no I/O): + + 1. PLAN — `build_ladder(...)`. Given a base config, a list of token budgets + (e.g. [42e6, 170e6, 670e6, 1190e6]), a tokens-per-step, and a seed schedule + (FEWER seeds allowed at larger budgets, where compute is dear), emit the + per-cell run specs for BOTH arms (e.g. NorMuon vs AdamW): budget -> steps + (via tok_per_step), per-cell seeds, tags, and a deterministic run_id. This + is the exact list of training jobs the next run launches. + + 2. ANALYZE — `fit_gap_trend(...)`. Given the per-budget BPB gaps measured after + the runs complete (treatment minus baseline, in the metric's own units), do + a simple ordinary-least-squares fit of gap vs log10(tokens). The SIGN of the + slope is the verdict: + slope ~ 0 and gap stays > floor -> PERSISTS (edge survives at scale) + slope > 0 (gap shrinks toward 0) -> CONVERGES (early-training speedup) + slope < 0 (gap grows) -> WIDENS (edge compounds) + "shrinks toward 0" is defined relative to the SIGN of the edge: a positive + edge (treatment better, gap measured as improvement>0) converges when the + fitted line trends toward 0; the code handles either edge direction. + +Honesty (mirrors §C16 / §C17): + * The verdict is DESCRIPTIVE with < 3 budget points (a 2-point "trend" is a + line through 2 dots, not evidence) — `descriptive_only` is set and the + verdict is suffixed accordingly. + * A trend is only declared when the slope is resolvable above the measurement + noise: the caller passes a `gap_noise` (the per-cell BPB noise floor from the + seed-variance gate, research/eval_stats.py). If the fitted slope's effect over + the budget range is within that floor, the verdict is FLAT/INCONCLUSIVE — we + do NOT crown "persists" or "converges" off slope noise. + +Everything is unit-tested directly on synthetic gaps (test_scaling_ladder.py) +without launching a single run. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from pathlib import Path + +# ----- gap-direction convention ------------------------------------------------- +# Gaps are stored as IMPROVEMENT of treatment over baseline in the metric's own +# direction-corrected units: gap > 0 means the treatment arm is BETTER. For BPB +# (lower is better) the caller computes gap = baseline_bpb - treatment_bpb. +# "Converges away" therefore always means "the gap moves toward 0". + + +# ===== PART 1: PLAN — emit per-cell run specs ================================== + +def _run_id(base_name: str, arm: str, tokens: float, seed: int) -> str: + """Deterministic, collision-resistant run id for a single ladder cell.""" + tok_tag = _human_tokens(tokens) + raw = f"{base_name}|{arm}|{int(round(tokens))}|{seed}" + h = hashlib.sha1(raw.encode()).hexdigest()[:6] + return f"{base_name}_{arm}_{tok_tag}_s{seed}_{h}" + + +def _human_tokens(tokens: float) -> str: + """42e6 -> '42M', 1.19e9 -> '1190M' (kept in M so the ladder reads uniformly).""" + millions = tokens / 1e6 + if millions >= 1000: + # keep whole-million resolution; 1190M, not 1.19B, so cells sort lexically + return f"{int(round(millions))}M" + if abs(millions - round(millions)) < 1e-6: + return f"{int(round(millions))}M" + return f"{millions:.1f}M" + + +def steps_for_budget(tokens: float, tok_per_step: float) -> int: + """tokens / tok_per_step, rounded UP (a partial final step still runs). + tok_per_step = global_batch_tokens = micro_batch * grad_accum * seq_len * dp.""" + if tokens <= 0: + raise ValueError(f"tokens must be > 0, got {tokens}") + if tok_per_step <= 0: + raise ValueError(f"tok_per_step must be > 0, got {tok_per_step}") + return int(math.ceil(tokens / tok_per_step)) + + +def _normalize_seed_schedule(token_budgets, seeds_per_budget): + """Resolve seeds_per_budget into a per-budget seed COUNT list. + + Accepts: + * an int -> same count at every budget; + * a list/tuple -> per-budget counts (len must match token_budgets); + * a dict {tokens: count} -> explicit count keyed by budget (matched by + nearest float key so 42e6 and 42000000.0 are the same cell). + Fewer seeds at larger budgets is the intended use (cost). Every count must be + >= 1; a budget with 0 seeds is a config error (it would silently drop a rung). + """ + n = len(token_budgets) + if isinstance(seeds_per_budget, int): + counts = [seeds_per_budget] * n + elif isinstance(seeds_per_budget, dict): + counts = [] + for b in token_budgets: + match = None + for k, v in seeds_per_budget.items(): + if math.isclose(float(k), float(b), rel_tol=1e-9): + match = v + break + if match is None: + raise ValueError(f"seeds_per_budget dict has no entry for budget {b}") + counts.append(match) + else: + counts = list(seeds_per_budget) + if len(counts) != n: + raise ValueError( + f"seeds_per_budget list length {len(counts)} != " + f"number of budgets {n}") + for b, c in zip(token_budgets, counts): + if int(c) < 1: + raise ValueError(f"budget {b} has < 1 seed ({c}) — would drop the rung") + return [int(c) for c in counts] + + +def build_ladder(base_config: dict, + token_budgets, + tok_per_step: float, + arms=("treatment", "baseline"), + seeds_per_budget=3, + base_seed: int = 0, + base_name: str = "ladder") -> dict: + """Emit the full per-cell run-spec list for a scaling ladder. + + base_config: the shared config dict (model, data, lr schedule, ...). Each cell + spec carries a COPY with `train_tokens` and `max_steps` set for that rung, + plus an `arm` key — the launcher applies the arm's minimal diff. + token_budgets: iterable of token counts (e.g. [42e6, 170e6, 670e6, 1190e6]). + tok_per_step: tokens consumed per optimizer step (global batch tokens). + arms: the two (or more) arms to compare, e.g. ("normuon", "adamw"). + seeds_per_budget: int | list | {tokens: count} — fewer seeds at larger budgets. + base_seed: seeds are base_seed, base_seed+1, ... per cell (shared ACROSS arms + at the same budget so the seed is a paired/blocking factor, not noise). + + Returns a dict with `cells` (flat list of run specs) and `summary` (counts + + the seed schedule), so the caller can both launch and cost the ladder. + """ + budgets = [float(b) for b in token_budgets] + if not budgets: + raise ValueError("token_budgets is empty") + if len(set(arms)) < 2: + raise ValueError("need at least 2 distinct arms to measure a gap") + seed_counts = _normalize_seed_schedule(budgets, seeds_per_budget) + + cells = [] + for tokens, n_seeds in zip(budgets, seed_counts): + steps = steps_for_budget(tokens, tok_per_step) + seeds = [base_seed + i for i in range(n_seeds)] + for seed in seeds: # seed is the OUTER paired factor + for arm in arms: + cfg = dict(base_config) + cfg["arm"] = arm + cfg["train_tokens"] = tokens + cfg["max_steps"] = steps + cfg["seed"] = seed + cells.append({ + "run_id": _run_id(base_name, arm, tokens, seed), + "arm": arm, + "train_tokens": tokens, + "tokens_human": _human_tokens(tokens), + "max_steps": steps, + "tok_per_step": tok_per_step, + "seed": seed, + "tags": [f"ladder:{base_name}", f"arm:{arm}", + f"budget:{_human_tokens(tokens)}", f"seed:{seed}", + "experiment:scaling-ladder"], + "config": cfg, + }) + + total_tokens = sum(c["train_tokens"] for c in cells) + return { + "base_name": base_name, + "arms": list(arms), + "tok_per_step": tok_per_step, + "budgets": budgets, + "seed_schedule": dict(zip((_human_tokens(b) for b in budgets), seed_counts)), + "cells": cells, + "summary": { + "n_cells": len(cells), + "n_budgets": len(budgets), + "n_arms": len(arms), + "total_train_tokens": total_tokens, + "cells_per_budget": { + _human_tokens(b): c * len(arms) + for b, c in zip(budgets, seed_counts) + }, + }, + } + + +# ===== PART 2: ANALYZE — fit the gap-vs-log(tokens) trend ===================== + +def _ols_line(xs, ys): + """Ordinary-least-squares slope+intercept for y = slope*x + intercept. + Pure stdlib. Returns (slope, intercept, r2). Requires >= 2 distinct x.""" + n = len(xs) + mx = sum(xs) / n + my = sum(ys) / n + sxx = sum((x - mx) ** 2 for x in xs) + if sxx == 0: + raise ValueError("all token budgets are identical — no trend to fit") + sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) + slope = sxy / sxx + intercept = my - slope * mx + # R^2 (coefficient of determination); syy==0 (flat gaps) -> r2 defined as 1.0 + syy = sum((y - my) ** 2 for y in ys) + if syy == 0: + r2 = 1.0 + else: + ss_res = sum((y - (slope * x + intercept)) ** 2 for x, y in zip(xs, ys)) + r2 = 1.0 - ss_res / syy + return slope, intercept, r2 + + +def fit_gap_trend(token_budgets, gaps, gap_noise: float = 0.0) -> dict: + """Fit per-budget gaps (treatment improvement over baseline, gap>0 == better) + against log10(tokens) and return the trend verdict. + + token_budgets: the budgets at which gaps were measured (one per gap). + gaps: the measured BPB gaps (baseline_bpb - treatment_bpb per budget). + gap_noise: per-cell measurement noise floor on a gap (from the seed-variance + gate, eval_stats.seed_delta_significant). The slope is only acted on when + its modeled effect across the budget range EXCEEDS this floor; otherwise + the trend is FLAT/inconclusive (we never crown a verdict off slope noise). + + Verdict (string in `verdict`): + PERSISTS — edge present (|gap| at the top rung > noise) and slope flat + (edge does NOT trend toward 0): the advantage survives. + WIDENS — edge magnitude grows with scale (slope pushes |gap| up). + CONVERGES — edge shrinks toward 0 with scale (slope pulls |gap| down past + the noise floor at the largest budget): early-training speedup + that washes out — the honest "no real advantage at scale". + FLAT — slope effect within noise AND no resolvable edge: inconclusive. + """ + budgets = [float(b) for b in token_budgets] + gp = [float(g) for g in gaps] + if len(budgets) != len(gp): + raise ValueError(f"len(token_budgets)={len(budgets)} != len(gaps)={len(gp)}") + if len(budgets) < 2: + raise ValueError("need >= 2 budget points to fit a trend") + if any(b <= 0 for b in budgets): + raise ValueError("token budgets must be > 0 (log10 taken)") + if any(not math.isfinite(g) for g in gp): + raise ValueError("non-finite gap (NaN/inf) — a diverged run?") + if gap_noise < 0: + raise ValueError("gap_noise must be >= 0") + + xs = [math.log10(b) for b in budgets] + slope, intercept, r2 = _ols_line(xs, gp) + + # Order by budget so "first"/"last" rung are the true extremes. + order = sorted(range(len(budgets)), key=lambda i: budgets[i]) + x_lo, x_hi = xs[order[0]], xs[order[-1]] + # fitted gap at the smallest and largest budget on the regression line + gap_lo_fit = slope * x_lo + intercept + gap_hi_fit = slope * x_hi + intercept + # observed edge sign uses the MEAN gap (robust to a single noisy rung) + mean_gap = sum(gp) / len(gp) + edge_sign = 1 if mean_gap > 0 else (-1 if mean_gap < 0 else 0) + + # Modeled change in gap across the whole budget span (the load-bearing number). + span_effect = gap_hi_fit - gap_lo_fit + # Is there a real edge at the LARGEST budget (where it matters for "at scale")? + edge_at_top = abs(gap_hi_fit) + edge_resolved = edge_at_top > gap_noise + # Is the slope's effect resolvable above per-cell noise? + slope_resolved = abs(span_effect) > gap_noise + + descriptive_only = len(budgets) < 3 + + # Decide direction of the slope RELATIVE to the edge sign: + # span_effect * edge_sign > 0 -> |gap| growing in the edge's direction (WIDENS) + # span_effect * edge_sign < 0 -> |gap| shrinking toward 0 (CONVERGES) + toward_zero = (edge_sign != 0) and (span_effect * edge_sign < 0) + + if not slope_resolved: + # slope is within noise: edge is flat across scale + if edge_resolved: + verdict = "PERSISTS" + rationale = ("gap is flat vs log(tokens) (slope effect within noise) " + "and the edge at the largest budget exceeds the noise " + "floor — the advantage survives at scale") + else: + verdict = "FLAT" + rationale = ("gap is flat vs log(tokens) and within the noise floor at " + "the largest budget — no resolvable advantage either way") + else: + if toward_zero: + # gap shrinking toward 0: does it land inside the noise floor at top? + if not edge_resolved: + verdict = "CONVERGES" + rationale = ("gap shrinks toward 0 with scale and falls within the " + "noise floor at the largest budget — an early-training " + "speedup that converges away (no advantage at scale)") + else: + verdict = "CONVERGES" + rationale = ("gap shrinks toward 0 with scale; still above noise at " + "the largest measured budget but trending out — the " + "edge is eroding, extend the ladder before claiming it") + else: + verdict = "WIDENS" + rationale = ("gap grows with scale (slope amplifies the edge beyond the " + "noise floor) — the advantage compounds at scale") + + if descriptive_only: + verdict_full = verdict + " (DESCRIPTIVE: <3 budgets — extend the ladder)" + else: + verdict_full = verdict + + return { + "verdict": verdict_full, + "verdict_code": verdict, + "slope": slope, + "intercept": intercept, + "r2": r2, + "mean_gap": mean_gap, + "edge_sign": edge_sign, + "gap_lo_fit": gap_lo_fit, + "gap_hi_fit": gap_hi_fit, + "span_effect": span_effect, + "edge_at_top": edge_at_top, + "edge_resolved": edge_resolved, + "slope_resolved": slope_resolved, + "toward_zero": toward_zero, + "gap_noise": gap_noise, + "n_budgets": len(budgets), + "descriptive_only": descriptive_only, + "rationale": rationale, + "fit": "gap = slope * log10(tokens) + intercept (OLS)", + } + + +# ===== CLI ==================================================================== + +def main(argv=None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + p_plan = sub.add_parser("plan", help="emit per-cell run specs (JSON)") + p_plan.add_argument("--config", type=Path, required=True, + help="JSON base config dict") + p_plan.add_argument("--budgets", type=float, nargs="+", required=True, + help="token budgets, e.g. 42e6 170e6 670e6 1190e6") + p_plan.add_argument("--tok-per-step", type=float, required=True) + p_plan.add_argument("--arms", nargs="+", default=["treatment", "baseline"]) + p_plan.add_argument("--seeds", type=int, nargs="+", default=[3], + help="one int (all budgets) or one per budget") + p_plan.add_argument("--name", default="ladder") + + p_fit = sub.add_parser("fit", help="fit gap-vs-log(tokens) trend (JSON)") + p_fit.add_argument("--budgets", type=float, nargs="+", required=True) + p_fit.add_argument("--gaps", type=float, nargs="+", required=True) + p_fit.add_argument("--gap-noise", type=float, default=0.0) + + a = ap.parse_args(argv) + if a.cmd == "plan": + cfg = json.loads(a.config.read_text()) + seeds = a.seeds[0] if len(a.seeds) == 1 else a.seeds + out = build_ladder(cfg, a.budgets, a.tok_per_step, arms=tuple(a.arms), + seeds_per_budget=seeds, base_name=a.name) + else: + out = fit_gap_trend(a.budgets, a.gaps, gap_noise=a.gap_noise) + print(json.dumps(out, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/tests/test_ablation_config.py b/research/tests/test_ablation_config.py new file mode 100644 index 0000000..53a7511 --- /dev/null +++ b/research/tests/test_ablation_config.py @@ -0,0 +1,145 @@ +"""Tests for research/ablation_config.py — the validated multi-seed, per-arm-tuned +cohort builder that makes the next ablation a 'clean win'. Pure, deterministic, +stdlib-only.""" +import pytest + +import ablation_config as ac +from data_decontam import cell_split_seed + + +def _good_arms(): + return [ + ac.Arm("baseline", hparams={"lr": 3e-4, "wd": 0.1}, is_baseline=True), + ac.Arm("treatment", hparams={"lr": 4e-4, "wd": 0.05}), # PER-ARM tuned + ] + + +# --- Arm validation ---------------------------------------------------------- + +def test_arm_requires_per_arm_hparams(): + with pytest.raises(ValueError): + ac.Arm("t", hparams={}, is_baseline=False).validate() # empty == shared confound + + +def test_arm_rejects_non_finite_hparam(): + with pytest.raises(ValueError): + ac.Arm("t", hparams={"lr": float("nan")}).validate() + with pytest.raises(ValueError): + ac.Arm("t", hparams={"lr": float("inf")}).validate() + + +def test_arm_accepts_numbers_bools_strings(): + ac.Arm("t", hparams={"lr": 3e-4, "use_muon": True, "sched": "cosine"}).validate() + + +# --- n_seeds gate ------------------------------------------------------------ + +def test_validate_refuses_fewer_than_three_seeds(): + with pytest.raises(ValueError): + ac.build_cohort(_good_arms(), n_seeds=2) + with pytest.raises(ValueError): + ac.build_cohort(_good_arms(), n_seeds=1) + + +def test_three_seeds_builds_but_is_scoped_not_clean(): + cfg = ac.build_cohort(_good_arms(), n_seeds=3, randomized_split=True) + assert cfg.verdict_tier() == "scoped" # >=3 builds, but <5 -> not clean + assert cfg.is_clean_tier is False + + +def test_clean_tier_requires_five_seeds_and_randomized_split(): + clean = ac.build_cohort(_good_arms(), n_seeds=5, randomized_split=True) + assert clean.is_clean_tier is True and clean.verdict_tier() == "clean" + # 5 seeds but FIXED split -> only scoped (init+shuffle variance only) + fixed = ac.build_cohort(_good_arms(), n_seeds=5, randomized_split=False) + assert fixed.is_clean_tier is False and fixed.verdict_tier() == "scoped" + + +# --- arm/baseline structural validation ------------------------------------- + +def test_requires_exactly_one_baseline(): + two_base = [ac.Arm("a", {"lr": 1e-4}, is_baseline=True), + ac.Arm("b", {"lr": 1e-4}, is_baseline=True)] + with pytest.raises(ValueError): + ac.build_cohort(two_base, n_seeds=5) + no_base = [ac.Arm("a", {"lr": 1e-4}), ac.Arm("b", {"lr": 1e-4})] + with pytest.raises(ValueError): + ac.build_cohort(no_base, n_seeds=5) + + +def test_requires_at_least_two_arms(): + with pytest.raises(ValueError): + ac.build_cohort([ac.Arm("solo", {"lr": 1e-4}, is_baseline=True)], n_seeds=5) + + +def test_rejects_duplicate_arm_names(): + arms = [ac.Arm("x", {"lr": 1e-4}, is_baseline=True), ac.Arm("x", {"lr": 2e-4})] + with pytest.raises(ValueError): + ac.build_cohort(arms, n_seeds=5) + + +def test_rejects_bad_val_fraction_and_direction(): + with pytest.raises(ValueError): + ac.build_cohort(_good_arms(), n_seeds=5, val_fraction=0.0) + with pytest.raises(ValueError): + ac.build_cohort(_good_arms(), n_seeds=5, direction="bigger") + + +# --- per-arm hparams flow through to cells ---------------------------------- + +def test_cells_carry_per_arm_hparams(): + cfg = ac.build_cohort(_good_arms(), n_seeds=5) + cells = cfg.cells() + assert len(cells) == 2 * 5 # arms x seeds + base_cells = [c for c in cells if c["arm"] == "baseline"] + treat_cells = [c for c in cells if c["arm"] == "treatment"] + assert all(c["hparams"] == {"lr": 3e-4, "wd": 0.1} for c in base_cells) + assert all(c["hparams"] == {"lr": 4e-4, "wd": 0.05} for c in treat_cells) + + +# --- split-seed wiring matches data_decontam -------------------------------- + +def test_fixed_split_every_cell_same_split_seed(): + cfg = ac.build_cohort(_good_arms(), n_seeds=5, base_split_seed=11, + randomized_split=False) + seeds = {c["split_seed"] for c in cfg.cells()} + assert seeds == {11} # fixed: one split for the whole cohort + + +def test_randomized_split_distinct_split_seed_per_train_seed(): + cfg = ac.build_cohort(_good_arms(), n_seeds=5, base_split_seed=11, + randomized_split=True) + cells = cfg.cells() + # the split seed each cell uses must match data_decontam's derivation exactly + for c in cells: + assert c["split_seed"] == cell_split_seed(11, c["train_seed"], randomized=True) + # across the 5 distinct train seeds there are 5 distinct split seeds + per_seed = {c["train_seed"]: c["split_seed"] for c in cells} + assert len(set(per_seed.values())) == 5 + # both arms at the SAME train seed share the SAME split (paired comparison) + by_seed = {} + for c in cells: + by_seed.setdefault(c["train_seed"], set()).add(c["split_seed"]) + assert all(len(v) == 1 for v in by_seed.values()) + + +def test_seed_start_shifts_the_seed_grid(): + cfg = ac.build_cohort(_good_arms(), n_seeds=5, seed_start=100) + assert cfg.seeds() == [100, 101, 102, 103, 104] + + +# --- serialization / dict-arm convenience ----------------------------------- + +def test_build_from_dict_arms_and_to_dict(): + arms = [{"name": "base", "hparams": {"lr": 3e-4}, "is_baseline": True}, + {"name": "treat", "hparams": {"lr": 4e-4}}] + cfg = ac.build_cohort(arms, n_seeds=5, randomized_split=True, notes="run-42") + d = cfg.to_dict() + assert d["verdict_tier"] == "clean" and d["n_cells"] == 10 + assert d["is_clean_tier"] is True and d["notes"] == "run-42" + + +def test_baseline_and_treatment_accessors(): + cfg = ac.build_cohort(_good_arms(), n_seeds=5) + assert cfg.baseline_arm().name == "baseline" + assert [a.name for a in cfg.treatment_arms()] == ["treatment"] diff --git a/research/tests/test_boot_resume.py b/research/tests/test_boot_resume.py new file mode 100644 index 0000000..71cd8fe --- /dev/null +++ b/research/tests/test_boot_resume.py @@ -0,0 +1,243 @@ +"""Guard-logic tests for research/boot_resume.sh — the generic @reboot cross-reboot +recovery hook. Drives its `--dry-run` decision path over crafted loop_state.json states +with BFS_ROOT redirected to a fixture, so the safety-critical resume/refuse decisions are +execution-verified WITHOUT a GPU or a real trainer (§C22, §C4/§C5/§C6 recovery). + +Covers the guards that gate an unattended relaunch on a box that hard-locks: + - nothing to resume (no in-flight run recorded) + - a run to resume, all clear -> would-resume + - sentinel-kill marker present -> refuse (a kill is not auto-resumable at same config) + - auto-resume cap reached -> refuse (never loop-crash the box) + - preflight failing -> do not launch (exit 1) +""" +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent # BuildFromScratch/ +BOOT = REPO / "research" / "boot_resume.sh" +REAL_LOOP_STATE = REPO / "research" / "loop_state.py" + +SENTINEL_OK = "import sys; sys.exit(0)\n" +SENTINEL_FAIL = ( + "import sys\n" + "sys.exit(1 if 'preflight' in sys.argv else 0)\n" +) + + +def _fixture(tmp_path, state: dict | None, *, preflight_ok=True, marker=False): + """Build a minimal BFS_ROOT: research/loop_state.py (real), loop_state.json (crafted), + a stub sentinel.py, and research/recovery/. Returns the root path.""" + root = tmp_path / "root" + (root / "research" / "recovery").mkdir(parents=True) + shutil.copy(REAL_LOOP_STATE, root / "research" / "loop_state.py") + (root / "sentinel.py").write_text(SENTINEL_OK if preflight_ok else SENTINEL_FAIL) + statef = root / "research" / "loop_state.json" + if state is not None: + statef.write_text(json.dumps(state, indent=2)) + if marker: + (root / "research" / "loop_state.json.sentinel_kill").write_text("{}") + return root + + +# Hermetic trainer pattern (boot_resume.sh's documented BOOT_TRAINER_RE test hook): +# matches no real process, so the §C4.5 concurrency gate cannot see a trainer that +# happens to be running on the box. Without this the default whitelist +# (`train_*.py|run_arms.sh|run_ladder.sh`) pgreps the REAL host, and every test whose +# decision lies past that gate silently degrades to `already-running` — i.e. the +# recovery-chain guards went unverified exactly when a trainer was live, which is the +# only time the recovery chain matters. (Caught 2026-07-20 beside a live train_hybrid.py; +# the later tests in this file already pass their own BOOT_TRAINER_RE.) +NO_TRAINER_RE = "__bfs_hermetic_no_such_trainer__" + + +def _run(root): + r = subprocess.run( + ["bash", str(BOOT), "--dry-run"], + env={**os.environ, "BFS_ROOT": str(root), + "BOOT_TRAINER_RE": NO_TRAINER_RE}, + capture_output=True, text=True, timeout=60, + ) + # DECISION= is printed to stdout; log lines go to stderr. + line = next((ln for ln in r.stdout.splitlines() if ln.startswith("DECISION=")), "") + return line.replace("DECISION=", "").strip(), r.returncode + + +def _live_state(**over): + st = {"schema_version": 1, "iteration_date": None, "stage": "S7", + "in_flight_run": "2026-07-19_qwen3_x", "train_pid": 4242, + "ckpt_path": "/x/ckpt.pt", "resume_cmd": "python3 train_ablation.py --resume /x/ckpt.pt", + "auto_resumes": 0, "last_radar": None, "objective": "any", "notes": "", "updated": None} + st.update(over) + return st + + +def test_no_in_flight_run_is_nothing_to_resume(tmp_path): + root = _fixture(tmp_path, _live_state(in_flight_run=None, resume_cmd=None)) + decision, rc = _run(root) + assert decision == "nothing-to-resume" and rc == 0 + + +def test_missing_state_file_is_nothing_to_resume(tmp_path): + root = _fixture(tmp_path, None) # no loop_state.json at all (fail-open) + decision, rc = _run(root) + assert decision == "nothing-to-resume" and rc == 0 + + +def test_clear_in_flight_run_would_resume(tmp_path): + root = _fixture(tmp_path, _live_state()) + decision, rc = _run(root) + assert decision == "would-resume" and rc == 0 + + +def test_sentinel_kill_marker_refuses(tmp_path): + root = _fixture(tmp_path, _live_state(), marker=True) + decision, rc = _run(root) + assert decision == "refuse-sentinel-kill" and rc == 0 + + +def test_auto_resume_cap_refuses(tmp_path): + root = _fixture(tmp_path, _live_state(auto_resumes=2)) + decision, rc = _run(root) + assert decision == "cap-reached" and rc == 0 + + +def test_preflight_failure_does_not_launch(tmp_path): + root = _fixture(tmp_path, _live_state(), preflight_ok=False) + decision, rc = _run(root) + assert decision == "preflight-fail" and rc == 1 + + +def test_guard_tests_are_hermetic_against_a_live_host_trainer(tmp_path): + """Regression lock for the isolation fix above: a REAL trainer-shaped process on the + box must not change any decision reached through `_run()`. Before the fix this test's + scenario silently turned `would-resume` into `already-running`, so the whole + recovery-guard suite went green-by-accident on an idle box and red during training.""" + tag = "h" + _uuid.uuid4().hex[:8] + probe = tmp_path / f"train_{tag}.py" # matches the DEFAULT whitelist + probe.write_text("import time; time.sleep(20)\n") + proc = subprocess.Popen(["python3", str(probe), "20"]) + try: + _time.sleep(1) # let it appear in the process table + assert subprocess.run(["pgrep", "-f", f"train_{tag}.py"], + capture_output=True).returncode == 0, "probe never started" + root = _fixture(tmp_path, _live_state()) + decision, rc = _run(root) # must be blind to the host process + assert (decision, rc) == ("would-resume", 0), decision + finally: + proc.kill() + proc.wait() + + +def test_dry_run_has_no_side_effects(tmp_path): + """--dry-run must not touch auto_resumes (no record-resume) or spawn anything.""" + root = _fixture(tmp_path, _live_state(auto_resumes=0)) + _run(root) + st = json.loads((root / "research" / "loop_state.json").read_text()) + assert st["auto_resumes"] == 0 # cap counter untouched by a dry run + + +# --- real-resume path tests (hermetic: SETTLE=0, own lock, a unique fake trainer) --------- +# Each uses a per-test-unique probe pattern so the guards can never match a real live trainer +# (run_tests.sh may run during training), and cleans up any process it spawns. +import os as _os +import signal as _signal +import time as _time +import uuid as _uuid + + +def _real_fixture(tmp_path, probe_tag, *, auto_resumes=0): + """Fixture whose resume_cmd launches a uniquely-named python sleeper, with match + patterns scoped to that unique tag so nothing else on the box can match.""" + probe = tmp_path / "root" / f"train_{probe_tag}.py" # name matches train_.py + root = _fixture(tmp_path, _live_state( + auto_resumes=auto_resumes, + resume_cmd=f"python3 {probe} 30", + )) + probe.write_text("import sys, time; time.sleep(int(sys.argv[1]) if len(sys.argv)>1 else 30)\n") + env = { + **os.environ, + "BFS_ROOT": str(root), + "BOOT_SETTLE_SECS": "0", + "BOOT_LOCK": str(root / "boot.lock"), + "BOOT_TRAINER_RE": f"train_{probe_tag}\\.py", + "BOOT_PIDCAP_RE": f"python[0-9.]* .*train_{probe_tag}\\.py", + } + return root, env, probe_tag + + +def _kill_probe(tag): + subprocess.run(["pkill", "-9", "-f", f"train_{tag}.py"], capture_output=True) + + +def _run_real(root, env): + r = subprocess.run(["bash", str(BOOT)], env=env, capture_output=True, text=True, timeout=60) + line = next((ln for ln in r.stdout.splitlines() if ln.startswith("DECISION=")), "") + return line.replace("DECISION=", "").strip(), r.returncode + + +def test_real_resume_launches_and_accounts(tmp_path): + """Full real path: relaunch resume_cmd, capture the python trainer pid, bump the cap + counter, write the new pid back — the untested branch the review flagged.""" + tag = "p" + _uuid.uuid4().hex[:8] + root, env, tag = _real_fixture(tmp_path, tag) + try: + decision, rc = _run_real(root, env) + assert decision == "resumed" and rc == 0 + st = json.loads((root / "research" / "loop_state.json").read_text()) + assert st["auto_resumes"] == 1 # record-resume accounted exactly once + assert isinstance(st["train_pid"], int) and st["train_pid"] > 0 + _os.kill(st["train_pid"], 0) # the written pid is a live process + finally: + _kill_probe(tag) + + +def test_flock_serializes_concurrent_resume(tmp_path): + """The CRITICAL fix: two simultaneous @reboot invocations (duplicate crontab lines) + must yield EXACTLY ONE resume, never two trainers (§C4.5). The flock guarantees it.""" + tag = "c" + _uuid.uuid4().hex[:8] + root, env, tag = _real_fixture(tmp_path, tag) + try: + p1 = subprocess.Popen(["bash", str(BOOT)], env=env, stdout=subprocess.PIPE, text=True) + p2 = subprocess.Popen(["bash", str(BOOT)], env=env, stdout=subprocess.PIPE, text=True) + outs = [p1.communicate(timeout=60)[0], p2.communicate(timeout=60)[0]] + decisions = sorted( + next((ln.replace("DECISION=", "").strip() + for ln in o.splitlines() if ln.startswith("DECISION=")), "") + for o in outs + ) + # exactly one resumed; the loser bounced off the lock (or the re-probe) + assert decisions.count("resumed") == 1, decisions + assert decisions[0] in ("already-locked", "already-running"), decisions + st = json.loads((root / "research" / "loop_state.json").read_text()) + assert st["auto_resumes"] == 1 # cap counter bumped once, not twice + running = subprocess.run(["pgrep", "-f", f"train_{tag}.py"], capture_output=True, text=True) + assert len([l for l in running.stdout.split() if l]) == 1 # exactly ONE trainer, not two + finally: + _kill_probe(tag) + + +def test_broad_trainer_re_guards_slug_named_trainer(tmp_path): + """The other CRITICAL fix: a running ablation-runner train_.py must trip the + already-running guard (the old static whitelist missed it -> double-launch).""" + tag = "g" + _uuid.uuid4().hex[:8] + probe = tmp_path / "root" / f"train_{tag}.py" + root = _fixture(tmp_path, _live_state(resume_cmd=f"python3 {probe} 30")) + probe.write_text("import time; time.sleep(30)\n") + proc = subprocess.Popen(["python3", str(probe), "30"]) + try: + _time.sleep(1) # let it show up in the process table + env = {**os.environ, "BFS_ROOT": str(root), + "BOOT_TRAINER_RE": f"train_{tag}\\.py"} # default whitelist WOULD also match now + r = subprocess.run(["bash", str(BOOT), "--dry-run"], env=env, capture_output=True, text=True, timeout=60) + decision = next((ln.replace("DECISION=", "").strip() + for ln in r.stdout.splitlines() if ln.startswith("DECISION=")), "") + assert decision == "already-running", r.stdout + finally: + proc.kill() + proc.wait() diff --git a/research/tests/test_cce_linear_ce.py b/research/tests/test_cce_linear_ce.py new file mode 100644 index 0000000..31fcd38 --- /dev/null +++ b/research/tests/test_cce_linear_ce.py @@ -0,0 +1,389 @@ +"""CPU unit tests for the chunked-torch CCE fused linear cross-entropy reference. + +These are the HARD correctness gate of SPEC_cce_fused_linear_ce.md, run on CPU at +tiny sizes so the whole CCE mechanism is proven WITHOUT a GPU: + + * forward loss matches the fp32 unfused oracle F.cross_entropy(H@W.T, y) (atol 1e-3) + * backward dH and dW match the oracle's autograd grads (rtol/atol 1e-2) + * the (N, V) logits tensor is NEVER materialized — enforced by an INDEPENDENT + global torch.matmul shape guard, not just the module's self-report. + * the online log-sum-exp equals a direct full-vocab logsumexp + * ignore_index rows, reduction='sum'/'none', and the eps gradient filter behave + * the eps filter is exercised in a regime where it ACTUALLY DROPS MASS (peaked, + non-zero-mean W) and a too-aggressive eps is DETECTED (so these tests are not a + rubber stamp), and a bf16-ROUNDED emulation of the Triton kernel path clears the + same 1e-2 grad gate. + +SCOPE (honest): this file gates the pure-torch reference (cce_linear_ce), which shares +the Triton kernel's math. It does NOT run the Triton kernels (needs CUDA) — their +bf16 tensor-core numerics and tile scheduling are gated off-box by +cce_triton.gate_against_reference(). CPU-green != kernel-correct. +""" +import sys +from pathlib import Path + +import pytest + +torch = pytest.importorskip("torch") +import torch.nn.functional as F # noqa: E402 + +# cce_linear_ce lives under research/kernel/, which conftest does not add to sys.path. +_KERNEL_DIR = Path(__file__).resolve().parents[1] / "kernel" +if str(_KERNEL_DIR) not in sys.path: + sys.path.insert(0, str(_KERNEL_DIR)) + +import cce_linear_ce # noqa: E402 +from cce_linear_ce import linear_cross_entropy, EPS_BF16 # noqa: E402 + + +# --------------------------------------------------------------------------- helpers +V_TINY, N_TINY, D_TINY = 512, 64, 32 +CHUNK = 128 # 4 vocab blocks over V=512 — strictly smaller than V + + +def _make_inputs(N=N_TINY, D=D_TINY, V=V_TINY, dtype=torch.float32, seed=0, scale=0.5): + g = torch.Generator().manual_seed(seed) + H = (torch.randn(N, D, generator=g, dtype=torch.float32) * scale).to(dtype) + W = (torch.randn(V, D, generator=g, dtype=torch.float32) * scale).to(dtype) + y = torch.randint(0, V, (N,), generator=g) + return H, W, y + + +def _oracle(H, W, y, ignore_index=-100, reduction="mean"): + """The HARD oracle: fp32 unfused CE. Returns (loss, dH, dW).""" + Href = H.detach().float().clone().requires_grad_(True) + Wref = W.detach().float().clone().requires_grad_(True) + logits = Href @ Wref.t() # (N, V) fp32 — the tensor CCE avoids + loss = F.cross_entropy(logits, y, ignore_index=ignore_index, reduction=reduction) + if reduction == "none": + loss.sum().backward() + else: + loss.backward() + return loss.detach(), Href.grad.detach(), Wref.grad.detach() + + +def _fused(H, W, y, chunk_size=CHUNK, ignore_index=-100, reduction="mean", + grad_filter_eps=0.0): + """Run the chunked reference in fp32 and collect (loss, dH, dW).""" + Hf = H.detach().float().clone().requires_grad_(True) + Wf = W.detach().float().clone().requires_grad_(True) + loss = linear_cross_entropy(Hf, Wf, y, chunk_size=chunk_size, + ignore_index=ignore_index, reduction=reduction, + grad_filter_eps=grad_filter_eps) + if reduction == "none": + loss.sum().backward() + else: + loss.backward() + return loss.detach(), Hf.grad.detach(), Wf.grad.detach() + + +def _fraction_softmax_below_eps(H, W, eps): + """Test-only (dense, tiny V): fraction of full-vocab softmax entries below eps. + Used to PROVE a filter test is actually in a regime where the filter drops mass.""" + logits = H.float() @ W.float().t() + p = torch.softmax(logits, dim=1) + return float((p < eps).float().mean()) + + +class _MatmulShapeGuard: + """Globally wrap torch.matmul (thread-safe: a plain attribute swap, so it intercepts + the autograd engine's backward thread too) and record every output shape. Asserts on + exit that no GEMM ever produced the forbidden (N, V) logits tensor. + + CONTRACT (pinned): the reference deliberately routes EVERY big GEMM through + torch.matmul (never the @ operator) so this independent guard can observe them. A + refactor to @ would blind the guard — but it fails LOUDLY ("saw no matmul at all") + rather than silently passing, so the contract self-enforces. This does NOT trust + cce_linear_ce.LAST_STATS.""" + + def __init__(self, N, V): + self.N, self.V = N, V + self.shapes = [] + self._orig = None + + def __enter__(self): + self._orig = torch.matmul + + def _wrapped(a, b, *args, **kwargs): + out = self._orig(a, b, *args, **kwargs) + try: + self.shapes.append(tuple(out.shape)) + except Exception: + pass + return out + + torch.matmul = _wrapped + return self + + def __exit__(self, *exc): + torch.matmul = self._orig + return False + + def assert_never_full_logits(self): + assert self.shapes, ("guard saw no matmul at all — the reference MUST use " + "torch.matmul (not the @ operator) for the guard to observe GEMMs") + forbidden = (self.N, self.V) + for shp in self.shapes: + assert shp != forbidden, f"(N, V) logits tensor {forbidden} was materialized!" + # Backstop pinned to the VOCAB axis (dim-1 spanning the full vocab), not any + # wide tensor: a 2-D GEMM output (N rows) x (>= V cols) is the full-logits + # tile. Safe against the (N, D) backward-dH product because D (1024) < V. + assert not (len(shp) == 2 and shp[0] == self.N and shp[1] >= self.V), \ + f"a full-vocab-width logits tile {shp} was formed" + + +# --------------------------------------------------------------------------- the gate + +def test_forward_loss_matches_fp32_oracle(): + H, W, y = _make_inputs() + ref_loss, _, _ = _oracle(H, W, y) + fz_loss, _, _ = _fused(H, W, y, grad_filter_eps=0.0) + torch.testing.assert_close(fz_loss, ref_loss, atol=1e-3, rtol=0) + + +def test_backward_grads_match_oracle_filter_off(): + """Filter OFF -> the reference is the exact CE; grads must match tightly.""" + H, W, y = _make_inputs() + _, ref_dH, ref_dW = _oracle(H, W, y) + _, fz_dH, fz_dW = _fused(H, W, y, grad_filter_eps=0.0) + torch.testing.assert_close(fz_dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(fz_dW, ref_dW, rtol=1e-2, atol=1e-2) + + +def test_backward_grads_match_oracle_with_eps_filter(): + """Filter ON at eps=2**-12 -> still within the loosened bf16 tolerance.""" + H, W, y = _make_inputs() + _, ref_dH, ref_dW = _oracle(H, W, y) + _, fz_dH, fz_dW = _fused(H, W, y, grad_filter_eps=EPS_BF16) + torch.testing.assert_close(fz_dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(fz_dW, ref_dW, rtol=1e-2, atol=1e-2) + + +def test_full_logits_tensor_is_never_materialized(): + """The load-bearing memory claim: run the full fwd+bwd under a global matmul + shape guard and assert (N, V) is never formed.""" + H, W, y = _make_inputs() + Hf = H.float().clone().requires_grad_(True) + Wf = W.float().clone().requires_grad_(True) + with _MatmulShapeGuard(N_TINY, V_TINY) as guard: + loss = linear_cross_entropy(Hf, Wf, y, chunk_size=CHUNK, grad_filter_eps=EPS_BF16) + loss.backward() + guard.assert_never_full_logits() + # widest vocab tile the impl self-reports must be a chunk, never the full vocab + assert cce_linear_ce.LAST_STATS["max_logits_tile_cols"] == CHUNK + assert cce_linear_ce.LAST_STATS["max_logits_tile_cols"] < V_TINY + + +def test_online_lse_equals_direct_logsumexp(): + """The streaming (running max, running sum) LSE must equal a direct full-vocab + logsumexp computed on the (small) dense logits.""" + H, W, y = _make_inputs() + H32, W32 = H.float(), W.float() + lse, zy, max_cols = cce_linear_ce._online_lse_and_target( + H32, W32, y, chunk_size=CHUNK, ignore_index=-100) + dense = H32 @ W32.t() # (N, V) — allowed here (tiny, test-only) + direct_lse = torch.logsumexp(dense, dim=1) + torch.testing.assert_close(lse, direct_lse, rtol=1e-5, atol=1e-5) + # gathered target logit equals the directly-indexed logit + direct_zy = dense[torch.arange(N_TINY), y] + torch.testing.assert_close(zy, direct_zy, rtol=1e-5, atol=1e-5) + assert max_cols == CHUNK + + +def test_loss_is_independent_of_chunk_size(): + """Correctness must not depend on the vocab block width (streaming LSE is exact).""" + H, W, y = _make_inputs() + losses = [] + for cs in (32, 128, 300, V_TINY): # incl. 300 (non-divisor of 512) and the full vocab + l, _, _ = _fused(H, W, y, chunk_size=cs, grad_filter_eps=0.0) + losses.append(float(l)) + for l in losses[1:]: + assert abs(l - losses[0]) < 1e-5, f"loss depends on chunk_size: {losses}" + + +def test_remainder_vocab_block_is_handled(): + """V not a multiple of chunk_size (V=512, chunk=300 -> blocks 300 + 212) must + still match the oracle — the remainder-mask path.""" + H, W, y = _make_inputs() + ref_loss, ref_dH, ref_dW = _oracle(H, W, y) + fz_loss, fz_dH, fz_dW = _fused(H, W, y, chunk_size=300, grad_filter_eps=0.0) + torch.testing.assert_close(fz_loss, ref_loss, atol=1e-3, rtol=0) + torch.testing.assert_close(fz_dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(fz_dW, ref_dW, rtol=1e-2, atol=1e-2) + + +def test_ignore_index_rows_dropped_like_oracle(): + """Rows labelled ignore_index contribute 0 to loss and 0 to gradient — matching + F.cross_entropy(ignore_index=...).""" + H, W, y = _make_inputs() + y = y.clone() + y[::7] = -100 # ignore every 7th row + ref_loss, ref_dH, ref_dW = _oracle(H, W, y, ignore_index=-100) + fz_loss, fz_dH, fz_dW = _fused(H, W, y, ignore_index=-100, grad_filter_eps=0.0) + torch.testing.assert_close(fz_loss, ref_loss, atol=1e-3, rtol=0) + torch.testing.assert_close(fz_dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(fz_dW, ref_dW, rtol=1e-2, atol=1e-2) + # the ignored rows have exactly-zero hidden gradient + assert torch.count_nonzero(fz_dH[::7]) == 0 + + +def test_reduction_sum_matches_oracle(): + H, W, y = _make_inputs() + ref_loss, ref_dH, ref_dW = _oracle(H, W, y, reduction="sum") + fz_loss, fz_dH, fz_dW = _fused(H, W, y, reduction="sum", grad_filter_eps=0.0) + torch.testing.assert_close(fz_loss, ref_loss, atol=1e-2, rtol=0) # sum is ~N*mean -> looser abs + torch.testing.assert_close(fz_dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(fz_dW, ref_dW, rtol=1e-2, atol=1e-2) + + +def test_reduction_none_matches_oracle_per_row(): + H, W, y = _make_inputs() + ref_loss, _, _ = _oracle(H, W, y, reduction="none") + fz_loss, _, _ = _fused(H, W, y, reduction="none", grad_filter_eps=0.0) + assert fz_loss.shape == (N_TINY,) + torch.testing.assert_close(fz_loss, ref_loss, atol=1e-3, rtol=0) + + +def test_bf16_inputs_upcast_and_match_oracle_loosely(): + """bf16 inputs (the production dtype) are upcast to fp32 internally; loss still + tracks the fp32 oracle within bf16 noise.""" + H, W, y = _make_inputs(dtype=torch.bfloat16) + ref_loss, _, _ = _oracle(H, W, y) # oracle upcasts bf16->fp32 + fz_loss, _, _ = _fused(H, W, y, grad_filter_eps=EPS_BF16) + torch.testing.assert_close(fz_loss.float(), ref_loss, atol=5e-3, rtol=0) + + +# --------------------------------------------------- eps filter: exercised WHERE IT BITES + +def test_default_eps_filter_active_and_within_tol_on_peaked_nonzero_mean_W(): + """The CCE headline trick, tested where it actually FILTERS. A peaked softmax at a + larger vocab (so many entries fall below eps) AND a NON-zero-mean W (so the dropped + mass does NOT conveniently cancel via E_p[W]~0, the trap that made the flat/zero-mean + case pass for free). Under production mean-reduction the default eps must still hold + the SPEC's 1e-2 backward gate. (The residual RELATIVE error at scale is the open + pretraining-scale A/B in Known-open — this bounds the ABSOLUTE error the gate uses.)""" + g = torch.Generator().manual_seed(11) + N, D, V = 128, 64, 4096 + H = torch.randn(N, D, generator=g) * 0.7 + W = torch.randn(V, D, generator=g) * 0.7 + 0.5 # non-zero mean shift + y = torch.randint(0, V, (N,), generator=g) + + # PROVE the filter is genuinely active here (drops a large fraction of entries), + # so this is not a no-op test that would pass with the filter disabled. + frac = _fraction_softmax_below_eps(H, W, EPS_BF16) + assert frac > 0.5, f"filter not exercised in this regime ({frac:.1%} below eps)" + + _, ref_dH, ref_dW = _oracle(H, W, y) # exact fp32 CE + _, f_dH, f_dW = _fused(H, W, y, chunk_size=512, grad_filter_eps=EPS_BF16) + torch.testing.assert_close(f_dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(f_dW, ref_dW, rtol=1e-2, atol=1e-2) + + +def test_too_aggressive_eps_is_detectable(): + """Guards against a rubber-stamp: an absurd eps=0.5 drops nearly all softmax mass, + and with a strongly NON-zero-mean W under SUM reduction (grads O(1)) the resulting + gradient MUST visibly break the 1e-2 gate. If this did not raise, the suite could + not tell a correct filter from a broken one.""" + g = torch.Generator().manual_seed(7) + N, D, V = 64, 32, 512 + H = torch.randn(N, D, generator=g) * 0.3 + W = torch.randn(V, D, generator=g) * 0.3 + 0.8 # strong non-zero mean + y = torch.randint(0, V, (N,), generator=g) + _, ref_dH, _ = _oracle(H, W, y, reduction="sum") + _, bad_dH, _ = _fused(H, W, y, reduction="sum", grad_filter_eps=0.5) + with pytest.raises(AssertionError): + torch.testing.assert_close(bad_dH, ref_dH, rtol=1e-2, atol=1e-2) + + +def test_bf16_rounded_gemm_emulation_within_tol(): + """CPU proxy for the Triton kernel's NUMERIC PATH (which the pure-fp32 tests do not + exercise): store H/W as bf16, recompute logits with bf16-rounded operands (fp32 + accumulate, as the kernel's tensor-core dot does), keep dlogit in fp32, and store + dH/dW as bf16. It must still clear the 1e-2 grad gate at a production-ish N. The + definitive check remains the off-box cce_triton.gate_against_reference().""" + g = torch.Generator().manual_seed(5) + N, D, V = 256, 64, 2048 + Hf = (torch.randn(N, D, generator=g) * 0.5) + Wf = (torch.randn(V, D, generator=g) * 0.5) + y = torch.randint(0, V, (N,), generator=g) + + # oracle: exact fp32 CE + ref_loss, ref_dH, ref_dW = _oracle(Hf, Wf, y) + + def _bf(x): # bf16 round-trip (operand rounding) + return x.bfloat16().float() + + # emulate the kernel: bf16-rounded logit operands, fp32 accumulate + softmax, + # fp32 dlogit, bf16-rounded W/H in the accumulate GEMMs, bf16-stored outputs. + Hb, Wb = _bf(Hf), _bf(Wf) + logits = torch.matmul(Hb, Wb.t()) # fp32 out, bf16-rounded operands + lse = torch.logsumexp(logits, dim=1, keepdim=True) + p = torch.exp(logits - lse) + p = torch.where(p >= EPS_BF16, p, torch.zeros_like(p)) # eps filter + onehot = F.one_hot(y, V).float() + dlogit = (p - onehot) / N # mean reduction, fp32 + dH = torch.matmul(dlogit, Wb).bfloat16().float() # bf16-rounded W operand, bf16 store + dW = torch.matmul(dlogit.t(), Hb).bfloat16().float() + + torch.testing.assert_close(dH, ref_dH, rtol=1e-2, atol=1e-2) + torch.testing.assert_close(dW, ref_dW, rtol=1e-2, atol=1e-2) + + +# --------------------------------------------------- edge cases matching F.cross_entropy + +def test_out_of_range_label_raises_like_cross_entropy(): + """y >= V or a stray negative (non-ignore) label must RAISE, exactly as + F.cross_entropy raises IndexError — never silently absorbed into a wrong loss.""" + H, W, y = _make_inputs() + Hf = H.float().clone().requires_grad_(True) + Wf = W.float().clone().requires_grad_(True) + y_hi = y.clone(); y_hi[0] = V_TINY # == V -> out of range + with pytest.raises(ValueError): + linear_cross_entropy(Hf, Wf, y_hi, chunk_size=CHUNK) + y_neg = y.clone(); y_neg[0] = -5 # negative, not ignore_index + with pytest.raises(ValueError): + linear_cross_entropy(Hf, Wf, y_neg, chunk_size=CHUNK) + + +def test_all_ignored_batch_returns_nan_like_oracle(): + """A fully-masked micro-batch is 0/0 -> NaN in F.cross_entropy(mean); the fused ref + must match (NOT a silent 0.0 that hides a labeling bug).""" + H, W, y = _make_inputs() + y = torch.full_like(y, -100) + ref = F.cross_entropy(H.float() @ W.float().t(), y, ignore_index=-100) # NaN + Hf = H.float().clone().requires_grad_(True) + Wf = W.float().clone().requires_grad_(True) + fz = linear_cross_entropy(Hf, Wf, y, chunk_size=CHUNK, ignore_index=-100, grad_filter_eps=0.0) + assert torch.isnan(ref) and torch.isnan(fz), (float(ref), float(fz)) + + +def test_cpu_fallback_entry_matches_reference(): + """cce_linear_ce.triton_linear_cross_entropy transparently falls back to the chunked + reference on CPU (no CUDA), so it imports + runs here and equals linear_cross_entropy.""" + H, W, y = _make_inputs() + Hf1 = H.float().clone().requires_grad_(True) + Wf1 = W.float().clone().requires_grad_(True) + loss1 = cce_linear_ce.triton_linear_cross_entropy(Hf1, Wf1, y, grad_filter_eps=0.0) + loss1.backward() + Hf2 = H.float().clone().requires_grad_(True) + Wf2 = W.float().clone().requires_grad_(True) + loss2 = linear_cross_entropy(Hf2, Wf2, y, grad_filter_eps=0.0) + loss2.backward() + torch.testing.assert_close(loss1, loss2) + torch.testing.assert_close(Hf1.grad, Hf2.grad) + torch.testing.assert_close(Wf1.grad, Wf2.grad) + + +def test_eps_constant_and_bf16_sub_ulp_rationale(): + """CORRECTED bf16 facts (the old comment was numerically wrong). bf16 has 7 EXPLICIT + mantissa bits, so ULP(1.0)=2**-7 and the round-to-1.0 boundary is ~2**-8. 2**-12 is + well BELOW that boundary -> it is a CONSERVATIVE SUB-ULP filter floor (it IS truncated + when added to 1.0), NOT 'the smallest non-truncated bf16 magnitude'.""" + assert EPS_BF16 == 2.0 ** -12 + # power of two -> exactly representable in bf16 (round-trips) + assert float(torch.tensor(EPS_BF16, dtype=torch.bfloat16).float()) == EPS_BF16 + one = torch.tensor(1.0, dtype=torch.bfloat16) + # sub-ULP: 2**-12 and 2**-8 both round (1.0 + x) back to 1.0; 2**-7 (= 1 ULP) does not. + assert (one + torch.tensor(2.0 ** -12, dtype=torch.bfloat16)) == one # truncated -> filter is safe + assert (one + torch.tensor(2.0 ** -8, dtype=torch.bfloat16)) == one # still rounds to 1.0 + assert (one + torch.tensor(2.0 ** -7, dtype=torch.bfloat16)) != one # 1 ULP -> representable diff --git a/research/tests/test_cka_probe.py b/research/tests/test_cka_probe.py new file mode 100644 index 0000000..5572a7e --- /dev/null +++ b/research/tests/test_cka_probe.py @@ -0,0 +1,44 @@ +"""Regression guard for research/interp/cka_probe.py's CKA math + decision-rule plumbing +(CPU-only; the GPU probe path is exercised separately). Locks in the invariances the +representational-convergence null depends on, and the anti-laundering direction of the +bootstrap decision rule.""" +import pathlib +import sys + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "interp")) +torch = pytest.importorskip("torch") +import cka_probe as cp # noqa: E402 + + +def test_cka_invariances(): + torch.manual_seed(0) + X = torch.randn(400, 48).double() + assert abs(cp.linear_cka(X, X) - 1.0) < 1e-9 # identity + assert abs(cp.linear_cka(X, X * 2.5 - 1.0) - 1.0) < 1e-6 # isotropic scale+shift + R = torch.linalg.qr(torch.randn(48, 48).double())[0] + assert abs(cp.linear_cka(X, X @ R) - 1.0) < 1e-6 # orthogonal transform + assert cp.linear_cka(X, torch.randn(400, 48).double()) < 0.2 # independent → low + + +def test_fp16_would_overflow_is_upcast(): + """The confound that faked the first null: large activations must not silently degrade. + linear_cka upcasts fp16 inputs; here we confirm a huge-magnitude pair still gives CKA≈1 + when it should (fp16 storage would have overflowed these to inf → CKA 0).""" + torch.manual_seed(0) + X = torch.randn(300, 32) * 3.0e5 # ~NorMuon-scale magnitudes + assert X.abs().max() > 65504 # beyond fp16 range + assert abs(cp.linear_cka(X.float(), X.float()) - 1.0) < 1e-5 + + +def test_decision_rule_direction(): + torch.manual_seed(0) + X = torch.randn(300, 32) + Y = torch.randn(300, 32) + clear = {"n": X, "a": Y, "t": X.clone()} # CKA(n,t)=1 ≫ CKA(a,t) → Δ≫0 + pt, lo, hi = cp.bootstrap_delta_ci(clear, "n", "a", "t", B=200) + assert lo > 0 and pt > 0.5 # disjoint-positive confirms + null = {"n": Y, "a": Y.clone(), "t": X} # both ≈0 vs t → Δ≈0 + pt2, lo2, hi2 = cp.bootstrap_delta_ci(null, "n", "a", "t", B=200) + assert lo2 <= 0 <= hi2 # straddles 0 → null diff --git a/research/tests/test_data_decontam.py b/research/tests/test_data_decontam.py new file mode 100644 index 0000000..d1afddb --- /dev/null +++ b/research/tests/test_data_decontam.py @@ -0,0 +1,97 @@ +"""Tests for research/data_decontam.py — document-disjoint splitting + n-gram +decontamination (the fix for the leak-suspect sequential val split). Pure, +deterministic, stdlib-only.""" +import data_decontam as dd + + +def test_split_is_deterministic_and_order_independent(): + docs = [f"document number {i} with some words" for i in range(200)] + a = [d for d in docs if dd.is_val_doc(d, seed=0, val_fraction=0.2)] + b = [d for d in reversed(docs) if dd.is_val_doc(d, seed=0, val_fraction=0.2)] + # same set regardless of order; a given doc always lands in the same bucket + assert set(a) == set(b) + # roughly val_fraction of docs (loose bound for n=200) + assert 0.1 < len(a) / len(docs) < 0.3 + + +def test_seed_changes_assignment(): + docs = [f"doc {i}" for i in range(300)] + s0 = {d for d in docs if dd.is_val_doc(d, seed=0, val_fraction=0.3)} + s1 = {d for d in docs if dd.is_val_doc(d, seed=1, val_fraction=0.3)} + assert s0 != s1 + + +def test_decontaminate_drops_overlapping_val_doc(): + train = ["the quick brown fox jumps over the lazy dog again and again today"] + clean = "an entirely unrelated sentence about marine biology and coral reefs ok" + leaked = train[0] # identical to a train doc -> must be dropped + kept, dropped, overlap = dd.decontaminate_val([clean, leaked], train, + n=5, threshold=0.5) + assert leaked not in kept and clean in kept + assert dropped == [1] and overlap[1] > overlap[0] + + +def test_split_and_decontaminate_report_shape(): + docs = [f"unique sentence alpha {i} bravo charlie delta echo foxtrot" for i in range(120)] + train, val, report = dd.split_and_decontaminate(docs, seed=7, val_fraction=0.25, + n=13, threshold=0.8) + assert set(train).isdisjoint(set(val)) # document-disjoint + assert report["split_seed"] == 7 and report["ngram_n"] == 13 + assert report["docs_dropped"] >= 0 + assert report["n_val_docs_kept"] == report["n_val_docs_raw"] - report["docs_dropped"] + # fixed-split lineage: base seed == effective seed, randomized flag off + assert report["randomized_split"] is False + assert report["base_split_seed"] == 7 and report["split_seed"] == 7 + + +# --- randomized per-cell split (the RS full-variance option) ----------------- + +def test_cell_split_seed_fixed_is_base_seed_for_every_cell(): + # fixed: every cell seed maps back to the SAME base split seed + for cs in range(10): + assert dd.cell_split_seed(42, cs, randomized=False) == 42 + + +def test_cell_split_seed_randomized_differs_per_cell_and_is_deterministic(): + seeds = {dd.cell_split_seed(42, cs, randomized=True) for cs in range(8)} + # a different split seed per cell (no collisions across 8 cells) + assert len(seeds) == 8 + # all non-negative and distinct from the trivial base seed + assert all(s >= 0 and s != 42 for s in seeds) + # deterministic: same (base, cell) -> same value + assert dd.cell_split_seed(42, 3, randomized=True) == dd.cell_split_seed(42, 3, randomized=True) + # base seed also mixes in: same cell seed, different base -> different split + assert dd.cell_split_seed(42, 3, randomized=True) != dd.cell_split_seed(43, 3, randomized=True) + + +def test_randomized_split_yields_different_partitions_across_cells(): + docs = [f"corpus document {i} with assorted lexical content here" for i in range(300)] + _, val0, r0 = dd.split_and_decontaminate(docs, seed=5, val_fraction=0.2, + n=13, threshold=0.8, + cell_seed=0, randomized=True) + _, val1, r1 = dd.split_and_decontaminate(docs, seed=5, val_fraction=0.2, + n=13, threshold=0.8, + cell_seed=1, randomized=True) + # different cell seeds -> different doc-disjoint partitions (corpus resampled) + assert set(val0) != set(val1) + assert r0["split_seed"] != r1["split_seed"] + assert r0["randomized_split"] is True and r0["base_split_seed"] == 5 + assert r0["cell_seed"] == 0 and r1["cell_seed"] == 1 + assert "randomized" in r0["method"] + + +def test_fixed_split_identical_partition_across_cells(): + docs = [f"corpus document {i} with assorted lexical content here" for i in range(300)] + _, val0, _ = dd.split_and_decontaminate(docs, seed=5, val_fraction=0.2, + cell_seed=0, randomized=False) + _, val1, _ = dd.split_and_decontaminate(docs, seed=5, val_fraction=0.2, + cell_seed=9, randomized=False) + # fixed split: membership is held constant regardless of cell seed + assert set(val0) == set(val1) + + +def test_randomized_requires_cell_seed(): + import pytest + with pytest.raises(ValueError): + dd.split_and_decontaminate(["a b c", "d e f"], seed=1, val_fraction=0.3, + randomized=True) # cell_seed missing diff --git a/research/tests/test_distributed_correctness.py b/research/tests/test_distributed_correctness.py new file mode 100644 index 0000000..af676dd --- /dev/null +++ b/research/tests/test_distributed_correctness.py @@ -0,0 +1,166 @@ +"""Tests for research/distributed_correctness.py — the §C21 correctness GATE. + +Pure math, no torch/CUDA/model. Proves the gate (a) PASSES a distributed run +that matches the single-GPU baseline within the seed-noise floor, (b) FAILS a +fast-but-wrong run whose loss is shifted beyond the floor, (c) hard-FAILS a +diverged (NaN/inf) distributed run, (d) derives its tolerance from real baseline +seed scatter (never an invented number), and (e) refuses to gate with no floor +or two floors. This is the "fast-but-wrong is caught BEFORE any MFU is trusted" +law, exercised entirely on CPU with synthetic numbers. +""" +import math + +import pytest + +import distributed_correctness as dc + + +# ---- noise_floor_from_seeds -------------------------------------------------- + +def test_noise_floor_is_k_sigma_of_seed_scatter(): + seeds = [2.50, 2.52, 2.48] # single-GPU baseline loss across 3 seeds + tol, mean, std = dc.noise_floor_from_seeds(seeds, k_sigma=2.0) + assert math.isclose(mean, 2.50, abs_tol=1e-9) + assert math.isclose(tol, 2.0 * std) + assert tol > 0 + + +def test_noise_floor_requires_two_seeds(): + with pytest.raises(ValueError): + dc.noise_floor_from_seeds([2.5]) + + +def test_noise_floor_rejects_nonfinite_seed(): + with pytest.raises(ValueError): + dc.noise_floor_from_seeds([2.5, float("nan")]) + + +def test_noise_floor_rejects_nonpositive_ksigma(): + with pytest.raises(ValueError): + dc.noise_floor_from_seeds([2.5, 2.6], k_sigma=0) + + +# ---- the gate: pass / fail --------------------------------------------------- + +def test_matching_distributed_run_passes_with_explicit_tolerance(): + r = dc.check_distributed_correctness( + distributed_metric=2.505, + single_gpu_baseline=2.50, + tolerance=0.05, + ) + assert r["passed"] is True + assert r["verdict"] == "pass" + assert r["margin"] >= 0 + assert "trust" in r["reason"].lower() or "pass" in r["reason"].lower() + + +def test_fast_but_wrong_run_fails(): + # Distributed loss shifted by 0.3 — way outside a 0.05 floor. The canonical + # sharding/all-reduce bug: descends, but to the wrong place. + r = dc.check_distributed_correctness( + distributed_metric=2.80, + single_gpu_baseline=2.50, + tolerance=0.05, + ) + assert r["passed"] is False + assert r["verdict"] == "fail" + assert r["margin"] < 0 + assert "wrong" in r["reason"].lower() or "diverge" in r["reason"].lower() + + +def test_diverged_nan_distributed_run_hard_fails(): + r = dc.check_distributed_correctness( + distributed_metric=float("nan"), + single_gpu_baseline=2.50, + tolerance=10.0, # huge tolerance must NOT rescue a NaN + ) + assert r["passed"] is False + assert math.isinf(r["abs_delta"]) + + +def test_inf_distributed_run_hard_fails(): + r = dc.check_distributed_correctness( + distributed_metric=float("inf"), + single_gpu_baseline=2.50, + tolerance=10.0, + ) + assert r["passed"] is False + + +# ---- tolerance derived from seeds (the §C17/§C21 link) ----------------------- + +def test_gate_derives_tolerance_from_baseline_seeds(): + seeds = [2.50, 2.52, 2.48] + # Distributed run lands at the seed mean -> trivially inside the floor. + r = dc.check_distributed_correctness( + distributed_metric=2.50, + single_gpu_baseline=None, # use the seed mean as reference + baseline_seeds=seeds, + k_sigma=2.0, + ) + assert r["passed"] is True + assert r["n_baseline_seeds"] == 3 + assert math.isclose(r["single_gpu_baseline"], 2.50, abs_tol=1e-9) + + +def test_gate_fails_when_outside_seed_floor(): + seeds = [2.50, 2.501, 2.499] # very tight scatter -> tiny floor + r = dc.check_distributed_correctness( + distributed_metric=2.70, # far outside the tight floor + single_gpu_baseline=None, + baseline_seeds=seeds, + k_sigma=2.0, + ) + assert r["passed"] is False + + +# ---- relative floor: reduction-order reassociation must not fail ------------- + +def test_bit_reassociation_within_relative_floor_passes(): + # Zero-scatter baseline (lucky identical seeds) but a 1e-7 relative diff from + # a different all-reduce summation order must NOT be rejected. + base = 2.5 + r = dc.check_distributed_correctness( + distributed_metric=base * (1 + 1e-7), + single_gpu_baseline=base, + tolerance=0.0, # the explicit floor is 0; relative floor must rescue + ) + assert r["passed"] is True + + +# ---- argument validation ----------------------------------------------------- + +def test_rejects_no_tolerance_source(): + with pytest.raises(ValueError): + dc.check_distributed_correctness(2.5, 2.5) + + +def test_rejects_two_tolerance_sources(): + with pytest.raises(ValueError): + dc.check_distributed_correctness( + 2.5, 2.5, tolerance=0.1, baseline_seeds=[2.5, 2.6] + ) + + +def test_explicit_tolerance_requires_baseline(): + with pytest.raises(ValueError): + dc.check_distributed_correctness(2.5, None, tolerance=0.1) + + +def test_negative_tolerance_rejected(): + with pytest.raises(ValueError): + dc.check_distributed_correctness(2.5, 2.5, tolerance=-0.1) + + +# ---- gate_or_raise ----------------------------------------------------------- + +def test_gate_or_raise_passes_through_on_pass(): + r = dc.gate_or_raise(2.505, 2.50, tolerance=0.05) + assert r["passed"] is True + + +def test_gate_or_raise_raises_on_fail(): + with pytest.raises(dc.CorrectnessGateError) as exc: + dc.gate_or_raise(2.80, 2.50, tolerance=0.05) + assert exc.value.result["passed"] is False + assert "fail" in str(exc.value).lower() or "wrong" in str(exc.value).lower() diff --git a/research/tests/test_eval_math_acc.py b/research/tests/test_eval_math_acc.py new file mode 100644 index 0000000..2b9f665 --- /dev/null +++ b/research/tests/test_eval_math_acc.py @@ -0,0 +1,148 @@ +"""CPU unit tests for research/eval_math_acc.py — the math-acc-v1 decision-metric scorer. +Covers the pinned extractor, the SymPy-normalized verifier, the Chen-2021 pass@k estimator, +the pass@1 Wilson aggregation, and the verifier-honesty (permissive-vs-strict) differential +fuzz. No model, no network, no CUDA.""" +import math + +import pytest + +from research.eval_math_acc import ( + EXTRACTOR_VERSION, extract_answer, is_equiv, pass_at_k, + verifier_false_positive_rate, run_math_acc, +) + +# --------------------------------------------------------------- extractor + +@pytest.mark.parametrize("text,expected", [ + (r"the reasoning ... so the answer is \boxed{72}.", "72"), + (r"\boxed{-3/4}", "-3/4"), + (r"first \boxed{1} then \boxed{2}", "2"), # LAST boxed wins + (r"\boxed{\frac{1}{2}}", r"\frac{1}{2}"), # balanced braces kept + ("Chain of thought...\n#### 18", "18"), # GSM8K gold convention + ("The final answer is 42", "42"), + ("blah blah 5 then 6 then 7", "7"), # last-number fallback +]) +def test_extract_answer(text, expected): + assert extract_answer(text) == expected + +def test_extract_answer_none(): + assert extract_answer("") is None + assert extract_answer("no digits or box here") is None + +# --------------------------------------------------------------- verifier (is_equiv) + +@pytest.mark.parametrize("pred,gold", [ + ("72", "72"), + (r"\boxed{72}", "72"), # extraction not needed; is_equiv normalizes box-free + ("1/2", "0.5"), + (r"\frac{1}{2}", "0.5"), + ("1,234", "1234"), # thousands separator + ("2^3", "8"), # sympy symbolic + ("-3/4", "-0.75"), + (" 6 ", "6"), + (r"\boxed{50\%}", "50"), # unit strip +]) +def test_is_equiv_true(pred, gold): + assert is_equiv(pred, gold) is True + +@pytest.mark.parametrize("pred,gold", [ + ("72", "73"), + ("1/2", "0.51"), + ("8", "9"), + (None, "5"), + ("5", None), + ("", "5"), +]) +def test_is_equiv_false(pred, gold): + assert is_equiv(pred, gold) is False + +def test_is_equiv_never_crashes_on_garbage(): + # a verifier must never throw during training — pathological latex ⇒ False, not an exception + for junk in [r"\frac{{{", "((((", r"\boxed{" * 50, "x" * 500, "1/0"]: + assert is_equiv(junk, "5") is False + +# --------------------------------------------------------------- pass@k (Chen 2021) + +def test_pass_at_k_edges(): + assert pass_at_k(4, 0, 2) == 0.0 # no correct samples + assert pass_at_k(4, 4, 2) == 1.0 # all correct + assert pass_at_k(4, 4, 1) == 1.0 + assert math.isclose(pass_at_k(4, 1, 1), 0.25) # k=1 ⇒ c/n + assert math.isclose(pass_at_k(10, 3, 1), 0.3) + +def test_pass_at_k_matches_closed_form(): + # 1 - C(n-c,k)/C(n,k) + for n, c, k in [(10, 2, 5), (16, 3, 8), (8, 1, 4), (20, 7, 16)]: + ref = 1.0 - math.comb(n - c, k) / math.comb(n, k) + assert math.isclose(pass_at_k(n, c, k), ref, rel_tol=1e-12) + +def test_pass_at_k_monotonic_in_k(): + vals = [pass_at_k(16, 3, k) for k in (1, 2, 4, 8, 16)] + assert all(a <= b + 1e-12 for a, b in zip(vals, vals[1:])) # pass@k non-decreasing in k + +def test_pass_at_k_bad_args(): + with pytest.raises(ValueError): + pass_at_k(4, 1, 0) + with pytest.raises(ValueError): + pass_at_k(4, 1, 5) # k > n + +# --------------------------------------------------------------- verifier honesty (differential) + +def test_verifier_false_positive_rate_strict_is_zero(): + # KNOWN-WRONG pairs: the strict verifier must accept NONE of them + wrong = [("72", "73"), ("1/2", "1/3"), ("10", "100"), ("5", "-5"), ("2^3", "9")] + assert verifier_false_positive_rate(wrong) == 0.0 + +def test_permissive_vs_strict_differential_fuzz(): + # The pinned strict verifier vs a naive PERMISSIVE one (substring / last-number match) + # on adversarial wrong pairs — the permissive verifier reward-hacks, the strict must not. + def permissive(pred, gold): # the tempting-but-wrong extractor + return gold in pred or extract_answer(pred) == extract_answer(gold) + adversarial = [ + ("the answer is 3 (not 30)", "30"), # substring 30⊄ but 3 present → naive trips on '3' + ("1234", "34"), # gold '34' is a substring of pred + ("100", "10"), # gold '10' substring of '100' + ("x = 5 or 15", "15"), + ] + strict_fp = sum(1 for p, g in adversarial if is_equiv(p, g)) + perm_fp = sum(1 for p, g in adversarial if permissive(p, g)) + assert strict_fp == 0 # pinned verifier: no false positives + assert perm_fp > strict_fp # the permissive one DOES get gamed → why we pin strict + +# --------------------------------------------------------------- end-to-end harness + +def _stub(correct_frac): + """A deterministic stub policy: emits the gold answer for the first round(correct_frac*n) + samples and a wrong one for the rest. No model.""" + def gen(prompt, n): + gold = "4" if "2+2" in prompt else "6" + n_ok = round(correct_frac * n) + return [rf"\boxed{{{gold}}}"] * n_ok + [r"\boxed{999}"] * (n - n_ok) + return gen + +def test_run_math_acc_all_correct(): + items = [{"prompt": "2+2?", "gold": "4"}, {"prompt": "3+3?", "gold": "6"}] + out = run_math_acc(_stub(1.0), items, n_samples=8, k_list=(1, 8)) + assert out["pass1_wilson_ci"]["acc"] == 1.0 + assert out["passk_chen2021"][8] == 1.0 + assert out["solved_items"] == 2 + assert out["extractor_version"] == EXTRACTOR_VERSION # extractor_pinned stamped + +def test_run_math_acc_all_wrong_is_clean_zero(): + # the honest-null case the plan predicts for our 0.6B base: pass@k == 0 is a REAL result + items = [{"prompt": "2+2?", "gold": "4"}] + out = run_math_acc(_stub(0.0), items, n_samples=16, k_list=(1, 8, 16)) + assert out["pass1_wilson_ci"]["acc"] == 0.0 + assert out["passk_chen2021"][16] == 0.0 + assert out["solved_items"] == 0 + +def test_run_math_acc_partial_passk_gt_pass1(): + # 2/8 correct per item ⇒ pass@1 = 0.25, pass@8 = 1.0 (Chen): pass@k must exceed pass@1 + items = [{"prompt": "2+2?", "gold": "4"}] + out = run_math_acc(_stub(0.25), items, n_samples=8, k_list=(1, 8)) + assert math.isclose(out["pass1_wilson_ci"]["acc"], 0.25) + assert out["passk_chen2021"][8] > out["passk_chen2021"][1] + +def test_run_math_acc_empty_raises(): + with pytest.raises(ValueError): + run_math_acc(_stub(1.0), [], n_samples=4) diff --git a/research/tests/test_framework.py b/research/tests/test_framework.py new file mode 100644 index 0000000..2523d32 --- /dev/null +++ b/research/tests/test_framework.py @@ -0,0 +1,76 @@ +"""Tests for research/harness_search/framework.py — the harness-search selection ++ promotion gate. Uses a SYNTHETIC scorer (no model, no GPU, instant) to pin the +behaviors that matter, above all the lesson our own experiment taught: a brittle +search-winner (timeout/invalid on an unseen seed) must NEVER be promoted, even +when it has the best search score. +""" +import pytest + +import framework as fw + +SEARCH_SEED = 1 +HELDOUT = [2, 3, 4] + +# name -> {search, heldout(list aligned to HELDOUT), valid(list, default all 1)} +SPEC = { + "inc": {"search": 0.866, "heldout": [0.869, 0.871, 0.870]}, + "good": {"search": 0.920, "heldout": [0.921, 0.919, 0.923]}, + # BEST search score, but one held-out seed times out (score 0, valid 0.4): + "brittle": {"search": 0.930, "heldout": [0.000, 0.922, 0.921], "valid": [0.4, 1, 1]}, + "tie": {"search": 0.900, "heldout": [0.870, 0.872, 0.869]}, + "hi_search_lo_held": {"search": 0.950, "heldout": [0.881, 0.879, 0.882]}, + "lo_search_hi_held": {"search": 0.910, "heldout": [0.921, 0.919, 0.923]}, +} + + +def scorer(path, seed): + s = SPEC[str(path)] + if seed == SEARCH_SEED: + return s["search"], 1.0 + i = HELDOUT.index(seed) + valid = s.get("valid", [1.0, 1.0, 1.0])[i] + return s["heldout"][i], valid + + +def run(cands): + return fw.select_and_promote(scorer, cands, "inc", SEARCH_SEED, HELDOUT, + direction="higher_is_better") + + +def test_promotes_significant_nonbrittle(): + r = run(["good"]) + assert r["promoted"] is True + assert r["challenger"] == "good" and r["significance"]["significant"] is True + + +def test_excludes_brittle_even_with_best_search_score(): + # 'brittle' has the highest search score (0.930) so it tops model-selection, + # but it times out on one held-out seed -> must be excluded; 'good' wins. + r = run(["brittle", "good"]) + assert "brittle" in r["excluded_brittle"] + assert r["promoted"] is True and r["challenger"] == "good" + + +def test_keeps_incumbent_when_not_significant(): + r = run(["tie"]) + assert r["promoted"] is False + assert "does not significantly beat" in r["reason"] + + +def test_all_brittle_no_promotion(): + r = run(["brittle"]) + assert r["promoted"] is False + assert r["reason"] == "all candidates brittle on held-out" + assert r["challenger"] is None + + +def test_selection_uses_heldout_not_search(): + # hi_search_lo_held has the best SEARCH (0.950) but worse HELD-OUT (~0.880); + # lo_search_hi_held wins on held-out (~0.921). Selection must follow held-out. + r = run(["hi_search_lo_held", "lo_search_hi_held"]) + assert r["challenger"] == "lo_search_hi_held" and r["promoted"] is True + + +def test_empty_pool_raises(): + with pytest.raises(ValueError): + run([]) diff --git a/research/tests/test_kernel_oracle.py b/research/tests/test_kernel_oracle.py new file mode 100644 index 0000000..d9ef265 --- /dev/null +++ b/research/tests/test_kernel_oracle.py @@ -0,0 +1,209 @@ +"""Tests for research/kernel_oracle.py — the correctness-first kernel gate (§C21). + +The single most important property: the oracle PASSES a correct candidate and +FAILS (discards) a subtly-wrong one, across a shape x dtype sweep, WITHOUT a GPU. +A fast-but-wrong kernel must never be allowed through to the roofline stage. + +Pure CPU, numpy-only. torch is optional and only exercised opportunistically. +""" +import math + +import numpy as np +import pytest + +import kernel_oracle as ko + + +# ----------------------------------------------------------- tolerance policy + +def test_dtype_name_normalizes_aliases(): + assert ko._dtype_name("torch.float16") == "float16" + assert ko._dtype_name("fp32") == "float32" + assert ko._dtype_name("bf16") == "bfloat16" + assert ko._dtype_name("half") == "float16" + assert ko._dtype_name(None) == "float32" + + +def test_tolerance_per_dtype_is_looser_for_low_precision(): + assert ko.tolerance_for("float32")["rtol"] < ko.tolerance_for("float16")["rtol"] + assert ko.tolerance_for("float16")["rtol"] < ko.tolerance_for("bfloat16")["rtol"] + # unknown dtype falls back to the conservative fp32 tol, never silently loose + assert ko.tolerance_for("int4-quark") == ko.DEFAULT_TOL + + +# ----------------------------------------------------------- compare() + +def test_compare_identical_passes(): + a = np.linspace(-3, 3, 100) + r = ko.compare(a, a.copy(), dtype="float32") + assert r["passed"] is True + assert r["n_mismatch"] == 0 + assert r["max_abs_err"] == 0.0 + + +def test_compare_accepts_plain_lists(): + r = ko.compare([1.0, 2.0, 3.0], [1.0, 2.0, 3.0]) + assert r["passed"] is True + + +def test_compare_small_drift_within_fp16_tol_passes_but_fails_fp32(): + ref = np.ones(64) + cand = ref + 5e-4 # drift ~5e-4 + assert ko.compare(ref, cand, dtype="float16")["passed"] is True + assert ko.compare(ref, cand, dtype="float32")["passed"] is False + + +def test_compare_shape_mismatch_is_structural_fail(): + r = ko.compare(np.zeros((4, 4)), np.zeros((4, 5))) + assert r["passed"] is False + assert "shape mismatch" in r["reason"] + + +def test_compare_nan_in_candidate_is_hard_fail(): + ref = np.ones(10) + cand = ref.copy() + cand[3] = np.nan + r = ko.compare(ref, cand, dtype="bfloat16") # even at the loosest tol + assert r["passed"] is False + assert "non-finite" in r["reason"] + + +def test_compare_inf_is_hard_fail(): + ref = np.ones(10) + cand = ref.copy() + cand[0] = np.inf + assert ko.compare(ref, cand)["passed"] is False + + +# ----------------------------------------------------------- the gate verdict + +SHAPES = [(8, 256), (1, 4096), (32, 1024)] +DTYPES = ("float32", "float16", "bfloat16") + + +def test_gate_passes_correct_rmsnorm_candidate(): + v = ko.gate(ko.rmsnorm_ref, ko.rmsnorm_candidate_correct, SHAPES, DTYPES) + assert v["passed"] is True + assert v["n_failed"] == 0 + assert v["n_cells"] == len(SHAPES) * len(DTYPES) + assert "PASS" in v["summary"] + + +def test_gate_discards_subtly_wrong_rmsnorm_candidate(): + v = ko.gate(ko.rmsnorm_ref, ko.rmsnorm_candidate_wrong, SHAPES, DTYPES) + assert v["passed"] is False + assert v["n_failed"] >= 1 + assert "DISCARDED" in v["summary"] + # every failing cell carries a numeric reason, not a crash + for cell in v["cells"]: + if not cell["passed"]: + assert cell["n_mismatch"] != 0 + + +def test_gate_discards_wrong_silu_candidate(): + # silu vs sigmoid: same shape, very different values -> must fail + v = ko.gate(ko.silu_ref, ko.silu_candidate_wrong, SHAPES, ("float32",)) + assert v["passed"] is False + + +def test_gate_passes_silu_against_itself(): + v = ko.gate(ko.silu_ref, ko.silu_ref, SHAPES, DTYPES) + assert v["passed"] is True + + +def test_gate_reference_and_candidate_get_identical_inputs(): + # If the two fns received different random inputs, an identity-vs-identity + # comparison would fail. It must pass -> same inputs feed both arms. + seen = {} + + def ref(x, dtype="float32"): + seen["ref"] = x.copy() + return x * 2.0 + + def cand(x, dtype="float32"): + seen["cand"] = x.copy() + return x * 2.0 + + v = ko.gate(ref, cand, [(16,)], ("float32",)) + assert v["passed"] is True + np.testing.assert_array_equal(seen["ref"], seen["cand"]) + + +def test_gate_is_deterministic_across_runs(): + v1 = ko.gate(ko.rmsnorm_ref, ko.rmsnorm_candidate_wrong, SHAPES, DTYPES, seed=7) + v2 = ko.gate(ko.rmsnorm_ref, ko.rmsnorm_candidate_wrong, SHAPES, DTYPES, seed=7) + assert v1["n_failed"] == v2["n_failed"] + for c1, c2 in zip(v1["cells"], v2["cells"]): + assert c1["max_abs_err"] == c2["max_abs_err"] + + +def test_gate_candidate_that_raises_is_fail_not_crash(): + def boom(x, dtype="float32"): + raise RuntimeError("kernel launch failed") + + v = ko.gate(ko.rmsnorm_ref, boom, [(8, 8)], ("float32",)) + assert v["passed"] is False + assert "raised" in v["cells"][0]["reason"] + + +def test_gate_rejects_empty_sweeps(): + with pytest.raises(ValueError): + ko.gate(ko.silu_ref, ko.silu_ref, [], ("float32",)) + with pytest.raises(ValueError): + ko.gate(ko.silu_ref, ko.silu_ref, [(8,)], ()) + + +def test_gate_custom_input_factory_is_used(): + # a two-input op (add) needs a factory yielding two arrays + def factory(shape, dtype, rng): + return rng.standard_normal(shape), rng.standard_normal(shape) + + def add_ref(a, b, dtype="float32"): + return np.asarray(a, np.float64) + np.asarray(b, np.float64) + + def add_wrong(a, b, dtype="float32"): + return np.asarray(a, np.float64) - np.asarray(b, np.float64) # BUG + + assert ko.gate(add_ref, add_ref, [(32,)], ("float32",), + input_factory=factory)["passed"] is True + assert ko.gate(add_ref, add_wrong, [(32,)], ("float32",), + input_factory=factory)["passed"] is False + + +# ----------------------------------------------------------- reference ops + +def test_rmsnorm_ref_unit_rms_input(): + # a vector with mean-square exactly 1 -> RMSNorm(eps->0) returns it ~unchanged + x = np.array([[1.0, -1.0, 1.0, -1.0]]) + out = ko.rmsnorm_ref(x, eps=0.0) + np.testing.assert_allclose(out, x, rtol=0, atol=1e-12) + + +def test_rmsnorm_ref_applies_weight(): + x = np.array([[3.0, 4.0]]) + w = np.array([2.0, 0.5]) + out = ko.rmsnorm_ref(x, weight=w, eps=0.0) + base = ko.rmsnorm_ref(x, eps=0.0) + np.testing.assert_allclose(out, base * w, rtol=0, atol=1e-12) + + +def test_wrong_rmsnorm_actually_differs_on_a_concrete_input(): + # mean-then-square vs square-then-mean: on a zero-mean input the wrong one + # divides by ~sqrt(eps) -> blows up, proving the bug is real (not cosmetic). + x = np.array([[2.0, -2.0, 2.0, -2.0]]) # mean 0, mean-square 4 + good = ko.rmsnorm_candidate_correct(x, eps=1e-6) + bad = ko.rmsnorm_candidate_wrong(x, eps=1e-6) + assert not np.allclose(good, bad) + + +# ----------------------------------------------------------- torch path (opt) + +def test_compare_accepts_torch_tensors_if_available(): + torch = pytest.importorskip("torch") + ref = torch.linspace(-2, 2, 50) + cand = ref.clone() + r = ko.compare(ref, cand, dtype="float32") + assert r["passed"] is True + # a torch tensor with a real drift still fails at fp32 tol + cand2 = ref + 0.01 + assert ko.compare(ref, cand2, dtype="float32")["passed"] is False diff --git a/research/tests/test_posttrain_losses.py b/research/tests/test_posttrain_losses.py new file mode 100644 index 0000000..3b667de --- /dev/null +++ b/research/tests/test_posttrain_losses.py @@ -0,0 +1,145 @@ +"""Tests for research/posttrain_losses.py — the SFT/DPO/GRPO math core of the +post-training executor. Pure, deterministic, no model. DPO/GRPO are subtle, so +each property is pinned against a hand-derived closed-form value. +""" +import math + +import pytest + +import posttrain_losses as pt + + +# ------------------------------------------------------------------- helpers + +def test_log_sigmoid_matches_definition(): + for x in (-5.0, -0.3, 0.0, 0.7, 4.0): + assert math.isclose(pt.log_sigmoid(x), math.log(1 / (1 + math.exp(-x))), abs_tol=1e-12) + + +# ------------------------------------------------------------------- SFT + +def test_sft_nll_no_smoothing_is_neg_logp(): + assert pt.label_smoothed_nll(-2.0, vocab_size=1000) == 2.0 + + +def test_sft_label_smoothing_interpolates(): + # eps=0.1, logp=-2, V=1000 -> 0.9*2 + 0.1*log(1000) + expected = 0.9 * 2.0 + 0.1 * math.log(1000) + assert math.isclose(pt.label_smoothed_nll(-2.0, 1000, smoothing=0.1), expected) + + +def test_sft_validates(): + with pytest.raises(ValueError): + pt.label_smoothed_nll(-1.0, 1000, smoothing=1.0) + with pytest.raises(ValueError): + pt.label_smoothed_nll(-1.0, vocab_size=1) + + +# ------------------------------------------------------------------- DPO + +def test_dpo_policy_equals_reference_is_log2(): + # policy == reference => logits 0 => loss = -log sigmoid(0) = log 2; reward 0; acc 0. + r = pt.dpo_loss([-3.0], [-3.0], [-5.0], [-5.0], beta=0.1) + assert math.isclose(r["loss"], math.log(2), abs_tol=1e-9) + assert r["chosen_reward"] == 0.0 and r["rejected_reward"] == 0.0 + assert r["accuracy"] == 0.0 and r["margin"] == 0.0 + + +def test_dpo_known_closed_form_value(): + # one pair: pol_chosen=-1, ref_chosen=-2 -> chosen_logratio=+1 + # pol_rej=-4, ref_rej=-2 -> rejected_logratio=-2 + # beta=0.5 -> logits = 0.5*(1 - (-2)) = 1.5 ; loss = -log sigmoid(1.5) + r = pt.dpo_loss([-1.0], [-2.0], [-4.0], [-2.0], beta=0.5) + assert math.isclose(r["loss"], -pt.log_sigmoid(1.5), abs_tol=1e-12) + assert math.isclose(r["chosen_reward"], 0.5, abs_tol=1e-12) # 0.5 * 1 + assert math.isclose(r["rejected_reward"], -1.0, abs_tol=1e-12) # 0.5 * -2 + assert r["accuracy"] == 1.0 and math.isclose(r["margin"], 1.5) + + +def test_dpo_strong_preference_drives_loss_down_and_acc_up(): + # policy pushes chosen way up, rejected way down -> large positive logits. + r = pt.dpo_loss([0.0, 0.0], [-3.0, -3.0], [-9.0, -9.0], [-3.0, -3.0], beta=0.5) + assert r["loss"] < 0.05 and r["accuracy"] == 1.0 + + +def test_dpo_length_mismatch_raises(): + with pytest.raises(ValueError): + pt.dpo_loss([-1.0], [-2.0], [-1.0], [-2.0, -3.0]) + with pytest.raises(ValueError): + pt.dpo_loss([], [], [], []) + + +# ------------------------------------------------------------------- GRPO + +def test_group_advantages_zero_mean_unit_scale(): + adv = pt.group_normalized_advantages([1.0, 2.0, 3.0]) + assert math.isclose(sum(adv), 0.0, abs_tol=1e-9) # mean-centered + # pstdev of [1,2,3] = sqrt(2/3); advantages = (-1,0,1)/(sqrt(2/3)+eps). + # tol allows for the 1e-6 GRPO stabilizer in the denominator. + s = math.sqrt(2 / 3) + assert math.isclose(adv[0], -1 / s, abs_tol=1e-3) and math.isclose(adv[2], 1 / s, abs_tol=1e-3) + assert math.isclose(adv[1], 0.0, abs_tol=1e-9) + + +def test_group_advantages_all_equal_is_zero(): + assert all(abs(a) < 1e-3 for a in pt.group_normalized_advantages([5.0, 5.0, 5.0])) + + +def test_grpo_clip_caps_positive_advantage(): + # ratio 2.0, advantage +1, eps 0.2 -> clipped to 1.2*1 = 1.2 (gain capped). + assert math.isclose(pt.grpo_clipped_objective([2.0], [1.0], clip_eps=0.2), 1.2) + + +def test_grpo_clip_negative_advantage_uses_unclipped_min(): + # ratio 2.0, advantage -1 -> min(2*-1, 1.2*-1) = min(-2, -1.2) = -2. + assert math.isclose(pt.grpo_clipped_objective([2.0], [-1.0], clip_eps=0.2), -2.0) + + +def test_grpo_ratio_one_returns_mean_advantage(): + # ratio==1 everywhere => surrogate == advantage => objective == mean(adv). + adv = [-0.5, 0.0, 0.5] + assert math.isclose(pt.grpo_clipped_objective([1.0, 1.0, 1.0], adv), 0.0) + + +def test_kl_k3_zero_when_equal_and_positive_otherwise(): + assert pt.kl_penalty_k3([-1.0, -2.0], [-1.0, -2.0]) == 0.0 + assert pt.kl_penalty_k3([-1.0, -2.0], [-1.5, -2.5]) > 0.0 # always >= 0 + + +def test_grpo_validations(): + with pytest.raises(ValueError): + pt.grpo_clipped_objective([1.0], [1.0, 2.0]) + with pytest.raises(ValueError): + pt.group_normalized_advantages([]) + with pytest.raises(ValueError): + pt.kl_penalty_k3([-1.0], []) + + +# --- assembled GRPO loss (sign-correct) + SFT response masking (audit fix) ----- + +def test_grpo_loss_decreases_as_policy_improves(): + # advantages favour the chosen direction; a policy that increases its prob on + # positive-advantage samples (ratio>1) must DRIVE THE ASSEMBLED LOSS DOWN. + adv = [1.0, 1.0, -1.0] + worse = pt.grpo_loss(ratios=[1.0, 1.0, 1.0], advantages=adv, + logp_pol=[0.0], logp_ref=[0.0])["loss"] + better = pt.grpo_loss(ratios=[1.3, 1.2, 0.8], advantages=adv, + logp_pol=[0.0], logp_ref=[0.0])["loss"] + assert better < worse + + +def test_grpo_loss_kl_penalty_raises_loss(): + base = pt.grpo_loss([1.1], [1.0], logp_pol=[-1.0], logp_ref=[-0.5], beta_kl=0.0) + pen = pt.grpo_loss([1.1], [1.0], logp_pol=[-1.0], logp_ref=[-0.5], beta_kl=0.5) + assert pen["kl"] > 0 and pen["loss"] > base["loss"] + + +def test_masked_sft_nll_ignores_prompt_tokens(): + # only the last two tokens are response; the first (huge-loss) prompt token + # must NOT affect the loss. + logp = [-9.0, -0.2, -0.4] + mask = [0, 1, 1] + got = pt.masked_sft_nll(logp, mask) + assert got == pytest.approx((0.2 + 0.4) / 2) + with pytest.raises(ValueError): + pt.masked_sft_nll([-0.1, -0.2], [0, 0]) # mask selects nothing diff --git a/research/tests/test_scaling_ladder.py b/research/tests/test_scaling_ladder.py new file mode 100644 index 0000000..38326b2 --- /dev/null +++ b/research/tests/test_scaling_ladder.py @@ -0,0 +1,214 @@ +"""Tests for research/scaling_ladder.py — pure stdlib, no GPU, no model loads. + +Two halves: + PLAN — build_ladder emits the right per-cell run specs (budget->steps, + shared seeds across arms, fewer seeds at larger budgets). + ANALYZE — fit_gap_trend turns synthetic per-budget gaps into the correct + PERSISTS / CONVERGES / WIDENS / FLAT verdict, with the noise floor + and the <3-budget descriptive guard honored. +""" +import math + +import pytest + +import scaling_ladder as sl + + +# ----- PART 1: PLAN ----------------------------------------------------------- + +BASE = {"model": "qwen3-0.6b", "lr": 3e-3, "seq_len": 2048} +BUDGETS = [42e6, 170e6, 670e6, 1190e6] +TPS = 2048 * 256 # tokens per step (seq_len * global batch) + + +def test_steps_for_budget_rounds_up(): + assert sl.steps_for_budget(1000, 100) == 10 + assert sl.steps_for_budget(1001, 100) == 11 # partial final step still runs + with pytest.raises(ValueError): + sl.steps_for_budget(0, 100) + with pytest.raises(ValueError): + sl.steps_for_budget(1000, 0) + + +def test_build_ladder_cell_count_and_steps(): + out = sl.build_ladder(BASE, BUDGETS, TPS, arms=("normuon", "adamw"), + seeds_per_budget=3, base_name="qwen3") + # 4 budgets * 3 seeds * 2 arms = 24 cells + assert out["summary"]["n_cells"] == 24 + assert out["summary"]["n_budgets"] == 4 + assert out["summary"]["n_arms"] == 2 + # steps derived from tok_per_step, rounded up + for c in out["cells"]: + assert c["max_steps"] == math.ceil(c["train_tokens"] / TPS) + + +def test_seeds_shared_across_arms_at_same_budget(): + # the seed must be a PAIRED factor: both arms train on the same seed set at a + # given budget, so seed is blocked out of the gap, not added as noise. + out = sl.build_ladder(BASE, [42e6], TPS, arms=("normuon", "adamw"), + seeds_per_budget=3, base_seed=10) + by_arm = {} + for c in out["cells"]: + by_arm.setdefault(c["arm"], set()).add(c["seed"]) + assert by_arm["normuon"] == by_arm["adamw"] == {10, 11, 12} + + +def test_fewer_seeds_at_larger_budgets_via_list(): + out = sl.build_ladder(BASE, BUDGETS, TPS, arms=("a", "b"), + seeds_per_budget=[3, 3, 2, 1]) + per = out["summary"]["cells_per_budget"] + # cells_per_budget counts BOTH arms: 3*2, 3*2, 2*2, 1*2 + assert per["42M"] == 6 + assert per["170M"] == 6 + assert per["670M"] == 4 + assert per["1190M"] == 2 + + +def test_fewer_seeds_via_dict_keyed_by_budget(): + out = sl.build_ladder(BASE, BUDGETS, TPS, arms=("a", "b"), + seeds_per_budget={42e6: 3, 170e6: 3, 670e6: 2, 1190e6: 1}) + assert out["summary"]["cells_per_budget"]["1190M"] == 2 + + +def test_seed_schedule_length_mismatch_rejected(): + with pytest.raises(ValueError): + sl.build_ladder(BASE, BUDGETS, TPS, seeds_per_budget=[3, 2]) # 2 != 4 + + +def test_zero_seeds_rejected(): + with pytest.raises(ValueError): + sl.build_ladder(BASE, BUDGETS, TPS, seeds_per_budget=[3, 3, 0, 1]) + + +def test_single_arm_rejected(): + with pytest.raises(ValueError): + sl.build_ladder(BASE, BUDGETS, TPS, arms=("only",)) + + +def test_run_ids_unique_and_config_carries_budget(): + out = sl.build_ladder(BASE, BUDGETS, TPS, arms=("normuon", "adamw")) + ids = [c["run_id"] for c in out["cells"]] + assert len(ids) == len(set(ids)) # no collisions + for c in out["cells"]: + assert c["config"]["train_tokens"] == c["train_tokens"] + assert c["config"]["max_steps"] == c["max_steps"] + assert c["config"]["arm"] == c["arm"] + assert c["config"]["model"] == "qwen3-0.6b" # base config preserved + assert f"arm:{c['arm']}" in c["tags"] + + +def test_total_train_tokens_summed(): + out = sl.build_ladder(BASE, [42e6, 170e6], TPS, arms=("a", "b"), + seeds_per_budget=2) + # 2 budgets * 2 seeds * 2 arms; each (42M+170M) per (seed,arm) pair... sum it + expect = sum(c["train_tokens"] for c in out["cells"]) + assert out["summary"]["total_train_tokens"] == expect + assert expect == (42e6 + 170e6) * 2 * 2 # 4 (seed,arm) combos + + +# ----- PART 2: ANALYZE -------------------------------------------------------- + +LADDER = [42e6, 170e6, 670e6, 1190e6] + + +def _gaps_from(slope, intercept): + """Synthesize gaps that lie EXACTLY on gap = slope*log10(tokens)+intercept.""" + return [slope * math.log10(b) + intercept for b in LADDER] + + +def test_persists_flat_positive_edge(): + # flat, clearly-positive gap -> edge survives at scale + gaps = [0.05, 0.051, 0.049, 0.05] + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.005) + assert out["verdict_code"] == "PERSISTS" + assert abs(out["slope"]) < 1e-2 + assert out["edge_resolved"] is True + + +def test_converges_positive_edge_shrinks_to_zero(): + # gap starts at 0.08 (42M) and decays to ~0 by 1190M -> early-training speedup + # slope is negative (gap falls as log tokens rise); edge sign positive. + gaps = _gaps_from(slope=-0.05, intercept=0.05 * math.log10(LADDER[-1])) + # check it actually crosses near zero at the top rung + assert gaps[0] > 0.05 and abs(gaps[-1]) < 0.01 + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.01) + assert out["verdict_code"] == "CONVERGES" + assert out["toward_zero"] is True + assert out["edge_resolved"] is False # lands inside noise at top + + +def test_widens_edge_grows_with_scale(): + # positive edge that GROWS with scale -> advantage compounds + gaps = _gaps_from(slope=0.04, intercept=-0.20) + assert gaps[-1] > gaps[0] > 0 + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.01) + assert out["verdict_code"] == "WIDENS" + + +def test_flat_within_noise_is_inconclusive(): + # tiny gaps entirely inside the noise floor -> FLAT, no advantage either way + gaps = [0.002, -0.001, 0.001, 0.0] + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.01) + assert out["verdict_code"] == "FLAT" + assert out["edge_resolved"] is False + + +def test_negative_edge_converges_handled(): + # treatment WORSE (gap<0) but converging toward 0 -> CONVERGES (edge erodes). + gaps = _gaps_from(slope=0.05, intercept=-0.05 * math.log10(LADDER[-1])) + assert gaps[0] < -0.05 and abs(gaps[-1]) < 0.01 # starts negative, rises to ~0 + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.01) + assert out["edge_sign"] == -1 + assert out["verdict_code"] == "CONVERGES" + + +def test_slope_within_noise_not_acted_on(): + # a real positive edge with a faint negative slope whose SPAN effect is inside + # the noise floor must NOT be called CONVERGES — it stays PERSISTS. + gaps = _gaps_from(slope=-0.002, intercept=0.10) + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.01) + assert out["slope_resolved"] is False + assert out["verdict_code"] == "PERSISTS" + + +def test_descriptive_only_with_two_budgets(): + out = sl.fit_gap_trend([42e6, 1190e6], [0.05, 0.05], gap_noise=0.005) + assert out["descriptive_only"] is True + assert "DESCRIPTIVE" in out["verdict"] + + +def test_three_budgets_not_descriptive(): + out = sl.fit_gap_trend([42e6, 170e6, 670e6], [0.05, 0.05, 0.05], gap_noise=0.005) + assert out["descriptive_only"] is False + assert "DESCRIPTIVE" not in out["verdict"] + + +def test_r2_near_one_on_exact_line(): + gaps = _gaps_from(slope=-0.03, intercept=0.30) + out = sl.fit_gap_trend(LADDER, gaps, gap_noise=0.0) + assert out["r2"] == pytest.approx(1.0, abs=1e-9) + + +def test_mismatched_lengths_rejected(): + with pytest.raises(ValueError): + sl.fit_gap_trend([42e6, 170e6], [0.05]) + + +def test_single_budget_rejected(): + with pytest.raises(ValueError): + sl.fit_gap_trend([42e6], [0.05]) + + +def test_non_finite_gap_rejected(): + with pytest.raises(ValueError): + sl.fit_gap_trend(LADDER, [0.05, float("nan"), 0.05, 0.05]) + + +def test_identical_budgets_rejected(): + with pytest.raises(ValueError): + sl.fit_gap_trend([42e6, 42e6, 42e6], [0.05, 0.04, 0.06]) + + +def test_negative_gap_noise_rejected(): + with pytest.raises(ValueError): + sl.fit_gap_trend(LADDER, [0.05, 0.05, 0.05, 0.05], gap_noise=-0.1) diff --git a/research/thermal_log.py b/research/thermal_log.py new file mode 100644 index 0000000..d35c852 --- /dev/null +++ b/research/thermal_log.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Dense, crash-durable thermal logger for the GB10 (added 2026-07-08). + +The box hard-locks under sustained 420M-training load every ~5-10h with NO kernel +trace (crashkernel=0M) and — critically — its temperature was never instrumented, so +the "it's crashing from heat" hypothesis had zero data. This is a PURE-OBSERVABILITY +sampler: it has NO authority to kill anything (that stays with sentinel.py's single +hardened kill path), it only records the box's thermal envelope so we can (a) confirm +or refute the heat theory and (b) have the last-second-before-a-freeze reading on disk +for forensics. + +Every sample is flushed + fsync'd, so the final line written before an instantaneous +hard-lock is durable (the whole point). Sources (all no-sudo): nvidia-smi for the GPU +die temp / power / SM clock / thermal+power throttle flags, /sys/class/thermal for the +7 Grace-SoC ACPI zones, /proc/meminfo for the unified-pool usage. + +Usage: python3 thermal_log.py [--interval SEC] [--out PATH] [--pid TRAINER_PID] +""" +from __future__ import annotations + +import argparse +import glob +import os +import subprocess +import time +from datetime import datetime, timezone + + +def nvidia_fields(): + """gpu temp C, power W, SM clock MHz, and throttle flags. '' for any N/A field.""" + q = ("temperature.gpu,power.draw,clocks.sm," + "clocks_throttle_reasons.hw_thermal_slowdown," + "clocks_throttle_reasons.sw_thermal_slowdown," + "clocks_throttle_reasons.hw_power_brake_slowdown," + "clocks_throttle_reasons.sw_power_cap") + try: + out = subprocess.run( + ["nvidia-smi", f"--query-gpu={q}", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=10, + ).stdout + except (OSError, subprocess.TimeoutExpired): + return {} + for line in out.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 7 and parts[0]: + return { + "gpu_c": parts[0], "pwr_w": parts[1], "sm_mhz": parts[2], + "hw_therm": parts[3], "sw_therm": parts[4], + "hw_pbrake": parts[5], "sw_pcap": parts[6], + } + return {} + + +def soc_zones(): + """List of (zone_index, milli-degC-as-C) for every ACPI thermal zone.""" + zones = [] + for p in sorted(glob.glob("/sys/class/thermal/thermal_zone*/temp")): + try: + with open(p) as f: + c = int(f.read().strip()) / 1000.0 + except (OSError, ValueError): + continue + idx = p.split("thermal_zone")[1].split("/")[0] + zones.append((idx, c)) + return zones + + +def pool_usage(): + """Unified-memory pool usage fraction from /proc/meminfo, or None.""" + try: + info = {} + with open("/proc/meminfo") as f: + for line in f: + k, _, rest = line.partition(":") + info[k] = int(rest.split()[0]) # kiB + total, avail = info["MemTotal"], info["MemAvailable"] + return (total - avail) / total + except (OSError, KeyError, ValueError, IndexError): + return None + + +def pid_alive(pid): + if pid is None: + return None + try: + os.kill(pid, 0) + return True + except (ProcessLookupError, PermissionError): + return False + except OSError: + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--interval", type=float, default=10.0, help="sample period seconds") + ap.add_argument("--out", default=str( + (os.path.dirname(os.path.abspath(__file__)) or ".") + "/thermal.log")) + ap.add_argument("--pid", type=int, default=None, help="trainer pid to note liveness") + a = ap.parse_args() + + f = open(a.out, "a", buffering=1) # line-buffered + f.write(f"# thermal_log start {datetime.now(timezone.utc).isoformat()} " + f"interval={a.interval}s pid={a.pid}\n") + f.write("# ts_utc epoch gpu_c pwr_w sm_mhz throttle pool% soc_hot_c soc_zones... pid_alive\n") + f.flush(); os.fsync(f.fileno()) + + while True: + nv = nvidia_fields() + zones = soc_zones() + pool = pool_usage() + hot = max((c for _, c in zones), default=None) + # compact throttle summary: T=hw/sw thermal, P=power-brake/cap; '-' if none/N-A + thr = [] + if nv.get("hw_therm") == "Active" or nv.get("sw_therm") == "Active": + thr.append("THERMAL") + if nv.get("hw_pbrake") == "Active" or nv.get("sw_pcap") == "Active": + thr.append("POWER") + thr_s = "+".join(thr) if thr else "-" + zone_s = " ".join(f"z{idx}={c:.1f}" for idx, c in zones) + pool_s = f"{pool*100:.1f}%" if pool is not None else "na" + hot_s = f"{hot:.1f}C" if hot is not None else "na" + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + row = (f"{ts} {time.time():.0f} gpu={nv.get('gpu_c','na')}C " + f"pwr={nv.get('pwr_w','na')}W sm={nv.get('sm_mhz','na')}MHz " + f"thr={thr_s} pool={pool_s} soc_hot={hot_s} {zone_s}") + alive = pid_alive(a.pid) + if alive is not None: + row += f" pid_alive={alive}" + f.write(row + "\n") + f.flush(); os.fsync(f.fileno()) # durable: survive an instant hard-lock + time.sleep(a.interval) + + +if __name__ == "__main__": + main() From 266487af6e45a6aa494569858a22d279ad8f1ff1 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Mon, 20 Jul 2026 05:14:13 +0000 Subject: [PATCH 04/35] =?UTF-8?q?README:=20the=20scaling-persistence=20lad?= =?UTF-8?q?der=20is=20DONE=20=E2=80=94=20the=20NorMuon=20win=20converges?= =?UTF-8?q?=20away?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section claimed the four 420M rungs were "running". They finished 2026-07-12: all ten cells carry .done markers, ladder.done is set, and score_ladder.py wrote verdict.json. Replaced with the actual result, every number traced to that file. The gap (AdamW − NorMuon, BPB) shrinks with budget on both corpora — wikitext-2 +0.474 [+0.443,+0.505] at 42M, +0.126 [+0.089,+0.163] at 168M, +0.073 [−0.038,+0.184] at 420M; code_py +0.502 → +0.176 → +0.192 — with OLS slopes over log10(tokens) of −0.416 (r² 0.92) and −0.328 (r² 0.81). Verdict CONVERGES on both, ledger verdict directional. Kept the honesty that the raw shape hides: the fitted slope is resolved but the edge at the top rung is NOT (edge_resolved=false — at n=2 the 420M CI holds both "converged" and "still ahead"), the code_py gap does not even shrink monotonically (+0.176 → +0.192; only the fit is negative), and the run carries an inherited confound that pushes the same direction — both learning rates were tuned at the 42M horizon and never re-tuned per budget, so some of the fade could be a mis-tuned-LR artifact rather than convergence. Stated what it would take to earn more: a 3rd 420M seed, an 840M rung, a per-horizon LR check. Also closed the loop on the thermal note: a 2026-07-09 hot spell had the ladder stuck at 4/10 cells with no net progress, and after the crash-survival work the four 420M rungs finished 2026-07-10→12 *through* repeated thermal kills. The recovery chain is load-bearing, not theoretical. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9e9f236..87e9338 100644 --- a/README.md +++ b/README.md @@ -101,13 +101,29 @@ with the Chen-2021 estimator, on decontaminated GSM8K + MATH-500) and tested it. The reasoning capability lives in SFT/distillation, not RL at this scale — the gate saved a multi-seed cohort before it was spent. -**7 · Scaling persistence of the NorMuon win (in progress).** Study #2 attributed +**7 · Scaling persistence of the NorMuon win (done, 2026-07-12).** Study #2 attributed the IMU-1 win largely to **NorMuon**; this ladder asks whether its **+0.474 wikitext BPB** edge over AdamW **persists or converges with budget**. At fixed N=596M it sweeps the token budget — 42M (reused) + **168M ×{NorMuon,AdamW}×3 seeds** -+ **420M ×2 seeds** — varying only `--steps`. The six 168M rungs are **done**; the -four 420M rungs (~16–18 h each) are **running**. Honest ceiling: **directional** — -the 420M top rung is n=2 (< 3 seeds); a headline needs a 3rd seed and/or an 840M rung. ++ **420M ×2 seeds**, ten cells — varying only `--steps`. All ten completed. + +- **The gap shrinks with budget, and both corpora agree.** wikitext-2 + (AdamW − NorMuon, BPB): **+0.474** [+0.443, +0.505] at 42M → **+0.126** + [+0.089, +0.163] at 168M → **+0.073** [−0.038, +0.184] at 420M — significant at + the two smaller budgets, **not significant** at the top. code_py: +0.502 → +0.176 + → +0.192. OLS over log10(tokens) gives slope **−0.416** (r² 0.92) on wikitext and + **−0.328** (r² 0.81) on code → **CONVERGES** on both. +- **Verdict: directional, not a headline** — the 420M rung is n=2 (< 3 seeds, §C17). +- **What is resolved, and what isn't.** The *slope* is resolved; the *edge at the top + rung* is not. At n=2 the 420M CI is wide enough to hold both "converged" and "still + ahead", and the code_py gap does not even shrink monotonically (+0.176 → +0.192 — + only the fitted slope is negative). A disclosed **inherited confound** cuts the same + way: both learning rates were tuned at the 42M horizon and never re-tuned per budget, + so part of the fade may be a mis-tuned-LR artifact rather than true convergence. + Earning more needs a 3rd 420M seed, an 840M rung, and a per-horizon LR check. +- Read it as: NorMuon looks like an **early-training speedup that converges away** — + exactly what IMU-1's own Limitation #3 warned it might be. Numbers: + `experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json`. > **GB10 thermal-survival note (2026-07-08→10).** The 420M rungs surfaced a hardware > reality: under sustained load in warm ambient the unified Grace-Blackwell package @@ -121,7 +137,9 @@ the 420M top rung is n=2 (< 3 seeds); a headline needs a 3rd seed and/or an 840M > records the full temperature envelope; and `run_ladder.sh` loops-until-done behind a > cool-down gate, backed by an `@reboot` auto-resume (`boot_resume.sh`). Net: the > multi-day ladder now survives each thermal event by losing ≤~15 min (one checkpoint -> interval) instead of a whole rung. Cooling the box's ambient is the high-leverage +> interval) instead of a whole rung. **It worked:** a 2026-07-09 hot spell had left the +> ladder stuck at 4/10 cells missing with no net progress, and the four 420M rungs then +> finished 2026-07-10→12 *through* repeated thermal kills. Cooling the box's ambient is the high-leverage > throughput fix — cooling *time* after a kill is only ~10–30 s; it's the ~3 min reheat > to 90 °C under warm ambient that throttles daytime throughput. From c43d022cd281bbb3f4b879a6a948f7b767d255b7 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Tue, 21 Jul 2026 14:56:54 +0000 Subject: [PATCH 05/35] Fix score_ladder.py's ledger write: invalid --type, and the silence that hid it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--type pretrain-ablation` is not a run type — it is an OBJECTIVE. argparse rejected the call with exit 2 on every scoring pass, so the scaling-persistence ladder was scored and never landed in the ledger. The entry that exists was put there by hand afterwards, which is why it carried started=2026-07-13 AFTER ended=2026-07-12. The typo is the small half. The reason a broken ledger write survived a multi-day ladder is that the call passed `check=False` inside a `try/except Exception`: a non-zero exit raises nothing, so nothing was caught, nothing was printed, and the script still returned 0. A failure that cannot be seen is worse than a crash. sync_ledger() now: * passes `--type scaling-fit` (a token-budget ladder IS a scaling-fit run), keeping `--objective pretrain-ablation` where it belongs; * captures the exit code and prints an unmissable banner naming the exit status, the full quoted argv and the CLI's own error, then returns False; * is idempotent — add-run REJECTS a duplicate run_id (exit 2), so a re-scored ladder falls back to update-run instead of losing the write; * records real dates and a terminal state: `started` parsed from the first dated START line in run_ladder.log, `ended` from the ladder.done mtime, and status=done (add-run defaults to `launched`), instead of letting both dates default to today. Verified against a sandbox copy of the ledger, all three paths: add-run on a fresh ledger creates type=scaling-fit with started(2026-07-05) <= ended (2026-07-12); a second call falls through to update-run and is idempotent; and an injected bad flag produces the loud banner and False rather than silence. The old argv was re-run against ledger.py to confirm it is still rejected — the fix was necessary, not cosmetic. Regression guard in test_ledger.py: a repo-wide lint asserting every literal --type handed to ledger.py is in RUN_TYPES, plus three tests that keep the lint honest — it must not be vacuous, it must flag the original broken argv shape (proven on a synthetic caller, not by reverting the real one), and it must not flag the fixed shape. Its one blind spot is documented in the docstring: a caller that assembles the CLI path piecewise would not be scanned. Also corrected the live ledger entry's inverted dates via ledger.py. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../score_ladder.py | 261 ++++++++++++++++++ research/tests/test_ledger.py | 80 ++++++ 2 files changed, 341 insertions(+) create mode 100644 Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/score_ladder.py diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/score_ladder.py b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/score_ladder.py new file mode 100644 index 0000000..bc5912f --- /dev/null +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/score_ladder.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Score the scaling-persistence ladder → the verdict on whether NorMuon's +0.474 BPB win +PERSISTS or CONVERGES with token budget (closes the IMU-1 RESULT.md Limitation #3). + +Per token budget: compute the AdamW−NorMuon BPB gap on wikitext-2 + code (text-lm-v2, reusing the +IMU-1 score_cohort scorer so numbers are byte-identical/comparable), with across-seed CI via the +tested eval_stats.seed_delta_significant; the 42M rung REUSES the existing IMU-1 cohort_bpb.json (no +retrain); then scaling_ladder.fit_gap_trend() over log10(tokens) → {PERSISTS, WIDENS, CONVERGES, FLAT}. + +Runs on the GB10 AFTER the ladder cells finish (run_ladder.sh calls it; GPU is free by then). Robust: +scores whatever budgets already have >=2 seeds/arm, so it can also be run mid-cohort for an early read. +Writes verdict.json + registers the ledger run. safe_cuda-guarded; sequential (parallel GPU evals crash). +""" +from __future__ import annotations +import json, math, pathlib, re, shlex, subprocess, sys, time + +ROOT = pathlib.Path("/home/yashb98/Downloads/BuildFromScratch") +IMU1 = ROOT / "Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw" +LDIR = ROOT / "Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence" +RES = IMU1 / "results" # train_ablation writes checkpoint_persist_*.pt here +for p in (IMU1, ROOT, ROOT / "research", ROOT / "Qwen3-0.6B"): + sys.path.insert(0, str(p)) +import safe_cuda # noqa: E402 +import torch # noqa: E402 +from transformers import AutoTokenizer # noqa: E402 +from model import Qwen3Config, Qwen3ForCausalLM # noqa: E402 +from eval_stats import seed_delta_significant # noqa: E402 +from scaling_ladder import fit_gap_trend # noqa: E402 +import score_cohort as sc # noqa: E402 (reuse score() + load_corpora(): text-lm-v2 SEQ/STRIDE/bpb) + +BUDGETS = [42_000_000, 168_000_000, 420_000_000, 840_000_000] +SEEDS = {42_000_000: [0, 1, 2], 168_000_000: [0, 1, 2], 420_000_000: [0, 1], 840_000_000: [0, 1]} +CORPORA = ["wikitext2_val", "code_py"] +tagM = lambda t: f"{t // 1_000_000}M" +LEDGER = ROOT / "research/ledger/ledger.py" + + +def _ladder_started(): + """Ladder start = the first DATED `START ` line in run_ladder.log. Without it + `add-run` defaults `started` to TODAY — exactly how this run's entry ended up with + started(2026-07-13) AFTER ended(2026-07-12). Returns None if unreadable.""" + try: + for line in (LDIR / "run_ladder.log").read_text(errors="replace").splitlines(): + m = re.match(r"\[(\d{4}-\d{2}-\d{2})[ T][^\]]*\]\s+START\b", line) + if m: + return m.group(1) + except OSError: + pass + return None + + +def _ladder_ended(): + """Run end = the mtime of `ladder.done`, which run_ladder.sh touches only after the + completion gate sees every cell's .done. Falls back to today (scoring date).""" + try: + return time.strftime("%Y-%m-%d", + time.localtime((LDIR / "ladder.done").stat().st_mtime)) + except OSError: + return time.strftime("%Y-%m-%d") + + +def sync_ledger(verdict, ledger_verdict, metrics) -> bool: + """Land the scored run in the ledger through ledger.py (§C11 — never hand-edit). + + Four things this gets right that the previous one-liner did not: + * `--type scaling-fit`. A token-budget ladder IS a scaling-fit run; + `pretrain-ablation` is an OBJECTIVE, not a run type, so passing it as --type made + argparse reject the whole call (exit 2) and NOTHING was ever written. + * The failure is LOUD. The old call passed `check=False`, so a non-zero exit raised + nothing and printed nothing — the write had been failing on every scoring pass, + invisibly, while the script still returned 0. That silence is why a broken ledger + call survived a multi-day ladder. + * Idempotent. `add-run` REJECTS a duplicate run_id (exit 2), so a re-scored ladder + falls back to `update-run` rather than losing the write. + * Real dates + terminal state. `started`/`ended` come from the log and the + `ladder.done` marker instead of defaulting to today, and the run is marked + `status=done` (add-run defaults to `launched`). + """ + started, ended = _ladder_started(), _ladder_ended() + sets = ["--set", "lifecycle_stage=scaling", + "--set", "status=done", + "--set", f"ended={ended}", + "--set", f"suite_version={verdict['suite_version']}", + "--set", f"verdict={ledger_verdict}", + "--set", f"metrics={json.dumps(metrics)}"] + if started: + sets += ["--set", f"started={started}"] + + proc = subprocess.run( + ["python3", str(LEDGER), "add-run", "--run-id", verdict["run_id"], + "--type", "scaling-fit", "--model-dir", "Qwen3-0.6B", + "--objective", verdict["objective"]] + sets, + capture_output=True, text=True) + if proc.returncode == 2 and "already exists" in (proc.stdout + proc.stderr): + proc = subprocess.run( + ["python3", str(LEDGER), "update-run", verdict["run_id"]] + sets, + capture_output=True, text=True) + + if proc.returncode != 0: + bar = "!" * 78 + print(f"\n{bar}\n" + f"LEDGER WRITE FAILED — verdict.json IS on disk, the ledger is NOT updated.\n" + f" exit : {proc.returncode}\n" + f" cmd : {' '.join(shlex.quote(c) for c in proc.args)}\n" + f" error: {(proc.stderr or proc.stdout).strip()[:400]}\n" + f" fix : re-run this scorer, or land it by hand via research/ledger/ledger.py\n" + f"{bar}\n", flush=True) + return False + print(f"ledger: {verdict['run_id']} verdict={ledger_verdict} " + f"started={started or 'today'} ended={ended}", flush=True) + return True + + +def score_ckpt(model, ckpt, corpora, tok): + sd = torch.load(ckpt, map_location="cpu", weights_only=False)["model"] + sd = {(k[10:] if k.startswith("_orig_mod.") else k): v for k, v in sd.items()} + model.load_state_dict(sd, strict=True) + return {c: sc.score(model, ids, tok)["bpb"] for c, ids in corpora.items()} + + +def main(): + safe_cuda.guard(0.85) + tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base") + corpora = sc.load_corpora(tok) + model = Qwen3ForCausalLM(Qwen3Config()).to(sc.DEVICE, sc.DTYPE).eval() + reuse = json.loads((RES / "cohort_bpb.json").read_text())["cells"] # 42M rung + + # collect per-budget BPBs (skip a budget until both arms have >=2 seeds) + per_budget = {} + for t in BUDGETS: + got = {"adamw": {}, "normuon": {}} + for arm in ("adamw", "normuon"): + for s in SEEDS[t]: + if t == 42_000_000: + cell = f"{arm}_seed{s}" + if cell in reuse: + got[arm][s] = {c: reuse[cell][c]["bpb"] for c in CORPORA} + else: + ck = RES / f"checkpoint_persist_{tagM(t)}_{arm}_s{s}.pt" + if ck.exists(): + t0 = time.time() + got[arm][s] = score_ckpt(model, ck, corpora, tok) + print(f" scored persist_{tagM(t)}_{arm}_s{s}: " + f"wt={got[arm][s]['wikitext2_val']:.4f} code={got[arm][s]['code_py']:.4f} " + f"({time.time()-t0:.0f}s)", flush=True) + if len(got["adamw"]) >= 2 and len(got["normuon"]) >= 2: + per_budget[t] = got + + # per-budget gap + CI, per corpus + gap_by_corpus = {} + for c in CORPORA: + pts = [] + for t in sorted(per_budget): + a = [per_budget[t]["adamw"][s][c] for s in sorted(per_budget[t]["adamw"])] + n = [per_budget[t]["normuon"][s][c] for s in sorted(per_budget[t]["normuon"])] + r = seed_delta_significant(a, n, direction="lower_is_better") + pts.append({"tokens": t, "budget": tagM(t), "gap_bpb": r["improvement"], + "ci95": r["ci95"], "significant": r["significant"], + "adamw_mean": sum(a) / len(a), "normuon_mean": sum(n) / len(n), + "n_seeds": [len(a), len(n)], "warning": r["warning"]}) + gap_by_corpus[c] = pts + + # Fit the trend on BOTH corpora (a headline needs BPB on >=2 corpora, §C10/§C16 — the old + # code fit wikitext ONLY, so a divergent code trend was silently missed). Noise floor = the + # MAX per-cell CI half-width across rungs, NOT the median: the median discards the noisiest + # (here the decisive n=2 420M top) rung and yields an anti-conservative "resolved" verdict. + def _fit_corpus(pts): + if len(pts) < 2: + return {"verdict": "descriptive_only", "verdict_code": "descriptive_only", + "reason": "need >=2 budgets scored"} + hw = [(p["ci95"][1] - p["ci95"][0]) / 2 for p in pts + if p["ci95"] and all(isinstance(x, (int, float)) and math.isfinite(x) for x in p["ci95"])] + noise = max(hw) if hw else 0.0 + tr = fit_gap_trend([p["tokens"] for p in pts], [p["gap_bpb"] for p in pts], gap_noise=noise) + tr["gap_noise_rule"] = "max per-cell CI half-width (conservative; keeps the n=2 top rung's uncertainty)" + return tr + + trend_by_corpus = {c: _fit_corpus(gap_by_corpus[c]) for c in CORPORA} + trend = trend_by_corpus["wikitext2_val"] # wikitext-2 = the headline corpus + v = trend.get("verdict_code", trend.get("verdict", "?")) + + # §C17/§C25 CAP: the "at scale" claim rests on the LARGEST scored budget. If that rung has + # <3 seeds in EITHER arm (the 420M rung is n=2), the headline caps to DIRECTIONAL regardless + # of what the 3-point OLS says — a trend fit cannot out-rank the seeds>=3 rule. + scored_budgets = sorted(per_budget) + top_t = scored_budgets[-1] if scored_budgets else None + top_seeds = ([len(per_budget[top_t]["adamw"]), len(per_budget[top_t]["normuon"])] + if top_t is not None else [0, 0]) + headline_capped = (min(top_seeds) < 3) if top_t is not None else True + codes = {trend_by_corpus[c].get("verdict_code") for c in CORPORA} + codes.discard("descriptive_only"); codes.discard(None) + corpora_agree = len(codes) <= 1 + + concl_map = { + "PERSISTS": "The NorMuon 2D-weight advantage PERSISTS with budget (edge at the top rung > noise, slope flat).", + "WIDENS": "The NorMuon advantage WIDENS with budget.", + "CONVERGES": "The NorMuon advantage CONVERGES toward 0 with budget — an early-training speedup, as the IMU-1 RESULT.md Limitation #3 predicted it might.", + "FLAT": "Inconclusive: slope within noise and no resolvable edge at the largest budget.", + "descriptive_only": "Descriptive only — not enough budgets scored yet for a trend verdict.", + } + concl = concl_map.get(v, f"trend verdict={v}") + if headline_capped and v not in ("descriptive_only", "FLAT"): + concl = (f"DIRECTIONAL ({v} shape on wikitext-2): the top budget ({tagM(top_t)}) rung is " + f"n={min(top_seeds)} seeds (<3, §C17), so this is a directional trend, NOT a headline " + f"win — add a 3rd 420M seed (and the 840M rung) to earn more. Shape: {concl}") + if not corpora_agree: + concl += (f" WARNING: corpora disagree — wikitext-2={trend_by_corpus['wikitext2_val'].get('verdict_code')} " + f"vs code_py={trend_by_corpus['code_py'].get('verdict_code')}; treat with extra caution.") + + # Ledger verdict: DIRECTIONAL unless a FULL-strength trend (>=3 budgets, top rung n>=3, both + # corpora agree, and a resolvable non-FLAT code). With 420M at n=2 this stays directional. + full_strength = (not headline_capped and corpora_agree + and v not in ("descriptive_only", "FLAT", "?")) + ledger_verdict = v.lower() if full_strength else "directional" + + verdict = { + "run_id": "2026-07-05_qwen3-0.6b_scaling-persistence", "lifecycle_stage": "scaling", + "objective": "pretrain-ablation", "suite_version": "text-lm-v2", + "question": "Does NorMuon's +0.474 BPB win over AdamW (2D weights, fixed N=596M) persist or converge with token budget?", + "budgets_scored": [tagM(t) for t in scored_budgets], + "top_budget": (tagM(top_t) if top_t is not None else None), + "top_budget_seeds": top_seeds, + "headline_capped_to_directional": headline_capped, + "cap_reason": (f"top budget {tagM(top_t)} rung is n={min(top_seeds)} (<3 seeds, §C17)" + if headline_capped else "top rung has >=3 seeds"), + "corpora_agree": corpora_agree, + "gap_by_corpus": gap_by_corpus, + "trend_by_corpus": trend_by_corpus, + "trend_wikitext2": trend, "trend_verdict": v, + "ledger_verdict": ledger_verdict, + "conclusion": concl, + "honesty": ("Achieved tok/s only, never %-of-peak MFU (GB10 peak estimated, §C24). A measured " + "CONVERGES is a PASSING result. Caps to DIRECTIONAL while the 420M top rung is n=2 " + "(<§C17 seeds>=3) and 840M is absent. Noise floor = MAX per-cell CI half-width (keeps " + "the n=2 rung, not the median). Gaps use the same unpaired-Welch test as IMU-1's 42M " + "headline (byte-comparable); the design is paired-by-seed, so a paired-t on per-seed " + "diffs is the stricter test (disclosed caveat). Inherited confound: AdamW/NorMuon LRs " + "tuned at 42M, not re-tuned per horizon (RESULT.md Limitation #1)."), + } + (LDIR / "verdict.json").write_text(json.dumps(verdict, indent=2, default=str)) + print(f"\nTREND wikitext-2={v} | code_py={trend_by_corpus['code_py'].get('verdict_code')} " + f"| corpora_agree={corpora_agree} | headline_capped={headline_capped} -> ledger={ledger_verdict}", + flush=True) + print(concl, flush=True) + print("budgets scored:", verdict["budgets_scored"], "top_seeds:", top_seeds, flush=True) + + # ledger (via ledger.py; never hand-edit) + try: + metrics = {"trend_verdict_wikitext": v, + "trend_code_py": trend_by_corpus["code_py"].get("verdict_code"), + "budgets": verdict["budgets_scored"], "top_budget_seeds": top_seeds, + "headline_capped": headline_capped, "corpora_agree": corpora_agree, + "conclusion": concl} + sync_ledger(verdict, ledger_verdict, metrics) + except Exception as e: # OSError etc.; a bad exit is handled inside + print("ledger write note:", repr(e)[:200], flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/tests/test_ledger.py b/research/tests/test_ledger.py index 1ae2d7a..44c6654 100644 --- a/research/tests/test_ledger.py +++ b/research/tests/test_ledger.py @@ -8,6 +8,7 @@ """ import hashlib import json +import pathlib import re import ledger @@ -303,3 +304,82 @@ def test_save_preserves_file_mode(ledger_path): ledger_path.chmod(0o640) run(ledger_path, "add-technique", "--slug", "t", "--title", "X") assert (ledger_path.stat().st_mode & 0o777) == 0o640 + + +# ------------------------------------------------- caller contract (repo-wide lint) +# 2026-07-20: score_ladder.py shipped `--type pretrain-ablation` — that is an OBJECTIVE, +# not a run type. argparse rejected every invocation (exit 2), and the caller passed +# `check=False`, so the ledger write failed SILENTLY on every scoring pass of a multi-day +# ladder while the script still returned 0. The scaling-persistence result was scored and +# never landed. A wrong --type must break a test, not a production run. + +REPO = pathlib.Path(ledger.__file__).resolve().parents[2] +_PY_TYPE = re.compile(r"""['"]--type['"]\s*,\s*['"]([^'"]+)['"]""") +_SH_TYPE = re.compile(r"--type[ \t]+['\"]?([A-Za-z][\w-]*)") + + +def _ledger_type_literals(root=None): + """Every literal `--type` value handed to ledger.py by a caller under `root` + (default: this repo). Skips ledger.py itself (it DEFINES the choices) and the test + tree (which passes invalid values on purpose to assert they are rejected). + + Known limitation, stated rather than hidden: a file only qualifies if the literal + text "ledger.py" appears in it, so a caller that assembles the path piecewise + (`ROOT / "research" / "ledger" / "ledger.py"`) is invisible to this scan. Every + current caller names it literally, and test_the_run_type_lint_is_not_vacuous fails + loudly if that ever stops being true for all of them at once.""" + root = REPO if root is None else pathlib.Path(root) + for path in sorted(root.rglob("*.py")) + sorted(root.rglob("*.sh")): + parts = set(path.parts) + if parts & {".git", "__pycache__", "tests"} or path.name == "ledger.py": + continue + try: + text = path.read_text(errors="replace") + except OSError: + continue + if "ledger.py" not in text or "add-run" not in text: + continue + rx = _PY_TYPE if path.suffix == ".py" else _SH_TYPE + for m in rx.finditer(text): + yield path.relative_to(root), m.group(1) + + +def test_every_caller_passes_a_valid_run_type(): + bad = [(p, t) for p, t in _ledger_type_literals() if t not in ledger.RUN_TYPES] + assert not bad, ("caller(s) passing an invalid ledger --type: " + + "; ".join(f"{p} -> {t!r}" for p, t in bad) + + f" (valid: {sorted(ledger.RUN_TYPES)})") + + +def test_the_run_type_lint_is_not_vacuous(): + """Guard the guard: if the scan ever matches nothing, the test above passes for the + wrong reason and this whole check silently stops protecting anything.""" + assert list(_ledger_type_literals()), "repo-wide --type scan matched no caller at all" + + +def test_the_run_type_lint_catches_the_original_bug(tmp_path): + """Prove the lint DETECTS the real defect, on a synthetic caller rather than by + reverting the fixed one. Reproduces score_ladder.py's exact broken argv shape.""" + (tmp_path / "caller.py").write_text( + 'subprocess.run(["python3", str(ROOT / "research/ledger/ledger.py"), "add-run",\n' + ' "--run-id", rid, "--type", "pretrain-ablation",\n' + ' "--model-dir", "Qwen3-0.6B"], check=False)\n') + (tmp_path / "caller.sh").write_text( + 'python3 research/ledger/ledger.py add-run --run-id "$RID" --type pretrain-ablation\n') + found = dict(_ledger_type_literals(tmp_path)) + assert {p.name for p in found} == {"caller.py", "caller.sh"}, found + assert set(found.values()) == {"pretrain-ablation"} + assert all(t not in ledger.RUN_TYPES for t in found.values()) # ...and flagged invalid + + +def test_the_run_type_lint_accepts_a_valid_caller(tmp_path): + """No false positives: the FIXED argv shape must not be flagged. Mirrors the real + score_ladder.py, which reaches the CLI through a `LEDGER = ROOT / ".../ledger.py"` + constant — the literal still appears in the file, which is what the scan keys on.""" + (tmp_path / "ok.py").write_text( + 'LEDGER = ROOT / "research/ledger/ledger.py"\n' + 'subprocess.run([str(LEDGER), "add-run", "--run-id", rid,\n' + ' "--type", "scaling-fit", "--objective", "pretrain-ablation"])\n') + found = dict(_ledger_type_literals(tmp_path)) + assert list(found.values()) == ["scaling-fit"] + assert all(t in ledger.RUN_TYPES for t in found.values()) From 75791ff6f7fe2834d6b9d6055413bc8cb3398975 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Tue, 21 Jul 2026 22:11:45 +0000 Subject: [PATCH 06/35] HybridSSM arch ladder: fix the resume budget bug, then queue all 15 cells to run continuously MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things had to be right before queuing 5.9 GPU-days, and two of them were wrong. 1. RESUME INFLATED THE BUDGET. train_hybrid.py looped `range(start_step, start_step + steps)`, so a resumed cell repeated the whole budget from its resume point — the pilot arm resumed at 400 and ran 21,156 steps (173.3M tok vs a declared 170.0M, +1.93%), and ran the optax cosine schedule past its end. Harmless at n=1; fatal for a ladder, because only the arms that happen to crash would get extra compute, in proportion to where they crashed, with nothing in the logs saying so. Now `range(start_step, steps)`: total steps are independent of the resume point. 2. TWO ARMS WOULD HAVE BEEN SILENT DUPLICATES. The first draft queued mixer="full" and mixer="none", but model.py dispatches `if is_full(i) -> attention; elif mixer=="swa128" -> SWA; else -> SSM`, so both fall into the else branch and are byte-identical to the base arm. That was ~33 GPU-h of duplicates. The all-attention control is expressed as attn_every=1 instead. An all-SSM arm turns out to be unreachable at all (is_full(i) = i % attn_every == 0 makes layer 0 full attention for every attn_every >= 1) and is documented as such rather than faked. 3. THE ARMS WERE NOT COMPARABLE. Non-embed params differ by 10-20% across arms (151.0M to 208.2M), so an iso-token ladder would have violated §C18's 5% tolerance and could never have been called a result. Token budgets are now matched on TOTAL train FLOPs, measured from the real built configs (6N + 12*L_full*H*Dh*T + 12*L_swa*H*Dh*w): worst mismatch 0.17%. The quadratic-attention term turns out to cancel the param difference for attn1to3 and fullattn, while swa128 needs 1.129x tokens; rungs were chosen so even that arm fits inside the 170.03M cache, so no arm wraps into a second epoch. The queue is 15 cells — 5 single-variable arms x 3 rungs (42M/85M/150M base-equivalent), n=1 scout, ordered CHEAP RUNG FIRST so the 42M rung yields a complete 5-arm comparison in ~22 h and partial completion is still a result. Seeds 1-2 are appended later only for arms that separate (§C17). run_arch_ladder.sh is the continuous driver: .done markers make it idempotent, each cell gets its own sentinel on the real python pid, a cool-down gate keeps it off a hot box, and it loops passes until every cell is done with an escalating hot-spell backoff and a completion gate. Relaunching it is always safe, so recovery needs no human decision. Its §C4.5 guard also had to be fixed after the first launch attempt deferred all 15 cells: a bare `pgrep -f train_*.py` matches any process merely MENTIONING a trainer, including the monitoring command watching it. sentinel's own preflight reported trainers=none on the same second. The guard now requires argv[0] to be a python interpreter, unit-tested against a shell decoy, a real python trainer, and a clean box. §C5.0 smoke: all 5 arm configs through the exact script, exit 0, 8.79-8.94 -> 0.0016-0.0054, ckpt round-trip max|Δ|=0.00e+00 — the only evidence attn_every=1 works, since verify.py's toggle sweep only covered attn_every in {2,4}. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../BUILD_STATUS.md | 47 +++- .../2026-07-19_hybrid-ssm-0.2b_build/train.py | 18 +- .../train_hybrid.py | 18 +- .../c5_evidence.json | 91 +++++++ .../cells.json | 246 ++++++++++++++++++ .../run_arch_ladder.sh | 155 +++++++++++ 6 files changed, 559 insertions(+), 16 deletions(-) create mode 100644 HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json create mode 100644 HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/cells.json create mode 100755 HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md index 6a1b4d9..cba6a89 100644 --- a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/BUILD_STATUS.md @@ -21,7 +21,39 @@ first pretrain arm (`ssm_base_s0`) is IN FLIGHT on real data.** - **Fit probes on real data** (`probe.log` / `probe2.log` / `probe3.log`, 15 / 12 / 30 steps): step-0 loss 12.4317 / 12.4312 / 12.4312 ≈ ln(151936)=11.93 + init noise, and 30 steps moves 12.43 → 8.42. -## Pretrain arm `ssm_base_s0` — IN FLIGHT +## Pretrain arm `ssm_base_s0` — COMPLETE (2026-07-20 15:04 UTC) + +Exited cleanly on its own: `[done] 21156 steps · final loss=3.8149`, `arm_ssm_base_s0.done` +written, final checkpoint saved, and the sentinel disarmed itself (`watched pid 3164922 +exited on its own; disarming (no kill)`). Wall clock **990 min / 16.5 h** for the final +process — this excludes the killed first attempt, whose start time is not on disk, so +total GPU time is a lower bound. + +| result | value | +|---|---| +| final train loss | **3.8149** | +| best val loss | **3.7839** @ step 19,200 (final eval 3.9245 @ 20,800 — noisy tail) | +| eval-harness `text-lm-v2` | PPL wikitext2_val **133.4628**, code_py **5142.6426** (`self_floor=true`; corpora pinned `wikitext-2-raw-v1:validation@b08601e`, `codeparrot-clean-valid@4db92d2`) | +| verdict | **directional** — n=1 seed, no comparand, no iso-FLOP match (§C17/§C18/§C25) | + +**Verify gate CLOSED.** `verify.py` was re-run 2026-07-20 17:46 against the post-`nn.remat` +`model.py` (last modified 07-19 22:27) — `verify.log`, 6/6 PASS, exit 0: scan-vs-reference +max|Δ|=2.38e-07, chunked-vs-naive CE |Δ|=4.77e-05, param count, forward finite/deterministic, +all 8 toggle combos finite. + +### ⚠️ Budget overshoot — a resume bug, and it matters for the ladder + +`train_hybrid.py:129` is `for s in range(start_step, start_step + steps)`. A **resumed** run +therefore repeats the FULL step budget from the resume point instead of finishing the +original one. This arm resumed at step 400, so it ran **21,156 steps = 173,309,952 tokens +against a declared budget of 170,034,304 (+1.93%)**, wrapping ~1.9% into a second epoch. + +Harmless at n=1, but it silently breaks **§C18 iso-FLOP** across arms: any arm that crashes +and resumes gets *more* compute than one that doesn't, scaling with the resume point — a +resume at step 5,000 would be **+24%**, far past the 5% tolerance, and nothing in the logs +would flag it. **Fix to `range(start_step, steps)` before running the ladder.** + +## Pretrain arm `ssm_base_s0` — configuration as launched Ledger run `2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0` (type=ablation, status=running, lifecycle_stage=architecture, framework=jax, technique `hybrid-attention-rethink`). @@ -53,14 +85,13 @@ pool 81% → ~40%. Resumed from the step-400 checkpoint at 22:34:29 and has run This was a *manual* recovery behind a config change, i.e. the §C5/S1-4a "not safe to auto-resume at the same config" path — `loop_state.auto_resumes` correctly stayed at 0. -## ⚠️ Open gate gap (must close before this arm is scored) +## Gate gap — CLOSED 2026-07-20 -`verify.py` last ran **2026-07-19 12:52** (per this file's previous revision — no verify log was captured -to disk). `model.py` was last modified **2026-07-19 22:27** to add `nn.remat`. **The verify gate has not -been re-run against the model that is actually training.** `nn.remat` is semantically identity -(rematerialization trades recompute for memory and must not change values), but "must not" is not -"verified on this box". Re-run `verify.py` and capture its output to `verify.log` **after** this arm -finishes — it is GPU work, and §C4.5 forbids co-running it beside the live trainer. +For the record, since it was flagged as blocking while the arm ran: `verify.py`'s original PASS +(2026-07-19 12:52) predated the 22:27 `nn.remat` change, so for the whole run the gate was stale +against the model actually training. It was re-run **2026-07-20 17:46**, after the arm finished +(GPU work — §C4.5 forbids co-running it beside a live trainer), and captured to `verify.log`: +**6/6 PASS, exit 0**. `nn.remat` is confirmed value-preserving here, as expected. ## Next diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py index 62afa4a..96c19cf 100644 --- a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train.py @@ -126,7 +126,14 @@ def save(step_i): fixed = make_batch(jax.random.PRNGKey(1234), B, T, cfg.vocab_size) if a.smoke else None losses = [] - for s in range(start_step, start_step + steps): + # NOTE (2026-07-21 fix): the bound is `steps`, NOT `start_step + steps`. The old form + # made a RESUMED run repeat the whole budget from the resume point — arm ssm_base_s0 + # resumed at 400 and ran 21,156 steps (173.3M tok) against a declared 170.0M (+1.93%). + # Harmless at n=1, fatal for §C18 iso-FLOP across a ladder: an arm that crashes and + # resumes would silently receive more compute than one that doesn't, in proportion to + # its resume point, and nothing in the logs would say so. It also ran the optax cosine + # schedule (decay_steps=steps) past its end for those extra steps. + for s in range(start_step, steps): rng, sk = jax.random.split(rng) if a.smoke: ids, tgt = fixed # overfit one fixed batch @@ -149,14 +156,17 @@ def save(step_i): save(s + 1) if not a.smoke: - save(start_step + steps) + save(steps) if a.done_marker: pathlib.Path(a.done_marker).touch() - print(f"[done] {start_step + steps} steps · final loss={losses[-1]:.4f}", flush=True) + # `losses` is empty when a resume finds the cell already complete (start_step >= steps), + # which is the idempotent no-op the ladder driver relies on — don't IndexError on it. + final = f"{losses[-1]:.4f}" if losses else "n/a (already complete at resume)" + print(f"[done] {steps} steps · final loss={final}", flush=True) if a.smoke: # ckpt round-trip: save, reload into a fresh param tree, assert identical - save(start_step + steps) + save(steps) with open(a.ckpt, "rb") as f: blob = pickle.load(f) fresh = net.init(jax.random.PRNGKey(99), jnp.asarray(ids0))["params"] diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py index 62afa4a..96c19cf 100644 --- a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py @@ -126,7 +126,14 @@ def save(step_i): fixed = make_batch(jax.random.PRNGKey(1234), B, T, cfg.vocab_size) if a.smoke else None losses = [] - for s in range(start_step, start_step + steps): + # NOTE (2026-07-21 fix): the bound is `steps`, NOT `start_step + steps`. The old form + # made a RESUMED run repeat the whole budget from the resume point — arm ssm_base_s0 + # resumed at 400 and ran 21,156 steps (173.3M tok) against a declared 170.0M (+1.93%). + # Harmless at n=1, fatal for §C18 iso-FLOP across a ladder: an arm that crashes and + # resumes would silently receive more compute than one that doesn't, in proportion to + # its resume point, and nothing in the logs would say so. It also ran the optax cosine + # schedule (decay_steps=steps) past its end for those extra steps. + for s in range(start_step, steps): rng, sk = jax.random.split(rng) if a.smoke: ids, tgt = fixed # overfit one fixed batch @@ -149,14 +156,17 @@ def save(step_i): save(s + 1) if not a.smoke: - save(start_step + steps) + save(steps) if a.done_marker: pathlib.Path(a.done_marker).touch() - print(f"[done] {start_step + steps} steps · final loss={losses[-1]:.4f}", flush=True) + # `losses` is empty when a resume finds the cell already complete (start_step >= steps), + # which is the idempotent no-op the ladder driver relies on — don't IndexError on it. + final = f"{losses[-1]:.4f}" if losses else "n/a (already complete at resume)" + print(f"[done] {steps} steps · final loss={final}", flush=True) if a.smoke: # ckpt round-trip: save, reload into a fresh param tree, assert identical - save(start_step + steps) + save(steps) with open(a.ckpt, "rb") as f: blob = pickle.load(f) fresh = net.init(jax.random.PRNGKey(99), jnp.asarray(ids0))["params"] diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json new file mode 100644 index 0000000..2a8973c --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json @@ -0,0 +1,91 @@ +{ + "run_id": "2026-07-21_hybrid-ssm-0.2b_arch-ladder", + "model_dir": "HybridSSM-0.2B", "lifecycle_stage": "architecture", "objective": "pretrain-ablation", "framework": "jax", + "technique_slug": "hybrid-attention-rethink", + + "purpose": "The architecture study ARCHITECTURE.md is designed around: does the efficient-mixer choice change WHAT the model reaches, or only HOW FAST it gets there? 5 single-variable arms x 3 iso-FLOP budget rungs, scored as an emergence-speed curve. Phase 1 is a scout at n=1 seed; seeds 1-2 are appended later ONLY for arms that separate (§C17).", + + "arm_plan": { + "base": "mixer=ssm, attn_every=2 (1:1 interleave), RoPE, seed 0", + "arms": [ + "ssm_base — the base cell", + "swa128 — mixer: ssm -> sliding-window attention (w=128)", + "swa128_nope — NoPE on the full-attn layers of the SWA hybrid (comparand: swa128, NOT base — the paper's headline knob)", + "attn1to3 — attention fraction 1:1 -> 1:3 (attn_every 2 -> 4)", + "fullattn — attention fraction 1:1 -> all-attention (attn_every=1), the dense control" + ], + "rungs_base_equivalent_tokens": [42000000, 85000000, 150000000], + "seeds": [0], + "new_cells": 15, + "order": "cheap rung first — the 42M rung completes in ~21.7 h and already yields a full 5-arm comparison; partial completion is still a result", + "not_expressible": "An all-SSM arm (zero full-attention layers) is NOT reachable: is_full(i) = (i % attn_every == 0), so layer 0 is a full-attention layer for every attn_every >= 1, and attn_every=0 divides by zero. It would need a model.py change, which would invalidate the current verify.log — deliberately not done.", + "dropped": "mixer='full' and mixer='none' were in the first draft of this queue and were REMOVED: model.py:105-111 dispatches `if is_full(i) -> attention; elif mixer=='swa128' -> SWA; else -> SSM`, so both values fall into the else branch and are SILENTLY IDENTICAL to the base arm. They would have burned ~33 GPU-h producing duplicates of ssm_base. The all-attention control is expressed as attn_every=1 instead." + }, + + "iso_flop": { + "problem": "The arms differ by 10-20% in non-embed params (ssm_base 189.1M, swa128 160.5M, attn1to3 208.2M, fullattn 151.0M), so an iso-TOKEN comparison would violate §C18's 5% tolerance and could not be called a result.", + "method": "Per-arm token budgets set so TOTAL train FLOPs match the base arm. FLOP/token measured from the real built configs as 6N + 12*L_full*H*Dh*T + 12*L_swa*H*Dh*w (PaLM App B shape, quadratic term applied only to full-attn layers, windowed term to SWA layers).", + "result": "attn1to3 (+0.1%) and fullattn (-0.2%) are already iso-FLOP at the SAME budget — the quadratic-attention term cancels their param difference. Only swa128/swa128_nope need more tokens (1.129x).", + "worst_total_flop_mismatch_pct": 0.17, + "tolerance_pct": 5.0, + "verdict": "PASS", + "epoch_guard": "Rungs were chosen so that even the 1.129x arm fits inside the 170,034,304-token cache (top rung: base 150M -> swa 170M). No arm wraps into a second epoch, so data-repetition never differs between arms.", + "approximation_disclosed": "The FLOP model is an approximation (it does not separately count the SSM scan's linear term beyond its parameters). It is applied identically to every arm, so it biases the comparison only to the extent the arms' non-matmul work differs." + }, + + "c5_0_smoke": { + "result": "pass", + "detail": "All 5 arm configs smoked through the EXACT training script (train_hybrid.py --smoke): exit 0, fixed-batch overfit 8.79-8.94 -> 0.0016-0.0054 (trending_down=True), grad norms healthy, checkpoint save->reload max|Δ|=0.00e+00 for every arm. This is the only evidence that attn_every=1 works at all — verify.py's 8-combo toggle sweep covered attn_every in {2,4} only.", + "log_path": "smoke_arms.log", + "src": "log" + }, + + "c5_1_concurrency": { + "result": "pass", + "detail": "sentinel preflight exit 0 at queue time: mem_available=82%, disk_free=3034GB, load1=1.09, trainers=none. The prior arm (PID 3164922) exited 2026-07-20 15:04Z. The driver ALSO re-checks `pgrep -f train_*.py` before every single cell and defers the pass if any trainer is alive.", + "src": "log" + }, + + "c5_2_budget": { + "tokens_total": 1191000000, + "source": "brief research/briefs/hybrid-attention-rethink.md:55 token-budget ladder; per-arm budgets derived by the iso-FLOP matching above", + "gpu_hours_est": 141.7, + "gpu_days_est": 5.9, + "src": "derived" + }, + + "c5_3_probe": { + "tokens_per_sec": 2863, + "peak_mem_gb": 16.6, + "fits": true, + "detail": "Measured from the completed pilot arm (20,756 steps in 16.50 h at this exact seq/batch/remat config), which is a far stronger probe than a few steps. Per-cell est_hours scale that rate by token budget. The all-attention arm (attn_every=1) has 24 quadratic-attention layers vs the base's 12, so its memory and step time may exceed this estimate — its 42M cell runs early and will correct the estimate before the expensive rungs.", + "src": "derived" + }, + + "c5_4_eta_hours": 141.7, + + "c5_5_resume": { + "result": "pass", + "detail": "Each cell passes --ckpt/--resume checkpoint_.pkl with --ckpt_every 200, and the driver re-attempts failed cells on the next pass so they resume from checkpoint. Resume was proven in production on the pilot arm (recovered from a step-400 checkpoint after a sentinel kill and ran 20,756 clean steps). Smoke re-confirmed the save->reload round-trip is exact for all 5 arms.", + "resume_bug_fixed": "train_hybrid.py:136 was `range(start_step, start_step + steps)`, which made a RESUMED cell repeat the entire budget from its resume point (the pilot ran 21,156 steps = +1.93% tokens). Fixed to `range(start_step, steps)` on 2026-07-21 BEFORE queuing this ladder — unfixed it would have silently broken iso-FLOP across exactly the arms that happen to crash.", + "src": "log" + }, + + "c5_6_sentinel": { + "result": "armed per cell", + "detail": "The driver arms `sentinel.py watch` on the PYTHON trainer pid (pgrep -n -f 'python train_hybrid.py .*', not the wrapper subshell) for every cell, with no --kill-at so it takes the §C6 module default 0.80 < safe_cuda 0.85 < the OOM cliff, plus the thermal kill-switch. Preflight gates the driver before the first launch.", + "src": "log" + }, + + "c5_7_guards": { + "result": "verified", + "detail": "JAX script: train_hybrid.py:10 `import jax_safe_env` precedes :15 `import jax`. Chunked CE (model.py:136) streams max+sumexp over vocab chunks and never materializes (N, 151936). nn.remat on the decoder block. Re-verified 2026-07-21; verify.log (2026-07-20) is 6/6 PASS against this model.py.", + "src": "log" + }, + + "verdict_metric": "Per cell: eval-harness text-lm-v2 BPB on wikitext-2 + code with the model's own Qwen3 tokenizer, suite_version stamped. Per arm: the quality-vs-FLOPs curve across the 3 rungs, then scaling_ladder.fit_gap_trend() on the arm-minus-base gap to ask whether the mixer choice changes the DESTINATION or only the SPEED. Honest ceiling for phase 1: DIRECTIONAL — every cell is n=1, and §C17 requires >=3 seeds for a win. Phase 2 appends seeds 1-2 only for arms that separate.", + + "not_claimed": "No result of any kind is claimed at queue time. The pilot arm's numbers (PPL wikitext2 133.46 / code_py 5142.64, self_floor) are a single undertrained n=1 point with no comparand and are NOT a baseline for these arms — the pilot ran 173.3M tokens under the pre-fix resume bug and is not FLOP-matched to any rung here.", + + "evidence_path": "HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json" +} diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/cells.json b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/cells.json new file mode 100644 index 0000000..f68f32d --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/cells.json @@ -0,0 +1,246 @@ +{ + "schema_version": 1, + "study": "hybrid-ssm-0.2b architecture ladder", + "model_dir": "HybridSSM-0.2B", + "lifecycle_stage": "architecture", + "seq": 2048, + "batch": 4, + "iso_flop": "Per-arm token budgets are set so TOTAL train FLOPs match the base arm within 0.2% (\u00a7C18 needs <=5%). Param counts differ by 10-20% between arms, so an iso-TOKEN comparison would NOT have been valid; the quadratic-attention term makes attn1to3/fullattn iso-FLOP at the same budget while swa128 needs 1.129x.", + "rungs": [ + 42000000, + 85000000, + 150000000 + ], + "seeds_phase1": [ + 0 + ], + "phase2_policy": "auto-seed-up: seeds 1,2 appended ONLY for arms separating from base at the top rung (\u00a7C17 needs n>=3 for a win; n=1 caps at directional)", + "pilot": "The completed 2026-07-19 ssm_base run (173.3M tok under the pre-fix resume bug) is retained as the PILOT, not a ladder cell: no other arm can match its FLOPs without wrapping the 170.03M cache into a 2nd epoch.", + "cells": [ + { + "id": "ssm_base_42M_s0", + "arm": "ssm_base", + "seed": 0, + "rung_base_tokens": 42000000, + "tokens": 42000000, + "steps": 5126, + "mixer": "ssm", + "attn_every": 2, + "nope": false, + "est_hours": 4.1, + "varies": "(base cell)", + "phase": 1, + "status": "queued" + }, + { + "id": "swa128_42M_s0", + "arm": "swa128", + "seed": 0, + "rung_base_tokens": 42000000, + "tokens": 48000000, + "steps": 5859, + "mixer": "swa128", + "attn_every": 2, + "nope": false, + "est_hours": 4.7, + "varies": "mixer: ssm -> swa128", + "phase": 1, + "status": "queued" + }, + { + "id": "swa128_nope_42M_s0", + "arm": "swa128_nope", + "seed": 0, + "rung_base_tokens": 42000000, + "tokens": 48000000, + "steps": 5859, + "mixer": "swa128", + "attn_every": 2, + "nope": true, + "est_hours": 4.7, + "varies": "NoPE on full-attn (comparand: swa128)", + "phase": 1, + "status": "queued" + }, + { + "id": "attn1to3_42M_s0", + "arm": "attn1to3", + "seed": 0, + "rung_base_tokens": 42000000, + "tokens": 42000000, + "steps": 5126, + "mixer": "ssm", + "attn_every": 4, + "nope": false, + "est_hours": 4.1, + "varies": "attention fraction 1:1 -> 1:3", + "phase": 1, + "status": "queued" + }, + { + "id": "fullattn_42M_s0", + "arm": "fullattn", + "seed": 0, + "rung_base_tokens": 42000000, + "tokens": 42000000, + "steps": 5126, + "mixer": "ssm", + "attn_every": 1, + "nope": false, + "est_hours": 4.1, + "varies": "attention fraction 1:1 -> all-attention control", + "phase": 1, + "status": "queued" + }, + { + "id": "ssm_base_85M_s0", + "arm": "ssm_base", + "seed": 0, + "rung_base_tokens": 85000000, + "tokens": 85000000, + "steps": 10375, + "mixer": "ssm", + "attn_every": 2, + "nope": false, + "est_hours": 8.2, + "varies": "(base cell)", + "phase": 1, + "status": "queued" + }, + { + "id": "swa128_85M_s0", + "arm": "swa128", + "seed": 0, + "rung_base_tokens": 85000000, + "tokens": 96000000, + "steps": 11718, + "mixer": "swa128", + "attn_every": 2, + "nope": false, + "est_hours": 9.3, + "varies": "mixer: ssm -> swa128", + "phase": 1, + "status": "queued" + }, + { + "id": "swa128_nope_85M_s0", + "arm": "swa128_nope", + "seed": 0, + "rung_base_tokens": 85000000, + "tokens": 96000000, + "steps": 11718, + "mixer": "swa128", + "attn_every": 2, + "nope": true, + "est_hours": 9.3, + "varies": "NoPE on full-attn (comparand: swa128)", + "phase": 1, + "status": "queued" + }, + { + "id": "attn1to3_85M_s0", + "arm": "attn1to3", + "seed": 0, + "rung_base_tokens": 85000000, + "tokens": 85000000, + "steps": 10375, + "mixer": "ssm", + "attn_every": 4, + "nope": false, + "est_hours": 8.2, + "varies": "attention fraction 1:1 -> 1:3", + "phase": 1, + "status": "queued" + }, + { + "id": "fullattn_85M_s0", + "arm": "fullattn", + "seed": 0, + "rung_base_tokens": 85000000, + "tokens": 85000000, + "steps": 10375, + "mixer": "ssm", + "attn_every": 1, + "nope": false, + "est_hours": 8.2, + "varies": "attention fraction 1:1 -> all-attention control", + "phase": 1, + "status": "queued" + }, + { + "id": "ssm_base_150M_s0", + "arm": "ssm_base", + "seed": 0, + "rung_base_tokens": 150000000, + "tokens": 150000000, + "steps": 18310, + "mixer": "ssm", + "attn_every": 2, + "nope": false, + "est_hours": 14.6, + "varies": "(base cell)", + "phase": 1, + "status": "queued" + }, + { + "id": "swa128_150M_s0", + "arm": "swa128", + "seed": 0, + "rung_base_tokens": 150000000, + "tokens": 170000000, + "steps": 20751, + "mixer": "swa128", + "attn_every": 2, + "nope": false, + "est_hours": 16.5, + "varies": "mixer: ssm -> swa128", + "phase": 1, + "status": "queued" + }, + { + "id": "swa128_nope_150M_s0", + "arm": "swa128_nope", + "seed": 0, + "rung_base_tokens": 150000000, + "tokens": 170000000, + "steps": 20751, + "mixer": "swa128", + "attn_every": 2, + "nope": true, + "est_hours": 16.5, + "varies": "NoPE on full-attn (comparand: swa128)", + "phase": 1, + "status": "queued" + }, + { + "id": "attn1to3_150M_s0", + "arm": "attn1to3", + "seed": 0, + "rung_base_tokens": 150000000, + "tokens": 150000000, + "steps": 18310, + "mixer": "ssm", + "attn_every": 4, + "nope": false, + "est_hours": 14.6, + "varies": "attention fraction 1:1 -> 1:3", + "phase": 1, + "status": "queued" + }, + { + "id": "fullattn_150M_s0", + "arm": "fullattn", + "seed": 0, + "rung_base_tokens": 150000000, + "tokens": 150000000, + "steps": 18310, + "mixer": "ssm", + "attn_every": 1, + "nope": false, + "est_hours": 14.6, + "varies": "attention fraction 1:1 -> all-attention control", + "phase": 1, + "status": "queued" + } + ] +} diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh new file mode 100755 index 0000000..dc78881 --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# HybridSSM-0.2B architecture ladder — continuous driver. +# +# Runs every queued cell in cells.json ONE AT A TIME (§C4.5) until all are done, so the +# ladder keeps going across thermal kills, crashes and reboots without a human restarting +# it. Modelled on the proven Qwen3 run_ladder.sh: .done markers make it idempotent, a +# cool-down gate keeps it off a hot box, each cell gets its own sentinel watcher, a +# loop-until-done pass structure re-attempts killed cells (they resume from checkpoint), +# and an escalating hot-spell backoff stops it thrashing at zero net progress. +# +# Cells are ordered CHEAP RUNG FIRST, so the 42M rung finishes in ~22 h and already yields +# a complete 5-arm architecture comparison; partial completion is still a result. +set -uo pipefail + +ROOT="${BFS_ROOT:-/home/yashb98/Downloads/BuildFromScratch}" +LDIR="$ROOT/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder" +BUILD="$ROOT/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build" +DATA="$ROOT/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/tokcache_170034304_300000_seed0_Qwen3-0.6B-Base.pt" +LOG="$LDIR/run_arch_ladder.log" +PY=python3 +COOL_C="${LADDER_COOL_C:-70}" # don't launch onto a box hotter than this +MAXPASS="${LADDER_MAXPASS:-100}" +SEQ=2048; BATCH=4; LR=3e-3; WARMUP=200 + +exec >> "$LOG" 2>&1 +echo "===== $(date '+%F %T') driver start (pid $$) =====" + +# Hottest of GPU die + all ACPI SoC zones, whole deg C (empty if unreadable). No sudo. +hottest_c () { + local g z zc max="" + g=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -dc '0-9') + [ -n "$g" ] && max=$g + for z in /sys/class/thermal/thermal_zone*/temp; do + [ -r "$z" ] || continue + zc=$(( $(cat "$z" 2>/dev/null || echo 0) / 1000 )) + { [ -z "$max" ] || [ "$zc" -gt "$max" ]; } && max=$zc + done + echo "$max" +} + +# Bounded, fail-open cool-down: wait up to ~30 min to drop below COOL_C. Unreadable => proceed. +cool_down () { + local tag=$1 h + for _ in $(seq 1 60); do + h=$(hottest_c) + [ -z "$h" ] && { echo "[$(date '+%T')] [cooldown] $tag: temp unreadable, proceeding"; return 0; } + [ "$h" -lt "$COOL_C" ] && { echo "[$(date '+%T')] [cooldown] $tag: ${h}C < ${COOL_C}C, launching"; return 0; } + echo "[$(date '+%T')] [cooldown] $tag: ${h}C >= ${COOL_C}C, waiting 30s" + sleep 30 + done + echo "[$(date '+%T')] [cooldown] $tag: still hot after 30min — launching anyway (bounded)" +} + +# Is a REAL trainer alive? A bare `pgrep -f 'train_*.py'` is not good enough: it matches any +# process whose cmdline merely MENTIONS a trainer — a grep, an editor, a monitoring command, +# another agent session. That false positive deferred all 15 cells on the first launch attempt +# (2026-07-21 22:03) while sentinel's own preflight, which filters probes, correctly reported +# trainers=none on the very same second. So: require argv[0] to be a python interpreter, which +# no shell wrapper or pgrep can satisfy, and never count ourselves. +trainer_alive () { + local p exe + for p in $(pgrep -f 'train_[A-Za-z0-9_]*\.py' 2>/dev/null); do + [ "$p" = "$$" ] && continue + exe=$(tr '\0' '\n' < "/proc/$p/cmdline" 2>/dev/null | head -1) + case "$(basename "${exe:-none}")" in python*) return 0 ;; esac + done + return 1 +} + +# cells.json -> "id tokens steps mixer attn_every nope" lines, queue order preserved. +cells () { $PY - "$LDIR/cells.json" <<'PYEOF' +import json, sys +for c in json.load(open(sys.argv[1]))["cells"]: + print(c["id"], c["tokens"], c["steps"], c["mixer"], c["attn_every"], int(c["nope"])) +PYEOF +} + +run_cell () { + local id=$1 tokens=$2 steps=$3 mixer=$4 every=$5 nope=$6 + [ -f "$LDIR/${id}.done" ] && { echo "[$(date '+%T')] [skip] $id"; return 0; } + + # §C4.5: never two trainers. A foreign trainer means someone else owns the GPU — wait it out. + if trainer_alive; then + echo "[$(date '+%T')] [wait] $id: another trainer is alive, deferring this pass"; return 1 + fi + cool_down "$id" + # unified pool headroom (shared CPU+GPU memory): wait for >= 60 GB available + for _ in $(seq 1 90); do a=$(free -g | awk '/Mem:/{print $7}'); [ "${a:-0}" -ge 60 ] && break; sleep 10; done + + local nopeflag=""; [ "$nope" = "1" ] && nopeflag="--nope_on_full" + echo "[$(date '+%F %T')] START $id tokens=$tokens steps=$steps mixer=$mixer attn_every=$every nope=$nope" + ( cd "$BUILD" && $PY train_hybrid.py --data "$DATA" --seq $SEQ --batch $BATCH \ + --tokens "$tokens" --lr $LR --warmup $WARMUP --mixer "$mixer" --attn_every "$every" $nopeflag \ + --ckpt "checkpoint_${id}.pkl" --resume "checkpoint_${id}.pkl" \ + --ckpt_every 200 --eval_every 400 --done_marker "$LDIR/${id}.done" \ + >> "$LDIR/${id}.log" 2>&1 ) & + local tpid=$! + # sentinel on the PYTHON trainer, not this subshell; no --kill-at so it takes the §C6 default + local spid="" + sleep 20 + local realpid; realpid=$(pgrep -n -f "python[0-9.]* train_hybrid\.py .*${id}" || echo "$tpid") + $PY "$ROOT/sentinel.py" watch --pid "$realpid" --log "$LDIR/sentinel_${id}.log" >/dev/null 2>&1 & + spid=$! + wait "$tpid"; local rc=$? + kill "$spid" 2>/dev/null + + if [ $rc -eq 0 ] && [ -f "$LDIR/${id}.done" ]; then + echo "[$(date '+%T')] [done] $id"; return 0 + fi + rm -f "$LDIR/${id}.done" # never trust a marker from a failed cell + echo "[$(date '+%T')] [FAIL rc=$rc] $id — will retry next pass (resumes from checkpoint)" + return 1 +} + +# ---- preflight once, then loop passes until every cell has its marker ------------------- +$PY "$ROOT/sentinel.py" preflight || { echo "preflight FAIL — abort"; exit 1; } + +pass=0; hot_backoff=0; prev_missing=99999 +while : ; do + fails=0 + while read -r id tokens steps mixer every nope; do + [ -z "${id:-}" ] && continue + run_cell "$id" "$tokens" "$steps" "$mixer" "$every" "$nope" || fails=$((fails+1)) + done < <(cells) + + missing=0 + while read -r id _; do [ -f "$LDIR/${id}.done" ] || missing=$((missing+1)); done < <(cells) + [ "$missing" -eq 0 ] && break + + pass=$((pass+1)) + [ "$pass" -ge "$MAXPASS" ] && { echo "[$(date '+%T')] MAXPASS=$MAXPASS, $missing incomplete — stopping (checkpoints preserved; re-run to continue)"; break; } + + # HOT-SPELL BACKOFF: a whole pass with failures and NO cell completing means the box is too + # hot to make progress — wait for a cooler window (5→30 min cap) instead of thrashing. + if [ "$fails" -gt 0 ] && [ "$missing" -ge "$prev_missing" ]; then + hot_backoff=$((hot_backoff+1)); wait_s=$((hot_backoff*300)); [ "$wait_s" -gt 1800 ] && wait_s=1800 + echo "[$(date '+%F %T')] pass $pass: $missing incomplete, no progress ($fails failed) — backing off ${wait_s}s" + sleep "$wait_s" + else + hot_backoff=0 + echo "[$(date '+%F %T')] pass $pass: $missing incomplete ($fails failed) — re-attempting" + fi + prev_missing=$missing +done + +# ---- completion gate: only claim done when EVERY cell has its marker -------------------- +missing=0 +while read -r id _; do [ -f "$LDIR/${id}.done" ] || { missing=$((missing+1)); echo " incomplete: $id"; }; done < <(cells) +if [ "$missing" -eq 0 ]; then + touch "$LDIR/ladder.done" + echo "===== $(date '+%F %T') ARCH LADDER COMPLETE — $LDIR/ladder.done =====" + [ -f "$LDIR/score_arch_ladder.py" ] && $PY "$LDIR/score_arch_ladder.py" +else + echo "===== $(date '+%F %T') LADDER INCOMPLETE — $missing cells missing .done =====" +fi From 040962045ba24f0e1cd982a6f2662c5117ba9c13 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Wed, 22 Jul 2026 23:42:57 +0100 Subject: [PATCH 07/35] =?UTF-8?q?Track=20the=20durable=20record=20in=20git?= =?UTF-8?q?=20=E2=80=94=20close=20the=20one-disk=20truth-store=20risk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit's #3 risk: ledger.json (the truth store) and the runs/*.md + briefs + digests (its human-readable trail) were git-untracked, living on one disk — and a branch switch already destroyed a ledger.json once (recovered from a git blob). Source .py/.sh became tracked yesterday; the record did not. Per the 2026-07-22 decision (batch 7 Q1), the record is now tracked too: ledger.json + research/ledger/runs/**/*.md + research/briefs + research/digests — 16 files, ~160K of text. Git history becomes the off-box backup; the rule is to commit the record before any branch switch (the hazard that bit us before). Kept deliberately local: loop_state.json (churns every loop wake — a stale tracked copy is worse than none), the single-generation ledger.json.bak, the dated backups/ snapshots, and the generated non-record artifacts (pulse/radar/provenance, datasets, checkpoints, logs). Verified by dry-run that all of those stay ignored. Also force-adds today's audit deliverables (LOOP_AUDIT + LOOP_UPGRADE_PLAN) so the 28-decision analysis survives. Note: 7 of 24 runs have a detail_md on disk; the other 17 dangling paths are a separate Tier-2 backfill item. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 24 +- research/LOOP_AUDIT_2026-07-22.md | 124 + research/LOOP_UPGRADE_PLAN_2026-07-22.md | 46 + research/briefs/block-text-diffusion.md | 164 ++ research/briefs/hybrid-attention-rethink.md | 122 + .../briefs/vibethinker-small-reasoning.md | 102 + research/briefs/zeta-dual-whitening.md | 92 + research/digests/2026-06-17.md | 57 + research/digests/2026-07-14.md | 55 + research/ledger/ledger.json | 2086 +++++++++++++++++ .../2026-06-16_qwen3-faithful_eval-first.md | 30 + .../runs/2026-06-16_qwen3_normuon-vs-adamw.md | 75 + .../2026-06-17_qwen3-0.6b_openr1-math-220k.md | 65 + ..._qwen3-0.6b_vibethinker-small-reasoning.md | 140 ++ .../runs/2026-06-27_qwen3-0.6b_sft-3seed.md | 76 + .../2026-06-30_qwen3-0.6b_midtrain-anneal.md | 96 + ...19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md | 107 + 17 files changed, 3456 insertions(+), 5 deletions(-) create mode 100644 research/LOOP_AUDIT_2026-07-22.md create mode 100644 research/LOOP_UPGRADE_PLAN_2026-07-22.md create mode 100644 research/briefs/block-text-diffusion.md create mode 100644 research/briefs/hybrid-attention-rethink.md create mode 100644 research/briefs/vibethinker-small-reasoning.md create mode 100644 research/briefs/zeta-dual-whitening.md create mode 100644 research/digests/2026-06-17.md create mode 100644 research/digests/2026-07-14.md create mode 100644 research/ledger/ledger.json create mode 100644 research/ledger/runs/2026-06-16_qwen3-faithful_eval-first.md create mode 100644 research/ledger/runs/2026-06-16_qwen3_normuon-vs-adamw.md create mode 100644 research/ledger/runs/2026-06-17_qwen3-0.6b_openr1-math-220k.md create mode 100644 research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md create mode 100644 research/ledger/runs/2026-06-27_qwen3-0.6b_sft-3seed.md create mode 100644 research/ledger/runs/2026-06-30_qwen3-0.6b_midtrain-anneal.md create mode 100644 research/ledger/runs/2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md diff --git a/.gitignore b/.gitignore index 835e2f3..8e965e5 100644 --- a/.gitignore +++ b/.gitignore @@ -53,16 +53,30 @@ SmolLM2-134(base)/results/lm_eval/ # record tracked; exclude only transient/runtime files". Source now follows that # policy again; state does not. # -# Deliberately still local-only: ledger.json / loop_state.json (churning state — -# a stale tracked copy is worse than none; durable backup = research/backup_ledger.sh), -# generated artifacts (digests/, pulse/, radar/, provenance/), prepared datasets, -# manuscript build output, and harness-search's machine-GENERATED candidates -# (search output, not authored source). +# The DURABLE RECORD is now tracked too (2026-07-22 decision): ledger.json is the +# truth store and the runs/*.md + briefs/ + digests/ are the human-readable record — +# a one-disk truth store was the audit's #3 risk, and a branch switch already +# destroyed a ledger.json once (recovered from a git blob). Git history IS the +# off-box backup; commit the record before any branch switch. The single-generation +# .bak and the dated backups/ snapshots stay local (git history is the real trail). +# +# Deliberately still local-only: loop_state.json (churns every loop wake — a stale +# tracked copy is worse than none), generated non-record artifacts (pulse/, radar/, +# provenance/), prepared datasets, manuscript build output, checkpoints/logs, and +# harness-search's machine-GENERATED candidates (search output, not authored source). research/** # re-include directories so git descends and the rules below can match !research/**/ !research/**/*.py !research/**/*.sh +# the durable record (small text; the truth store + its human-readable trail) +!research/ledger/ledger.json +!research/ledger/runs/**/*.md +!research/briefs/**/*.md +!research/digests/**/*.md +# ...but never the churn/backup copies of the ledger +research/ledger/ledger.json.bak +research/ledger/backups/ research/harness_search/archive/ research/harness_search/targets/*/candidates/ diff --git a/research/LOOP_AUDIT_2026-07-22.md b/research/LOOP_AUDIT_2026-07-22.md new file mode 100644 index 0000000..4183c49 --- /dev/null +++ b/research/LOOP_AUDIT_2026-07-22.md @@ -0,0 +1,124 @@ +# Research-loop deep audit — 2026-07-22 + +Source: 8-agent read-only workflow (wf_641090e2-442) + first-hand session record. +74 evidence-anchored weaknesses. The 2 mining agents + synthesizer hit the session +limit; synthesis + pain-mining done by the main loop from the 8 reports + lived history. + +## Scorecard (what this system actually is, as evidenced) + +A single-box (GB10) ML-research state machine with 27 skills and a §C1–§C27 contract +spine. In 6 weeks it produced **24 ledger runs, ~15–19 GPU-days, 2 wins** (1 stage-win +in 13 stages), and the flagship optimizer "win" is now measured to **converge away**. +Its real, differentiated product is **rigor** — it caught an eval-token confound, a +token-shuffle bug, an implausible verdict, a resume-budget iso-FLOP violation, a +silent ledger-write failure, and pre-registered the RLVR null. The "autonomous nightly +loop" has closed **one** CPU-light iteration (launch declined); **cron was never armed**; +nearly every headline came from human-driven sessions. + +## Top weaknesses, ranked (merged across the 8 reports) + +1. **Autonomy is inert.** No BuildFromScratch cron installed (nightly/liveness/@reboot); + a *foreign* project (forge-loop) now owns the crontab. A reboot tonight strands the + live 15-cell ladder until a human intervenes. weekly-retro never ran → the §C15.3.8 + calibration loop never closed → every score is unfalsified self-assessment. +2. **Specified path ≠ actual path.** /ablation-runner ran ~2× while ~24 runs landed + hand-driven; "no brief, no run" is not operative (11/24 runs technique_slug=null); + the flagship win has no c5_evidence; out-of-loop launches are the dominant mode and + the contracts define no protocol for them (the current live run mutates loop_state + while skipping the digest/provenance machinery). +3. **Truth store has zero off-box durability.** ledger.json is git-untracked; 17/24 + runs point at detail_md files that don't exist; 60+ ad-hoc --set keys (eval metrics + stored outside metrics{}); techniques strand in non-selectable states so next-best is + actively wrong; working branch is 6 commits ahead of origin, 58 dirty paths, and + gitignored evidence lives on one disk (already destroyed once via branch switch). +4. **Eval starvation + monopoly-in-name-only.** score_arch_ladder.py doesn't exist (the + LIVE run's scoring hook is a silent no-op); the continuous driver leaves no GPU-free + window so no suite number for ~6 days; the "eval-harness is the ONLY comparable source" + monopoly is honored in stamp but not code — 6 hand-rolled score_cohort.py copies; + text-lm-v2/v3 governance contradiction; §C25 HARD registry is ahead of its tooling so + several stages are structurally capped at "directional" regardless of result quality. +5. **"directional" compresses opposite realities** — genuine nulls (sft-masking, grpo) + and big *significant* effects capped only by one missing HARD item (data-mix +0.59 + code BPB, significant). A ledger reader can't tell "found nothing" from "found + something big, one gate short." +6. **Safety debt on the newest layer.** The thermal-kill path (added because the box + hard-locks from heat) has ZERO tests; thermal_log.py is NOT running beside the live + run; safe_cuda / jax_safe_env / cron_runner / liveness_cron / the arch-ladder driver + are all untested; kdump/panic still disabled so every future hard-lock is trace-less. +7. **Intake is 100% trend-following and stale.** idea-selection §C15.3 (bandit/cascade/ + red-team) has never executed (it's "sort, not bandit"); taxonomy_gap has never + produced a candidate; pulse ran 3× in 40 days; dedup is exact-slug only; the candidate + pile grows monotonically with no aging/kill policy. +8. **Win ceiling.** Wins correlate with huge effects + cheap batteries (data-stage, + on-box passkey); optimizer/architecture effects at this budget are small or fade, and + several stages are capped by structurally expensive HARD items — so on-box ablations + of the same shape are near-guaranteed "directional." The one big open question + (NorMuon at scale, ~17–34h) is unresolved while 141h goes to the hybrid ladder. +9. **Publication drift.** arXiv tarball is stale vs the 2026-07-20 rebuilt sections; + qwen3-study state.json says phase 3 while Phase-6 artifacts exist; the normuon paper + is orphaned and its headline may not survive (converges); both master strategy docs + open with leaked LLM meta-commentary. +10. **Contract drift.** S10 has 3 contradictory specs (one names a nonexistent skill); + §C27.6 S4 stage-plan gate unimplemented (the 141h arch stage has no plan.md); pinned + schemas lag the code; §C13 objective vocab can't express most §C25 lifecycle stages. + +Full per-area weakness list with file:line evidence: see workflow journal +(subagents/workflows/wf_641090e2-442/journal.jsonl) — 8 result lines, one per area. + +## Decision plan — 7 batches × 4 questions (28 total) + +1. North star & what "best" means +2. Automation & autonomy level (cron, gates, unattended rights) +3. Contract-vs-reality operating model (out-of-loop runs, launcher, c5) +4. Research intake & selection policy +5. Execution & compute (phase-2, Muon, off-box, NorMuon-at-scale) +6. Evaluation rigor & economics (suite, seeds, scoring scheduling) +7. Durability, engineering & publication (git policy, test debt, arXiv) + +Answers get folded into a follow-up implementation plan. + +## Decisions (recorded as answered) + +### Batch 1 — North star +1. Optimize to be: **the rigor factory** (de-confounding/verification harness). +2. Primary 1–2mo deliverable: **published papers** (arXiv qwen3 study + hybrid-SSM study). +3. Next GPU-week: **finish NorMuon-at-scale** (3rd 420M seed + 840M rung; rescues the paper). +4. Verdict vocabulary: **split it** (null / promising-capped / win — stop compressing opposites). + +### Batch 2 — Automation & autonomy +1. Cron: **recovery crons only** (@reboot boot_resume + */30 liveness; protects the live ladder, no auto-launch). +2. Launch rights: **propose-only GPU, auto CPU** (loop briefs/preps/scores autonomously; human triggers every GPU run). +3. Box sharing: **shared-box lock/handshake** both BFS and forge-loop honor. +4. Calibration: **wire outcomes→calibration** + real retro cadence (make scores falsifiable). + +### Batch 3 — Contract vs reality +1. Out-of-loop runs: **codify the manual path** (adopted-run protocol: c5 + ledger entry + digest stub + loop_state). +2. Brief gate: **required for new techniques only** (not re-runs/controls/seeds/scaling rungs). +3. c5 evidence: **validated schema + pre-launch lint** (refuse launch on incomplete evidence). +4. Ledger timing: **entry-at-launch wins; fix CLAUDE.md** (can't have a verdict before the run). + +### Batch 4 — Research intake & selection +1. Intake: **your taste, trend as input** (pulse/radar surface options; you pick the questions). +2. Candidates: **auto-expire unbriefed** after N weeks (out of next-best; reversible). +3. Scan cadence: **weekly** (not daily). +4. Dedup: **arXiv-id + fuzzy-title** (not slug-only). + +### Batch 5 — Execution & compute +1. Muon: **accept AdamW as the ladder baseline** (note deviation; don't restart 5 done cells). +2. Mixer confound: **cheap per-arm LR probe** before any mixer-type claim (de-confound first). +3. Off-box: **defer, stay on-box** (build remote only when a specific run needs it). +4. Phase-2 seeds: **seed-up only after de-confounding**, only arms that still separate. + +### Batch 6 — Evaluation rigor & economics +1. Eval timing: **score between rungs** (pause driver after each rung, BPB-score its cells, resume). +2. Scorer: **one real eval-harness impl** (delete the 6 hand-rolled copies; fix the suite stamp). +3. §C25 tooling: **build only for stages you'll publish** (pretraining/architecture HARD items). +4. Noise floor: **fixed reference-model floor** (not self-floor on undertrained checkpoints). + +### Batch 7 — Durability, engineering & publication +1. Git policy: **track the durable record** (ledger.json + runs/*.md + briefs + digests; only ckpts/logs/data ignored). +2. Test debt: **safety-killers first** (thermal-kill, safe_cuda/jax guards, driver concurrency/PID). +3. arXiv paper: **rebuild the package + fix state.json, then user submits** (I refresh, human uploads). +4. Crash forensics: **kdump/panic fix at next safe reboot** (crashkernel=512M, panic=10). + +All 28 answered. Implementation roadmap: research/LOOP_UPGRADE_PLAN_2026-07-22.md. diff --git a/research/LOOP_UPGRADE_PLAN_2026-07-22.md b/research/LOOP_UPGRADE_PLAN_2026-07-22.md new file mode 100644 index 0000000..f6dc307 --- /dev/null +++ b/research/LOOP_UPGRADE_PLAN_2026-07-22.md @@ -0,0 +1,46 @@ +# Research-loop upgrade plan — 2026-07-22 + +Derived from the 28-decision audit (`LOOP_AUDIT_2026-07-22.md`). One-line identity: +**a rigor factory that ships papers, runs its safety/prep autonomously, and keeps a human on every GPU-spend trigger.** All work below is CPU-only and safe beside the live ladder unless marked [GPU] or [HUMAN]. + +## Tier 0 — Immediate, cheap, protects live work + +1. **Print recovery cron lines** for the user to paste [HUMAN paste, §C4.2]: + `@reboot bash /research/boot_resume.sh` and `*/30 * * * * /research/liveness_cron.sh`. + → the live 141h ladder currently has NO reboot protection. +2. **Track the durable record in git** (batch7 Q1): un-ignore ledger.json + runs/*.md + briefs/ + digests/ (keep ckpts/*.pkl, *.pt, *.log, datasets ignored). Commit. Closes the #3 risk (one-disk truth store; a branch switch destroyed a ledger.json once). Commit BEFORE any future branch switch. +3. **Rebuild the arXiv package** (batch7 Q3): refresh tarball/.bbl/state.json to match the 2026-07-20 sections, re-run the honesty scrub. [HUMAN does the actual arXiv upload.] + +## Tier 1 — The rigor engine (papers depend on it) + +4. **Wire ladder scoring** (batch6 Q1): write the missing `score_arch_ladder.py`; implement "score between rungs" — after each rung the driver pauses, eval-harness BPB-scores its 5 cells, then resumes. Backfill: BPB-score the 42M rung (and 85M when done) at the next rung gap. [GPU, brief] +5. **Per-arm LR probe** (batch5 Q2): small LR sweep at 42M per mixer to de-confound the SSM-vs-attention gap before it's a claim. The 42M rung's mixer finding stays "directional + LR-confounded" until this runs. [GPU] +6. **One real eval-harness** (batch6 Q2): make eval-harness the single scorer; delete the 6 hand-rolled score_cohort.py copies; fix the suite-version stamp so it guarantees what ran. +7. **Fixed reference-model noise floor** (batch6 Q4): replace self-floor with a stable reference checkpoint so CIs are meaningful. +8. **Split the verdict vocabulary** (batch1 Q4): add null / promising-capped / win to the ledger schema; relabel the 9 existing "directional" runs. +9. **NorMuon-at-scale** (batch1 Q3) [GPU, HUMAN-gated]: 3rd 420M seed (~17h) + 840M rung (~34h) to resolve the flagship open question and rescue the orphaned paper. SEQUENCING (one point needing your ok): finish the current hybrid 85M rung, then insert NorMuon-at-scale, then resume the hybrid 150M rung — respects "NorMuon next GPU-week" without wasting the running ladder. + +## Tier 2 — Contract & hygiene reconciliation (CPU-only) + +10. **Codify the out-of-loop path** (batch3 Q1): an "adopted run" protocol — any manual launch must write c5 + a ledger entry + a digest stub + register in loop_state. +11. **Brief gate = new techniques only** (batch3 Q2); **c5 validated schema + pre-launch lint** (batch3 Q3); **fix CLAUDE.md verdict-timing** to entry-at-launch (batch3 Q4). +12. **Ledger integrity**: backfill/stop-stamping the 17 missing detail_md; move eval metrics into metrics{}; un-strand techniques stuck in non-selectable states. +13. **Candidate auto-expire** (batch4 Q2) + **arXiv-id/fuzzy dedup** (batch4 Q4). +14. **Wire outcomes→calibration** + a weekly retro (batch2 Q4). +15. **Reframe docs**: "autonomous AI scientist" → "rigor factory" (batch1 Q1); scrub the leaked LLM meta-commentary from both strategy docs. + +## Tier 3 — Safety & durability engineering + +16. **Safety-killer tests first** (batch7 Q2): thermal-kill parsing/debounce, safe_cuda/jax_safe_env guards, arch-ladder driver concurrency/PID. Then the recovery chain. +17. **Shared-box lock/handshake** with forge-loop (batch2 Q3): one cross-project GPU lock both honor. +18. **loop_state.py fsync parity** with ledger.py (recovery-critical file lacks the parent-dir fsync). +19. **kdump/panic fix** at next safe reboot (batch7 Q4) [HUMAN, sudo]. + +## Tier 4 — Standing policy (no build, just adopt) + +20. Weekly scan cadence (batch4 Q3). Taste-driven intake, trend as input (batch4 Q1). +21. Propose-only GPU / auto CPU (batch2 Q2). Defer off-box; stop maintaining remote as live surface (batch5 Q3). +22. Accept AdamW as the hybrid ladder baseline; note the ARCHITECTURE.md deviation (batch5 Q1). + +## The one sequencing question for you +Item 9: insert NorMuon-at-scale after the current 85M rung (pausing the hybrid ladder ~34h), or let the whole hybrid ladder finish first? Your batch-1 answer said "NorMuon next GPU-week," which implies the former — confirm when you want it. diff --git a/research/briefs/block-text-diffusion.md b/research/briefs/block-text-diffusion.md new file mode 100644 index 0000000..251651e --- /dev/null +++ b/research/briefs/block-text-diffusion.md @@ -0,0 +1,164 @@ +# Brief: Block text diffusion LM head (DiffusionGemma-style parallel denoising decode) (block-text-diffusion) +- researched: 2026-06-17 · by: ml-research · fetch_level: non-arxiv-page (+ ar5iv/HF supporting fetches) +- paper_date: 2026-06-10 (cutoff_3m this run: 2026-03-17) +- modality: text · verdict: propose-only +- objective: finetune (§C13) · taste_score: 3.0 (§C15.2) + +> **Fetch-level disclaimer.** The in-window subject is a Google product blog + the +> DiffusionGemma model card (no arXiv paper for DiffusionGemma itself this run), so all +> *training* details below that go beyond the blog/model-card text are `inferred` from the +> background block-diffusion literature (BD3-LM 2503.09573; dLLM 2602.22661; DiffuLLaMA/Dream) +> and flagged as such. No PDF text was fetchable; nothing here is from memory. + +## Sources (all fetch-verified this run, §C3) +| url | what it is | accessed | replication status | +|---|---|---|---| +| https://blog.google/innovation-and-ai/technology/developers-tools/diffusion-gemma-faster-text-generation/ | Google announcement of DiffusionGemma (the in-window candidate, dated 2026-06-10) | 2026-06-17 | claim, unreplicated (vendor) | +| https://ai.google.dev/gemma/docs/diffusiongemma/model_card | DiffusionGemma model card — architecture/diffusion config | 2026-06-17 | claim, unreplicated (vendor) | +| https://huggingface.co/papers/2602.22661 | dLLM: Simple Diffusion Language Modeling — converts **Qwen3-0.6B** to MDLM/BD3LM via SFT | 2026-06-17 | independently discussed, not re-measured at our scale (no sub-1B PPL) | +| https://arxiv.org/pdf/2503.09573 | BD3-LM (Block Diffusion, ICLR 2025 Oral) — the foundational block-diffusion method | 2026-06-17 | background only (2025-03, < cutoff); cited not as subject | +| https://arxiv.org/pdf/2604.02718 | "Generative Frontiers: Why Evaluation Matters for Diffusion LMs" — eval critique | 2026-06-17 | independently discussed (eval methodology) | +| https://arxiv.org/html/2512.10858v2 | Scaling Behavior of Discrete Diffusion LMs | 2026-06-17 | independently discussed, not re-measured | + +Alternatives considered for the subject paper (headless pick, §C4.1): the in-window release with the +strongest direct claim is DiffusionGemma (2026-06-10). Background block-diffusion sources (BD3-LM +2025-03, dLLM 2602.22661) are pre/edge-of-cutoff and are cited as background, not as the subject (§C2). + +## What it changes +Replaces autoregressive left-to-right token-by-token decoding with **block / masked discrete +diffusion**: text is generated by initializing a fixed-size block ("canvas") of placeholder/mask +tokens and **iteratively denoising the whole block in parallel** over a small number of refinement +steps, with full bidirectional attention inside the block (block-autoregressive across blocks). +DiffusionGemma's stated config: canvas length 256, "Maximum number of Denoising Steps = 48", +temperature linear decay 0.8→0.4 (model card, fetched). The headline is generation **speed** (vendor +"up to 4x faster", "1000+ tok/s on one H100"), at an explicitly **lower output quality than the +autoregressive Gemma 4** ("DiffusionGemma's overall output quality is lower than standard Gemma 4", +blog, fetched). This is fundamentally a different **decode + training objective**, not a knob on a +causal LM. + +## Taxonomy (§C12 — axes touched) +architecture: **diffusion-LM** (block / masked discrete diffusion) — NOT decoder-only · training-stage: +base → diffusion-adapted · modality: text · context/position: full bidirectional attention within a +block + RoPE · size-band: DiffusionGemma is mid/large-MoE (25.2B total / 3.8B active); the only +sub-1B reference is the dLLM Qwen3-0.6B conversion. **No repo checkpoint sits on the diffusion-LM axis.** + +## Objective (§C13) +- type: finetune (the only conceivable on-repo route is *adapting* an existing AR checkpoint into a + diffusion-LM — DiffuLLaMA/Dream/dLLM style — never an architecture knob on the AR model) +- finetune only: base checkpoint **cannot be used as-is** — see below. Resolved on-disk AR candidate + that the literature actually converts: `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` + (the 1.19B-token faithful baseline, PPL 28.65, verified this run). · adaptation method: continued-pretrain / + SFT under a masked-diffusion (MDLM) or BD3LM denoising loss (dLLM 2602.22661 does exactly this on + Qwen3-0.6B) · catastrophic-forgetting probe: **cannot be defined comparably** — see Baseline + win. + +## Exact recipe +**Abstract/blog-only for the subject; training rows are `inferred` from background literature.** + +| Hyperparameter | Value | Flag | Provenance | +|---|---|---|---| +| Decode paradigm | block diffusion, canvas=256, denoising steps ≤48, temp 0.8→0.4 | reported | DiffusionGemma model card (fetched 2026-06-17) | +| Base architecture | Gemma-4 26B-A4B MoE encoder-decoder (8/128 experts +1 shared, 30 layers) | reported | model card | +| Params | 25.2B total / 3.8B active; vocab 262K; context up to 256K | reported | model card | +| Training objective | discrete (masked/absorbing) diffusion denoising loss | inferred | BD3-LM 2503.09573; dLLM 2602.22661 (model card says "discrete diffusion", not which) | +| AR→diffusion adaptation cost | "~10B tokens recovers most of base AR accuracy" | inferred | DiffuLLaMA/Dream adaptation summary (fetched search) | +| Adaptation method on Qwen3-0.6B | SFT only (no continual pretrain), MDLM & BD3LM objectives | reported (for dLLM, not DiffusionGemma) | dLLM 2602.22661 (fetched) | +| Optimizer / LR / batch / warmup / precision | not reported | not reported | neither blog nor card states training HPs | +| DiffusionGemma training token budget | not reported | not reported | model card: "doesn't provide total training token budget" | + +## Recommended budget (scaled) +- paper budget + where it's stated: **DiffusionGemma states no training token budget** (model card, + fetched). The only numeric AR→diffusion adaptation cost in the literature is the **inferred** "~10B + tokens recovers most of base AR accuracy" (DiffuLLaMA/Dream adaptation summary). +- scaling reasoning: a §C5.2 launchable `TOKEN_BUDGET` requires a defensible per-model number. The + subject release gives none; the only candidate (~10B tokens, inferred, on a 7B AR base) does not + scale cleanly to 596M and — more importantly — would feed a **diffusion training loss the repo's + trainer does not implement**. There is no honest budget to hand to a runnable ablation. +- TOKEN_BUDGET per model (source attached — §C5.2): **cannot be determined for a runnable ablation** — + no diffusion trainer/loss exists in-repo and the subject reports no budget. (Were a diffusion + trainer ever built, the dLLM Qwen3-0.6B precedent — SFT, 10–20 epochs, seq 512–4096 — is the + nearest starting point; recorded in the proposal, not as a launch budget.) + +## Framework / runtime fit (§C14) +- recommendation: **pytorch** — both repo baselines (SmolLM2-135M, Qwen3-0.6B) are PyTorch; any + adaptation would inherit that framework (jax_vs_pytorch_tradeoffs.md, read this run: JAX is not + meaningfully faster on one GB10 and the verify story is PyTorch-only). +- portability / kernel flags: the entire reference stack (BD3-LM `kuleshov-group/bd3lms`, dLLM, Dream) + is PyTorch, so framework is not the blocker. The blocker is that none of it is a *minimal diff* on a + causal-LM trainer — it is a **new training loss (diffusion denoising) + a new parallel-denoising + inference loop + a new generative-quality eval**. That is new model-class code, not a fused kernel. + +## Baseline + win condition +- baseline checkpoint: nearest AR candidate verified on disk this run — + `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` + (PPL 28.65 @ 1.19B tokens; provenance: that build's README.md + `results/qwen3_baseline2tpp_after.txt`, + vocab 151,936 from `Qwen3-0.6B/model.py:37`, all read this run). +- win = **cannot be defined under the standing §C10 suite.** eval-harness text-lm-v2 measures **exact + autoregressive next-token perplexity** with the model's own tokenizer. A diffusion-LM exposes only an + **ELBO upper bound on NLL**, not exact PPL (fetched: 2604.02718, 2512.10858 — "the loss only indicates + an upper bound on NLL"; the field has moved to *generative* perplexity, which the suite does not + implement). So neither the target metric nor the catastrophic-forgetting probe (a §C10 PPL number) is + comparable to the AR baseline — the win condition itself is undefinable without a new suite version. + +## Research-taste verdict (§C15.2) +- taste_score: 3.0 · axes: mechanism **sound** (block diffusion is a real, ICLR-Oral-grade mechanism — + parallel denoising for decode-speed) · evidence **vendor-only for the subject** (DiffusionGemma is a + blog+card with no paper, no ablations, and a self-reported quality *regression* vs AR; background + papers are honest that DLMs "still trail their AR counterparts" — dLLM, fetched) · reproduction + **AR-conversion replicates broadly** (DiffuLLaMA/Dream/dLLM independently convert AR→diffusion, + including Qwen3-0.6B) **but no sub-1B PPL win is reported anywhere** · scaling-to-our-scale **poor / + unknown** (gains are a decode-*speed* story at serving scale; quality at ≤596M is reported worse, not + better) · ROI **low** (the deliverable would be a slower-quality model measured by a metric our suite + can't compute) · simplicity/blast-radius **very high blast radius** (new loss + new inference loop + + new eval suite — not a minimal diff) · safety **fine** (CPU/analytic; no box risk). +- The mechanism is real and topical, which is why it scored as a high-buzz candidate (triage score 10). + But buzz is a discovery signal, not priority (§C15). On deep read it fails the three things that set + priority: there is **no model of the right class in the repo** (DiffusionGemma is a 25.2B MoE + diffusion-LM; both repo checkpoints are AR decoder-only); the only on-repo route (AR→diffusion + adaptation) needs **new model-class machinery** — a diffusion training loss, a parallel-denoising + decoder, and a generative-quality eval — none of which exist, so it is not a minimal-diff ablation; + and even if built, the standing §C10 suite **cannot produce a comparable win number** for a diffusion + LM. Score 3.0: real mechanism, but propose-only at our scale, low ROI, large blast radius. + +## Reproductions & criticism +- DiffusionGemma itself: **no independent reproduction** as of 2026-06-17 (vendor announcement + + NVIDIA partner blog only). Queries run this run: "DiffusionGemma ... arxiv 2026"; "block diffusion + LM BD3-LM reproduction didn't work / no improvement small scale"; "adapt AR LLM to diffusion ... + continued pretraining tokens"; "diffusion LM perplexity not comparable ELBO bound ... small scale". +- Block diffusion as a method **does replicate**: BD3-LM (ICLR 2025 Oral), Dream-7B, DiffuLLaMA, and + dLLM (2602.22661) independently train/adapt diffusion LMs — the last explicitly converts **Qwen3-0.6B** + to MDLM/BD3LM. Honest finding from those: DLMs **"still trail their AR counterparts on most knowledge + and reasoning benchmarks"** (dLLM, fetched) and **no sub-1B PPL win is published**. +- Eval criticism (fetched 2604.02718 / 2512.10858): diffusion-LM likelihood is an **ELBO bound**, not + exact NLL — directly comparable perplexity against an AR baseline is not well-defined; the field uses + generative metrics instead. This is the decisive reason the §C10 win condition cannot be stated. + +## Failure modes & abort criteria +(Recorded for the proposal; this technique is not launchable as a §C5 run.) +- No diffusion training loss in `train_qwen3.py` / SmolLM2 trainer → the run cannot even smoke-test + (§C5.0) against the AR loss; building one is new model-class code (propose-only, §C4.2). +- Metric incomparability: any "PPL" emitted by a diffusion-LM is an ELBO bound, so an eval-harness + number would be a category error vs the AR baseline (§C10) — an automatic inconclusive-by-construction. +- DiffusionGemma is 25.2B-A3.8B MoE @ 262K vocab — far outside the GB10 trainable band even ignoring + the architecture mismatch. + +## GB10 feasibility (§C1) +- Memory arithmetic (subject model, illustrative): 25.2B params × 2 B (bf16) ≈ **50.4 GB weights alone**, + before grads/optimizer/activations/262K-vocab logits — and §C1 plans ≤60% of the ~119 GB pool (~71 GB). + Even inference of the released model is borderline; **training is infeasible** on-box, and the 262K + vocab would force chunked-CE were it AR (§C1) — moot, since it isn't AR. +- Modality/architecture fit: the repo's only trainable checkpoints are **AR decoder-only text LMs** + (SmolLM2-135M: 134,515,008 params, vocab 49,152, PPL 15.37; Qwen3-0.6B: 596M, vocab 151,936, PPL 28.65 + — all read this run). **There is NO diffusion-LM checkpoint, trainer, or eval here.** Per §C12 a + technique needing a model class the repo lacks is at most propose-only. +- aarch64 deps: not blocking and not the deciding factor — the reference stack (bd3lms, dLLM, Dream) is + pure PyTorch + HF, which already runs on this box; the blocker is the missing model class + eval, not + a wheel. +- The probe is the launch authority (§C5.3) — moot here: ml-research launches nothing, and there is no + runnable script to probe. + +## Verdict +**propose-only** — block diffusion needs a diffusion-LM model class the repo does not have (the subject +DiffusionGemma is a 25.2B MoE diffusion-LM; both repo checkpoints are AR decoder-only), the only on-repo +route (AR→diffusion adaptation) requires new training-loss + inference + eval machinery rather than a +minimal-diff ablation, and the §C10 suite cannot produce a comparable win number for a diffusion LM. +Wrote `research/proposals/block-text-diffusion.md` (§C4.2). diff --git a/research/briefs/hybrid-attention-rethink.md b/research/briefs/hybrid-attention-rethink.md new file mode 100644 index 0000000..8de988e --- /dev/null +++ b/research/briefs/hybrid-attention-rethink.md @@ -0,0 +1,122 @@ +# Brief: Rethinking the Role of Efficient Attention in Hybrid Architectures (hybrid-attention-rethink) +- researched: 2026-07-19 · by: ml-research · fetch_level: fulltext-arxiv-html +- paper_date: 2026-06-13 (cutoff_3m this run: 2026-04-19) +- modality: text · verdict: propose-only +- objective: pretrain-ablation (§C13) · taste_score: 8 (§C15.2) + +## Sources (all fetch-verified this run, §C3) +| url | what it is | accessed | replication status | +|---|---|---|---| +| https://arxiv.org/abs/2606.15378 | abstract, authors (Qiao, Xu, Xiao … Zhiyuan Liu; Tsinghua + OpenBMB), cs.CL/cs.LG | 2026-07-19 | primary | +| https://arxiv.org/html/2606.15378 | full text (setup, ablations, NoPE table, limitations) | 2026-07-19 | primary | +| WebSearch "…Large-Window Laziness reproduction/ablation" | reproduction hunt | 2026-07-19 | no independent reproduction found | +| https://arxiv.org/html/2603.22473 (Component Ablation for Hybrid LMs) | adjacent, not a reproduction | 2026-07-19 | independently discussed, not re-measured | + +## What it changes +A systematic empirical study of **hybrid attention + efficient-attention** LMs (full attention interleaved +with sliding-window attention (SWA) and recurrent sequence mixers — Mamba-2, Gated DeltaNet, Lightning +Attention). Three findings: (1) **scaling** — efficient-attention design controls *how fast* long-context +capability EMERGES, not its ultimate level; different hybrids **converge** to comparable long-context +performance under sufficient training. (2) **mechanism** — long-range retrieval is carried by the *full* +attention layers; efficient attention shapes the optimization trajectory ("Large-Window Laziness": larger +SWA windows delay retrieval-head formation in the full-attention layers). (3) **design** — applying **NoPE +(no position embedding) to only the full-attention layers** of a small-window (SWA-128) hybrid substantially +improves long-context with negligible short-context cost. + +## Taxonomy (§C12 — axes touched) +architecture: **hybrid (full-attention + SSM/SWA efficient mixer)** — NEW family for the repo · size-band: +tiny/edge (15M–477M tested) · training-stage: base (from-step-0) · modality: text · context/position: +long-context (16K→32K), the finding is about **RoPE-vs-NoPE placement** in the attention layers · specialization: general. + +## Objective (§C13) +- type: **pretrain-ablation** (architectural, from step 0). + +## Exact recipe +Fetch level fulltext-arxiv-html; Appendix-C training details were not in the extracted body → those rows are `inferred`/`not reported`. + +| Hyperparameter | Value | Flag | Provenance | +|---|---|---|---| +| Model sizes (non-embed) | S1 15M · S2 31M · S3 65M · S4 104M · **S5 477M** | reported | §experimental setup | +| Context length (pretrain) | **16K** (extend to 32K eval) | reported | "pretrained with a 16K context length" | +| Token budget | D ∈ {100N,200N,300N,400N,500N,1000N}, N=non-embed params; largest actual ≈**100B** (at S5) | reported | setup + limitations ("at most ≈100B") | +| Data | 1:1 mixture of long + short datasets | reported | setup | +| Optimizer | **Muon** (Jordan et al. 2024) | reported (name only) | Table 8 / App. C | +| Peak LR / schedule / warmup / wd / batch(tokens) | not reported (App. C not extracted) | not reported | — | +| Efficient mixers compared | SWA (window **128 / 512 / 2048**); Lightning Attention; **Mamba-2**; **Gated DeltaNet**; full-attn baseline | reported | §setup | +| Layer placement | **1:1 interleaved** full:efficient (main); **1:3** sparse ≈ same val loss; head-wise mixing no advantage over layer-wise | reported | §6.1, §6.2 | +| Headline design knob | **NoPE on full-attention layers only** of SWA-128 hybrid | reported | §6 / Table 2 | +| NoPE gain (S5, ≈100B tok) | RULER-NIAH 46.13→**52.88** (+6.75); LongBench 65.91→**82.31** (+16.40); ShortAvg 41.31→41.32 (~0) | reported | Table 2 | +| Precision / init | not reported | not reported | — | + +## Recommended budget (scaled) +- Paper budget: D=100N–1000N tokens (N=non-embed), largest run ≈100B at S5(477M) ≈ ~200N. +- Scaling reasoning [inferred]: the load-bearing finding is about **emergence SPEED** — visible EARLY in + training, which a small token budget can see. Mirror the 596M study's approach: a from-scratch hybrid at + **~200–370M** on a **token-budget ladder** (e.g. 42M/168M/420M-analog, i.e. ~100N–1000N scaled to the box), + which is exactly the instrument that measured "the disappearing win." A full ≈100B-token S5 replication is + off-box; the emergence-speed + NoPE-placement questions are answerable at the box's ~1–2B-token scale. +- TOKEN_BUDGET per model — **N/A for existing checkpoints**: this needs a NEW hybrid build (below). Proposed + target: a ~200–370M hybrid, ~2–4 tok/param proxy budgets per arm (~0.5–1.5B tokens), single-variable ladder. + +## Framework / runtime fit (§C14) +- recommendation: **jax** — user-approved for this build; SSM/recurrent-mixer scans are `jax.lax.associative_scan`-native. + JAX+Flax now installed + verified on the GB10 (jax 0.11.0, associative_scan runs). §C14: on one GB10 JAX ≈ PyTorch + speed (±10-20%, `jax_vs_pytorch_tradeoffs.md`) — the reason is design-fit, not speed. +- portability / kernel flags: the paper's **Muon** optimizer exists in-repo (PyTorch, `normuon.py`) → a **JAX Muon + port** is required (portability cost, flagged). Mamba-2 / Gated DeltaNet reference impls are PyTorch/CUDA → the + scan must be re-implemented in JAX (novel-design cross-check at ~1e-2, not bit-exact). CCE fused CE (validated) is + PyTorch — the 152k-vocab CE in JAX needs its own chunked/fused path (or a smaller vocab). + +## Baseline + win condition +- baseline checkpoint: **none exists** — the repo has only dense-attention checkpoints; this technique's baseline + is the hybrid's OWN full-attention arm (the paper's full-attn baseline), built in the same run. +- win = the STUDY result under the repo's evidence standard: single-variable iso-FLOP arms (attention-fraction / + SWA-window / mixer-type / NoPE-on-full-attn), ≥3 seeds, BPB CIs on ≥2 corpora + a long-context retrieval probe + (RULER-NIAH-style), beating the noise floor. The headline object is the **emergence-speed curve** (quality vs + token budget per hybrid) + a **validation of the NoPE-on-full-attention** recommendation at box scale — not a + single checkpoint beating another. + +## Research-taste verdict (§C15.2) +- taste_score: **8** · axes: mechanism **strong** (Large-Window Laziness is a mechanistic explanation, not a curve + fit) · evidence **strong** (5 scales, systematic, honest limitations, reputable group) · reproduction **none yet** + (paper ~5 weeks old — a risk, not a red flag) · scaling-to-our-scale **excellent** (sub-1B IS the paper's regime; + emergence-speed is early-visible so the box can see it) · ROI **high** (new architecture family + a studiable + question with direct continuity to "the disappearing win") · simplicity/blast-radius **moderate** (needs a new + build + JAX ports of Muon/SSM) · safety **fine** (from-scratch, reversible). +- Why 8: strongest possible scale-fit for a novel-architecture study, a real mechanistic finding, and a headline + (efficient-attention affects emergence SPEED, converging under training) that is the *same shape* as this repo's + crown-jewel result — an unusually coherent next chapter. Held below 9 by the new-build cost, the JAX porting + surface (Muon + SSM scans + large-vocab CE), 16K-context memory tightness on one box, and zero reproductions yet. + +## Reproductions & criticism +No independent reproduction found as of 2026-07-19 (queries: exact title + "reproduction/ablation/Large-Window +Laziness"; only the paper's own arXiv/HF pages + adjacent-but-distinct work — Component Ablation 2603.22473, Every +Attention Matters 2510.19338 — surfaced). Zero-reproduction + a from-scratch-build requirement → `propose-only`. + +## Failure modes & abort criteria +- **16K-context memory blow-up** on one GB10: full-attention layers at 16K × batch can exceed the pool. Abort/mitigate: + probe peak mem (§C5.3); if > 60% of pool at micro_batch=1, drop pretrain context (e.g. 4K) and study emergence-speed + at shorter context, or shrink the target size. +- **JAX SSM-scan / Muon port incorrect**: verify cross-check vs a PyTorch reference op must pass ≤1e-2 before any + training (novel-design gate). A failed cross-check → discard, do not train. +- **grad-norm > X or NaN/Inf** at any step → instant abort (recurrent mixers can be init-sensitive). +- **no emergence-speed separation** between hybrids after the first ~30% of the token ladder when the paper predicts + an early gap → the effect isn't reproducing at box scale; report the null (directional), don't chase it. + +## GB10 feasibility (§C1) +- Memory (analytic, ~119 GB pool, plan ≤60%): a ~300M-param hybrid bf16 ≈ 0.6 GB params + 0.6 GB grads + Muon/AdamW + optimizer state (~2.4–4.8 GB) — trivial. The real cost is **activations at 16K context**: full-attention layers + scale O(seq²); with a 1:1 SWA-128 hybrid the attention memory is bounded by the window on half the layers, but the + full-attn half at 16K is the pressure point → keep micro_batch small / consider 4K pretrain context (probe decides, + §C5.3). SSM state is small. Vocab: reuse Qwen3's 151,936 → **chunked/fused CE required** (§C1) — a JAX path is needed + (the validated CCE kernel is PyTorch). +- Modality fit: text — the repo's home turf; but there is **no hybrid baseline checkpoint** → propose-only. +- aarch64 deps: JAX+Flax already import + run on the box (verified 2026-07-19); no extra CUDA-x86-only dependency + identified (Mamba-2/GDN scans re-implemented in native JAX, not the CUDA `mamba_ssm` package). +- Probe is the launch authority (§C5.3); this brief launches nothing. + +## Verdict +**propose-only** — requires a NEW from-scratch hybrid model (§C4.2, no hybrid checkpoint exists); the build is +user-approved and specced at `research/proposals/build-hybrid-ssm-attention-jax.md` (JAX/Flax, user-confirmed +2026-07-19). Next: `/from-scratch-build` (architecture-design phase consumes this brief; pauses for approval at +each phase, GPU-gated at training). diff --git a/research/briefs/vibethinker-small-reasoning.md b/research/briefs/vibethinker-small-reasoning.md new file mode 100644 index 0000000..7765761 --- /dev/null +++ b/research/briefs/vibethinker-small-reasoning.md @@ -0,0 +1,102 @@ +# Brief: VibeThinker-3B — Spectrum-to-Signal verifiable-reasoning post-training (vibethinker-small-reasoning) +- researched: 2026-06-17 · by: ml-research · fetch_level: fulltext-arxiv-html +- paper_date: 2026-06-15 (cutoff_3m this run: 2026-03-17) +- modality: text-lm · verdict: runnable-now +- objective: finetune (§C13) · taste_score: 6.0 (§C15.2) + +## Sources (all fetch-verified this run, §C3) +| url | what it is | accessed | replication status | +|---|---|---|---| +| https://export.arxiv.org/api/query?id_list=2606.16140 | arXiv API metadata (published 2026-06-15T02:57:19Z, cs.AI/cs.CL, v1) | 2026-06-17 | primary source | +| https://arxiv.org/html/2606.16140 | full paper HTML — recipe, stages, limitations, hypothesis | 2026-06-17 | claim, unreplicated (1 day old) | +| https://arxiv.org/abs/2606.16140 | abstract / authors (WeiboAI team) / categories | 2026-06-17 | primary source | +| https://huggingface.co/api/models/WeiboAI/VibeThinker-3B | model weights exist, ungated, sha 51e5928c3cc79ad954fc7a66cc17aa91be7581d7 | 2026-06-17 | weights released (not independently re-evaluated) | +| https://news.ycombinator.com/item?id=45910410 | HN thread on predecessor VibeThinker-1.5B (14 comments) | 2026-06-17 | independently discussed, not re-measured | +| https://news.ycombinator.com/item?id=48562111 | HN thread on the 3B (3 comments, 1-day-old) | 2026-06-17 | independently discussed, not re-measured | +| WebSearch "VibeThinker-3B … reproduction" | landscape: HF papers, GitHub WeiboAI/VibeThinker, VentureBeat | 2026-06-17 | no independent repro found | +| WebSearch "VibeThinker 1.5B … contamination/general knowledge" | predecessor critique + the team's contamination defense (AIME25/HMMT25 post-date the base) | 2026-06-17 | mixed: defense + practitioner skepticism | + +Note: VentureBeat "Why Weibo's tiny VibeThinker-3B has the AI world arguing over benchmarks again" surfaced in search results but the page returned HTTP 429 on two WebFetch attempts this run — recorded as a not-fetched lead, NOT cited as evidence. + +## What it changes +VibeThinker-3B is a post-training *recipe*, not an architecture change: starting from a dense base (the paper uses Qwen2.5-Coder-3B), it applies the "Spectrum-to-Signal" pipeline — (1) curriculum-based SFT (broad-coverage distillation of multi-path reasoning traces, then a hard-sample second stage), (2) multi-domain RL with verifiable rewards (their MGPO/GRPO-family algorithm over Math→Code→STEM, rewards from answer-checking and sandbox code execution), and (3) offline self-distillation on the model's own high-"learning-potential" trajectories, plus an instruct-RL pass. The thesis (Parametric Compression-Coverage Hypothesis) is that *verifiable* reasoning is a compressible "signal" that a tiny model can hold, even though broad knowledge ("spectrum") needs many parameters — so a sub-1B base should be able to absorb the reasoning gain. For our purposes the importable core is: **curriculum SFT on reasoning traces → GRPO with a verifiable (correctness) reward**, which maps directly onto the repo's `research/posttrain_losses.py` (SFT CE + GRPO group-advantage + clipped surrogate + k3 KL). + +## Taxonomy (§C12 — axes touched) +architecture: decoder-only · training-stage: reasoning (RLVR/GRPO) on a base checkpoint · modality: text · specialization: code+math · size-band: SLM/tiny (our base is 596M) · context/position: RoPE θ=1e6 / GQA (inherited unchanged from the Qwen3-0.6B base) · openness: fully-open (weights+code on HF/GitHub). + +## Objective (§C13) +- type: finetune +- base checkpoint: `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` (verified on disk this run; 596,049,920 params, FineWeb-Edu val PPL 28.65, vocab 151,936, seq_len 4096, PyTorch). NOTE: the paper's base is Qwen2.5-Coder-3B (a *code-pretrained, 5× larger* model); our base is a knowledge-dense FineWeb-Edu reproduction ~10× *under*-trained (2.14× PPL gap to real Qwen3-0.6B). This is a major extrapolation gap, recorded in Research-taste and Failure modes. +- adaptation method: curriculum **SFT** (reasoning-trace distillation, masked-completion CE) → **GRPO** with a verifiable correctness reward (math: answer-check; code: sandbox pass/fail). Both losses already implemented + unit-tested in `research/posttrain_losses.py`; the GPU launch routes through `/post-train` → `/ablation-runner` (§C5/§C11). Self-distillation/instruct-RL stages are out of scope for a first run (kept as later phases). +- catastrophic-forgetting probe (§C10, must not regress): **FineWeb-Edu held-out PPL with the model's own Qwen3 tokenizer** (the base's 28.65 number, same `eval_original_vs_repro.py` 300k-token slice) — the general-competence metric the paper itself admits is at risk ("knowledge-intensive benchmarks still expose a clear gap"; GPQA-Diamond lags). This becomes a required `metrics` field on the eventual `finetune` run. + +## Exact recipe +All values from `arxiv.org/html/2606.16140` (fulltext) unless flagged. Paper trains a 3B base; our base is 596M, so transferred values are `inferred`. + +| Hyperparameter | Value | Flag | Provenance | +|---|---|---|---| +| Base model (paper) | Qwen2.5-Coder-3B base | reported | "Qwen2.5-Coder-3B base, a compact 3B dense foundation model" | +| Base model (ours) | faithful Qwen3-0.6B repro ckpt (596M) | inferred | repo-fit substitution (Phase 5) | +| SFT stage-1 optimizer | not stated (AdamW assumed) | inferred | paper omits; modern-LLM + repo convention | +| SFT stage-1 batch size | global batch 128 | reported | "global batch size of 128" | +| SFT stage-1 peak LR | 5e-5, cosine → 8e-8, 5% linear warmup | reported | "initial learning rate to 5×10⁻⁵ … cosine annealing … 8×10⁻⁸ … 5% linear warmup" | +| SFT stage-1 epochs | 5 | reported | "trained for 5 epochs" | +| SFT stage-2 (hard) | +2 epochs, same HPs, on hard subset | reported | "additional 2 epochs … exact hyperparameter configuration from the first stage" | +| SFT hard-sample filter | reasoning trace ≥5K tokens; error-rate ≥0.75 (ref = VibeThinker-1.5B) | reported | data-filtering description | +| SFT data hygiene | n-gram filter, LLM query assessment, answer-check + sandbox-exec verification, eval-set de-contam | reported | quality-control section | +| RL algorithm | MGPO (MaxEnt-Guided Policy Optimization, GRPO family) | reported | "MaxEnt-Guided Policy Optimization (MGPO) retained from 1.5B work" | +| RL context window | single 64K long-context | reported | "single 64K long-context window" | +| RL domain order | Math → Code → STEM (sequential) → Instruct RL | reported | training-sequence description | +| RL reward (math/code) | binary correctness (answer-check / sandbox pass) | reported | "correctness binary signals" | +| Long2Short reward λ | 0.2 (max redistribution magnitude, zero-sum) | reported | "λ = 0.2 controlling maximum redistribution magnitude" | +| RL clip ε / KL β / steps / samples | not reported | reported-as-absent | "Not stated: clipping coefficient ε … number of training steps … KL penalty coefficients" | +| Self-distillation | length-normalized NLL "learning-potential" score; pick mid-high band per domain length bucket | reported | stage-3 description | +| Instruct RL | rule-based validators + rubric reward models | reported | stage-4 description | +| Eval sampling | T=1.0, top-p=0.95, top-k=-1; 64 gens/math problem | reported | evaluation protocol | +| Total compute / dataset sizes | not reported | reported-as-absent | "no information on total training compute … dataset sizes … samples" | + +## Recommended budget (scaled) +- paper budget + where it's stated: SFT = 5 epochs broad + 2 epochs hard (token/sample counts **not reported**); RL step/sample counts **not reported**; total compute **not reported** (a real transparency gap). The only firm SFT anchors are batch=128, LR 5e-5→8e-8 cosine, 5%/-warmup. +- scaling reasoning: with no paper token count, scale by epochs over the *prepared* dataset, not a token target. **SFT (inferred):** with the repo's measured ~7,300 tok/s, a ~50–150M-token reasoning-trace SFT (≈ a few epochs over a ~20–40M-token curated trace set, e.g. an OpenR1-Math subset) is a 2–6 h job — a cheap, low-blast-radius first arm. **GRPO (inferred):** budget by *rollouts*, not pretrain tokens; a first GRPO arm of ~2k–5k prompts × group size 8–16 × a few epochs is the right exploratory size at our scale (the paper's 64K context is **not** affordable here — use 2k–4k completion length, an explicit downscale flagged inferred). LR for GRPO ~1e-6 (inferred, standard small-model RLVR). +- TOKEN_BUDGET per model (source attached — §C5.2): + - **Qwen3-0.6B (target):** SFT arm ≈ 100M tokens over the prepared trace shards (inferred: epochs × dataset size, paper gives only batch/LR/epochs); GRPO arm budgeted as ~3k prompts × 8 samples × ≤4k tokens ≈ exploratory, ETA computed by ablation-runner's probe. First run = **SFT only** (cheapest path to a measurable signal); GRPO is a queued follow-on. + - **SmolLM2-134M:** out of scope for the first run — only the Qwen3 base is named here; a SmolLM2 reasoning post-train is a separate future candidate. + +## Framework / runtime fit (§C14) +- recommendation: **pytorch** — the base checkpoint is PyTorch (§C14(a), inherited), the repo's `posttrain_losses.py` is PyTorch, and you cannot cheaply port an existing checkpoint. `jax_vs_pytorch_tradeoffs.md` (read this run) confirms JAX is **not meaningfully faster** on a single GB10 for a sub-1B transformer (±10–20%; measured PyTorch baseline ≈ 7,300 tok/s, Qwen3 mb4@4096) and carries an XLA-preallocation memory-safety tax — no reason to switch. +- portability / kernel flags: GRPO rollout *generation* (autoregressive decode) is the hot path; a served/batched generation backend (vLLM via `/serving-bench`) would speed rollouts, but that is an optimization opportunity to flag, **not** an unattended kernel write (§C14, §C5). The paper's MGPO is a GRPO variant — we implement the standard GRPO surrogate in `posttrain_losses.py`, treating the MaxEnt/Long2Short reward shaping as an optional, separately-flagged delta (avoid bundling per §C18). + +## Baseline + win condition +- baseline checkpoint: `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` (provenance: faithful-build README + parent Qwen3-0.6B README read this run — val PPL 28.65, 596,049,920 params). +- win = a **TWO-part test (§C13)**: (i) a reasoning-task gain that clears the §C17 noise floor with seeds ≥ 3 and a paired-difference CI excluding 0 — measured via `/eval-harness` (target metric: an in-distribution math/code accuracy probe on a held-out split prepared with dataset-forge hygiene, e.g. GSM8K-style exact-match or a small MATH subset; the base's near-zero reasoning accuracy is the floor), **AND** (ii) the catastrophic-forgetting probe (FineWeb-Edu PPL under the Qwen3 tokenizer) does **not** regress beyond the noise floor from 28.65. A reasoning gain bought with PPL regression is a `loss` (§C13). Both numbers are required `metrics` on the `finetune` run. + +## Research-taste verdict (§C15.2) +- taste_score: 6.0 · axes: mechanism **strong** (curriculum-SFT→RLVR is the most-validated post-training recipe of 2025–26; verifiable rewards are honest, non-gameable signals; the compression-coverage thesis is coherent) · evidence **mixed** (clean contamination defense — AIME25/HMMT25 post-date the base — but a real transparency gap: no compute/dataset/sample counts, no base-vs-tuned general-benchmark table, and the bundle confounds SFT+RL+self-distill+instruct-RL, which §C18 forbids running as one arm) · reproduction **none yet** (paper 1 day old; predecessor 1.5B was widely replicable as weights but practitioners report a benchmark-vs-utility gap and narrow specialization) · scaling-to-our-scale **risky** (paper's base is a 3B *code*-pretrained model; ours is a 596M FineWeb-Edu repro ~10× under-trained — the "compressible reasoning core" claim is least tested exactly at our size/under-training) · ROI **good IF data exists** (the recipe is cheap and the SFT-first arm is low-cost/low-blast-radius; the gain is plausible because RLVR reliably lifts in-distribution reasoning even on small bases) · simplicity/blast-radius **good** (decompose into single-variable arms: SFT-only first, GRPO second — `posttrain_losses.py` already exists; no canonical-file edits) · safety **good** (a finetune in an `experiments/` dir; reversible; memory fits — §C1). +- One paragraph: This is a high-mechanism, low-novelty recipe — curriculum SFT then GRPO with verifiable rewards is the dominant reasoning-post-training playbook, and that *raises* my confidence in the direction even though the paper itself is buzz-heavy ("beats Gemini 3 Pro") in a narrow band it admits doesn't transfer to knowledge tasks. I weight mechanism + the SFT-first ROI up, and weight the missing-budget transparency, the confounded bundle (§C18), and the 3B-code-base → 596M-FineWeb-base extrapolation down. The decisive blocker is data: there is **no verifiable-reward dataset on disk**, so this cannot be `runnable-now`. Decomposed to an SFT-only first arm on a forged reasoning-trace set, with the FineWeb-PPL forgetting guard, it is a worthwhile mid-priority experiment — hence 6.0, above pure-buzz candidates but below a clean, data-ready single-variable ablation. + +## Reproductions & criticism +- The 3B paper is **1 day old (2026-06-15)** — **no independent reproduction found as of 2026-06-17** (queries: title + "reproduction results"; "VibeThinker-3B … small language model reproduction"; HN Algolia "VibeThinker" → only WeiboAI-origin and discussion posts, ≤3 comments on the 3B). Recorded as a finding: zero independent re-measurement, which pushes the first budget small. +- Predecessor **VibeThinker-1.5B** (Nov 2025, WeiboAI, also fully open) is the track record: the team's contamination defense is genuine (AIME25/HMMT25 released after the Qwen2.5-Math base, so the math gains are not pure leakage). But HN practitioners report (i) narrow specialization — "specifically trained on maths," general coding/explanation "break completely," repetition loops; (ii) a benchmark-vs-real-utility gap. For the 3B, HN testers note it "reliably writes working Python" but "takes shortcuts / writes wrong steps" and found 0 security bugs. Net: the *reasoning-on-verifiable-tasks* gain is credible; the *general-competence* claims are where to be skeptical — which is exactly what the forgetting probe guards. + +## Failure modes & abort criteria +All measurable from artifacts ablation-runner/eval-harness already produce: +- **Forgetting (the primary risk):** FineWeb-Edu PPL regresses > noise floor above 28.65 for 2 consecutive logged evals → abort (a reasoning gain bought with general regression is a `loss`, §C13). +- **RL instability:** grad-norm > 5× the SFT run's median, or NaN/Inf at any step → instant abort (GRPO + tiny under-trained base is init/LR-sensitive). +- **Reward hacking / KL blow-up:** k3 KL to the reference policy grows monotonically while train reward rises but the held-out reasoning probe does not → abort (policy is gaming the verifier, not reasoning). +- **No early signal:** after the first 25% of the SFT/GRPO budget, the in-distribution reasoning probe delta is below the §C10 noise floor → abort (recipe not transferring at 596M under-trained scale, the stated extrapolation risk). +- **Throughput:** rollout tok/s degraded > 30% vs the run's own probe (§C5.3) → flag overhead (RL generation overhead beyond plan). + +## GB10 feasibility (§C1) +- Memory (this run's arithmetic, 596M params): params bf16 1.19 GB + grads bf16 1.19 GB + AdamW fp32 m+v 4.77 GB + frozen reference policy (bf16, needed for GRPO ratio/KL) 1.19 GB = **~8.3 GB** weights/state. Activations dominate; the base pretrain measured **52.4 GB peak** at mb4×seq4096 with chunked CE — well under the 60%-of-119 GB ≈ **71 GB** plan cap. GRPO uses shorter completions (≤4k, downscaled from the paper's 64K), so it fits; the §C5.3 measured probe (ablation-runner) is the launch authority, not this estimate. +- vocab 151,936 > 64k → **chunked cross-entropy is mandatory** (§C1; the exact allocation that crashed the box on 2026-06-08). The repo trainer already does this. +- modality fit: text-LM, matches the repo baselines — no modality gap. +- aarch64 deps: no extra CUDA-only x86 library required — SFT+GRPO run on the in-repo `research/posttrain_losses.py` with the installed stack (torch 2.11+cu130, transformers 5.8.0, both importable this run); no `trl`/`verl` dependency (not installed, not needed). A sandbox code-verifier (for code-RL rewards) is the one extra component — pure-Python/subprocess, no x86 wheel issue — but code-RL is a later phase, not the first SFT arm. + +## Verdict +**runnable-now** (flipped 2026-06-17, research-loop S6 — SFT dataset now prepared on disk; see Addendum). Originally **needs-dataset** — the recipe, base checkpoint, adaptation method (SFT→GRPO via `posttrain_losses.py`), forgetting probe, budget, and memory fit are all concrete and on-box, but there is **no reasoning-trace SFT set or verifiable-reward GRPO prompt set prepared on disk** (`research/datasets/` is empty). dataset-forge must produce, for the Qwen3-0.6B base: (a) an SFT shard set of curated math/code reasoning traces (candidate source: `open-r1/OpenR1-Math-220k`, HF http 200, ungated, sha e4e141ec9dea9f8326f4d347be56105859b2bd68 — verified this run) with response-masking + a hygienic held-out reasoning eval split; and (b) for the queued GRPO follow-on, a prompt+verifiable-answer set (candidate `agentica-org/DeepScaleR-Preview-Dataset`, http 200, ungated, sha b6ae8c60f5c1f2b594e2140b91c49c9ad0949e29; eval anchor `HuggingFaceH4/MATH-500`, http 200, ungated, sha 6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be) with a checkable reward function. First run should be **SFT-only** (single variable, §C18); GRPO is a separate queued arm once SFT clears the forgetting probe. + +## Addendum (2026-06-17, research-loop S6) +Verdict flipped `needs-dataset` → `runnable-now`. The SFT dataset is now prepared on disk: +- **dataset:** `research/datasets/math-reasoning-openr1-math-220k/` — 125,000,592 train tokens (35,512 verified OpenR1-Math-220k reasoning traces, Qwen3 tokenizer, response-masked via per-sample `prompt_len`), held-out reasoning eval (66 docs / 232,918 tok, 0 leakage) + the §C13 FineWeb-Edu forgetting probe (202 docs / 167,242 tok). +- **forge ledger run:** `2026-06-17_qwen3-0.6b_openr1-math-220k` (type=dataset-prep, done); HF sha `e4e141ec9dea9f8326f4d347be56105859b2bd68`. +- **first run:** SFT-only (~100–125M tokens, single variable §C18) on `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt`; win-metric = held-out reasoning gain clearing the §C17 noise floor (seeds≥3, paired CI excludes 0) AND FineWeb-Edu PPL must not regress past 28.65 beyond floor (§C13). GRPO remains a separate queued arm. diff --git a/research/briefs/zeta-dual-whitening.md b/research/briefs/zeta-dual-whitening.md new file mode 100644 index 0000000..3232850 --- /dev/null +++ b/research/briefs/zeta-dual-whitening.md @@ -0,0 +1,92 @@ +# Brief: Zeta: Dual Whitening for Matrix Optimization via Coordinate-Adaptive Preconditioning (zeta-dual-whitening) +- researched: 2026-06-19 · by: ml-research · fetch_level: fulltext-arxiv-html +- paper_date: 2026-06-12 (cutoff_3m this run: 2026-03-19) +- modality: text · verdict: runnable-now +- objective: pretrain-ablation (§C13) · taste_score: 7.5 (§C15.2) + +## Sources (all fetch-verified this run, §C3) +| url | what it is | accessed | replication status | +|---|---|---|---| +| https://export.arxiv.org/api/query?id_list=2606.14187 | arXiv metadata (date 2026-06-12, cs.LG, authors Chen et al.) | 2026-06-19 | primary | +| https://arxiv.org/html/2606.14187 | full paper (Algorithm 2, §4 config, §4.1–4.4 results, App. A.5/D) | 2026-06-19 | claim, unreplicated | +| https://github.com/AIGCodeOS/aigcode_zeta_optimizer | authors' reference code (cited in abstract; not fetched/run here) | 2026-06-19 | author code | +| https://kvfrans.com/matrix-whitening/ | "What really matters in matrix-whitening optimizers?" (adjacent critical analysis) | 2026-06-19 | independently discussed, not re-measured | +| https://arxiv.org/abs/2509.02046 | "Fantastic Pretraining Optimizers and Where to Find Them" (optimizer-eval rigor critique) | 2026-06-19 | background | + +## What it changes +A drop-in **optimizer swap** (no architecture/data change). Zeta is a Muon-family +matrix optimizer that fixes a stated Muon weakness: Muon's Newton–Schulz (NS) step +assumes a well-conditioned input, but raw momentum matrices have severe +coordinate-wise scale heterogeneity (the paper verifies this with a chi-square +uniformity test). Zeta applies **dual whitening**: (1) *coordinate whitening* — +an AdamW-style per-entry second-moment normalization `G̃ = M/(√V+ε)` — then (2) +*spectral whitening* — `U = NewtonSchulz(G̃, K=5)`, with the standard Muon +update `ΔW = 0.2·√(mn)/(‖U‖_F+ε)·U`, decoupled weight decay. It is applied to **2D +hidden matrices** (attention + FFN projections); biases/LayerNorm/embeddings fall +back to AdamW — the **identical param-split shape as our verified NorMuon run**. + +## Taxonomy (§C12 — axes touched) +architecture: decoder-only · training-stage: base (pretrain) · optimizer · modality: text · size-band: SLM (0.6B) + +## Objective (§C13) +- type: pretrain-ablation (changes training from step 0 — the optimizer for 2D matrices) + +## Exact recipe +From `arxiv.org/html/2606.14187` Algorithm 2 + §4 "Training Configuration": + +| Hyperparameter | Value | Flag | Provenance | +|---|---|---|---| +| Zeta β₁ (momentum) | 0.95 | reported | §4 | +| Zeta β₂ (2nd moment) | 0.99 | reported | §4 | +| Newton–Schulz iters K | 5 | reported | §4 | +| NS coeffs (a,b,c) | 3.4445, −4.7750, 2.0315 | reported | "standard values, cited" | +| RMS scale | 0.2·√(mn)/‖U‖_F | reported | §4 (standard Muon scaling) | +| weight decay (decoupled) | 0.1 | reported | §4 | +| ε | not reported | inferred → 1e-8 | Alg.2 shows ε, no value | +| LR schedule | cosine, 1% warmup | reported | §4 | +| peak LR (Qwen3-0.6B) | 9e-4 | reported | §4 per-model table | +| param scope | 2D matrices → Zeta; biases/LN/embeds → AdamW | reported | §4 dual-path | +| paper batch / seq (0.6B) | 256 × 4096 = 1.05M tok/step | reported | §4 | +| paper budget (0.6B) | ~20.9B tok (20k iters) ≈ 35 TPP | reported | §4 | +| custom CUDA kernel | none (standard PyTorch matmul/elementwise) | reported | "not specified"; we reimplement | + +LR-tuning protocol (paper): each optimizer's LR is **tuned individually on +Qwen3-0.6B**, then unified for larger models (§4) — a clean, non-confounded control. + +## Recommended budget (scaled) +- paper budget: ~20.9B tok (35 TPP) on Qwen3-0.6B — far above our regime. +- scaling reasoning (inferred): match the **verified NorMuon-vs-AdamW ablation budget** so the two optimizer results are directly comparable on the SAME baseline/metric → Zeta's lift can be ranked against NorMuon's measured **+0.4743 bpb** win. +- **TOKEN_BUDGET — Qwen3-0.6B: 6-cell cohort (2 arms × 3 seeds), 640 steps × 65,536 tok = 41.9M tok/cell, 251.7M total** (source §C5.2: parity with run `2026-06-16_qwen3_normuon-vs-adamw`). Optional confirmation at 2000 steps if the 640-step signal is borderline. (SmolLM2-134M: not the target here — Qwen3 is the paper's tested model.) + +## Framework / runtime fit (§C14) +- recommendation: **pytorch** — the baseline (faithful Qwen3-0.6B) is PyTorch and the verify gate is PyTorch-only (§C14a); Zeta is a drop-in optimizer reimplemented in-repo as a single `.py` (exactly as `normuon.py` was), no port. +- portability / kernel flags: none. NS iteration = matmuls + element-wise ops → no custom CUDA, no x86-only dep, **aarch64-safe by construction** (we implement from Algorithm 2; the authors' code is reference only). A fused NS/whitening Triton kernel is a *future* optimization opportunity, not needed to run. + +## Baseline + win condition +- baseline checkpoint / recipe: `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` (596,049,920 params, single AdamW @ tuned LR; provenance: faithful build README + checkpoint inventory read this run). The A/B trains BOTH arms from random init at matched budget (Type-STEP0): **AdamW baseline vs Zeta(2D)+AdamW(1D)**. +- win = across-seed (3-seed) wikitext-2 **BPB** (text-lm-v2, §C10) for Zeta significantly **lower** than the AdamW baseline — 95% CI excludes 0 (`eval_stats.seed_delta_significant`) and clears the noise floor; code-PPL reported as the OOD check. **Bonus comparison:** rank Zeta's lift against NorMuon's measured +0.4743 bpb [0.4435, 0.5052] (same budget/baseline → "does this newer Muon-variant beat the one we verified?"). + +## Research-taste verdict (§C15.2) +- taste_score: 7.5 · axes: mechanism **strong** (whitening-conditions-NS is principled, chi-square-verified) · evidence **good but single-org** (clean per-optimizer LR control + β-grid robustness across 0.6B/1.7B/8B/MoE + downstream, no independent repro yet) · reproduction **none found as of 2026-06-19** · scaling-to-our-scale **strong** (paper tests OUR exact model Qwen3-0.6B) · ROI **high** (drop-in, ~4% wall-clock overhead, AdamW-equal memory, same cheap shape as the NorMuon win) · simplicity/blast-radius **low** (one optimizer, one .py) · safety **high** (optimizer-only, reversible). +- Weighting mechanism + evidence + ROI over buzz: this is a principled, cheap, single-variable optimizer ablation on the project's exact model, directly comparable to an already-verified win — the strongest "another one" in the candidate pool. The one real discount is zero independent reproduction (it's 7 days old), mitigated by the cheap first budget and the paper's clean controls. Below the NorMuon precedent only because that one is already verified in-repo; as a *new* pick it is the highest-ROI optimizer candidate. + +## Reproductions & criticism +- **No independent reproduction of Zeta found as of 2026-06-19** (paper is 7 days old). Queries run: "Zeta dual whitening optimizer Muon variant reproduction results"; arXiv/HN surfaced only adjacent Muon-variant work (MuonAll 2511.06086, Muon-in-ViT 2605.24770, Muon convergence 2509.15816), not Zeta itself. +- Adjacent critical context (cite, don't over-weight): kvfrans "What really matters in matrix-whitening optimizers?" (questions which whitening components actually matter — relevant since Zeta's gain is the *coordinate*-whitening add-on) and "Fantastic Pretraining Optimizers and Where to Find Them" (2509.02046, warns optimizer wins often shrink under honest matched-LR eval). Both raise the bar for our own controlled A/B. + +## Failure modes & abort criteria +- **LR-coupling confound** (Zeta's RMS-scaled LR is on a different scale than AdamW's; the verdict must not be an LR artifact). Mitigation: use AdamW-best 2.4e-3 (from the verified sweep) for the baseline and the paper's reported 9e-4 for Zeta; if the result is borderline, a 3-point Zeta LR mini-sweep is required before claiming a win. +- **NS instability** if coordinate whitening fails to isotropize → NaN/Inf in the optimizer step ⇒ **instant abort**. +- loss > AdamW-baseline-at-equal-tokens by >10% for 2 consecutive logged evals ⇒ abort. +- grad-norm > 10 or NaN/Inf at any step ⇒ instant abort. +- tokens/sec degraded > 15% vs the run's own §C5.3 probe (paper claims ~4% wall-clock overhead; a larger hit means the in-repo NS/whitening impl is inefficient) ⇒ flag, not necessarily abort. +- no eval signal after 25% of TOKEN_BUDGET when the paper predicts an early speedup ⇒ likely `inconclusive` at this budget. + +## GB10 feasibility (§C1) +- Memory (≤60% of ~119 GB pool): params 596M×2B (bf16) ≈ 1.2 GB; grads ≈ 1.2 GB; optimizer state **O(2mn)** for 2D matrices (M+V, fp32) + AdamW(2 states) for the rest ≈ AdamW-equal (~5 GB); activations + **chunked CE** over 151,936 vocab (REQUIRED, §C1 — already in `train_ablation.py`). The faithful 2-TPP run measured **52.4 GB peak** at this exact config; Zeta's O(2mn) state ≈ AdamW → same envelope, comfortably under 60%. +- modality fit: text decoder-only → the faithful Qwen3-0.6B checkpoint is the baseline. ✓ +- aarch64 deps: **none** — pure PyTorch, reimplemented in-repo from Algorithm 2 (the `normuon.py` pattern); no pip wheel, no x86 CUDA kernel. +- The §C5.3 measured probe (run by ablation-runner) is the launch authority; this is paper-math only. + +## Verdict +**runnable-now** — recipe complete, baseline checkpoint + FineWeb-Edu data already on the box (same pipeline as the de-confound cohort), memory fits the measured 52 GB envelope, pure-PyTorch/aarch64-safe. Queue it; launch the instant the GPU frees from the IMU-1 de-confound cohort (one-trainer-at-a-time, §C4.5). diff --git a/research/digests/2026-06-17.md b/research/digests/2026-06-17.md new file mode 100644 index 0000000..1cc58b8 --- /dev/null +++ b/research/digests/2026-06-17.md @@ -0,0 +1,57 @@ +# Research-loop digest — 2026-06-17 + +**Headline.** First full end-to-end iteration: after the box freed at 16:03 the loop ran +S0→S9 and **launched its first real training run** — a VibeThinker-style **SFT post-train +of the faithful Qwen3-0.6B** on 125M math-reasoning tokens (PID 1479165, ETA ~4.65 h, +smoke-passed + sentinel-armed). (An earlier 11:21 iteration correctly deferred — box was +busy with another session's ablation cohort.) + +## Experiments +- **LAUNCHED (in-flight):** `2026-06-17_qwen3-0.6b_vibethinker-small-reasoning` (type=finetune, + SFT-only, single-variable §C18). PID 1479165, ckpt rolling every 20 steps. Budget + **125,000,000 tok** (238 steps × 524,288), probe **7,465 tok/s**, peak **52.39 GB** (<71.4 GB + cap), **ETA ~4.65 h**. smoke=**PASS** (loss-oracle exact, byte-identical save/reload, resume + proven); §C5 preconditions in the ledger BEFORE spawn; **sentinel** PID 1479850 (kill_at 0.80); + detached (survives the session). Pre-SFT held-out reasoning PPL **14.262** recorded as the floor. + **S8 eval queued for tomorrow:** held-out reasoning accuracy (§C17 seeds≥3 / paired CI) + + §C13 FineWeb-Edu forgetting probe (must not regress past **28.65**). +- **Context (other-session work, not launched by this loop):** `2026-06-16_qwen3_normuon-vs-adamw` + recorded **verdict=win** (NorMuon −AdamW = **+0.474 BPB** wikitext, 95% CI [+0.443,+0.505], + 3 seeds, iso-FLOP) + a 3-arm AdamW LR-sweep robustness check — **all finished 16:03**. ⚠ This + win is **not yet §C19-verified** (read-only verifier owed before it's manuscript-eligible). +- **Manual builds (observed):** `prope10` (10% partial-RoPE) **died incomplete** at step 5,450; + `prope25` finished at val PPL **29.54** (loses to faithful 28.65 / IMU-1 23.52 at matched compute). + +## Found today +- **Pulse (5 candidates):** top — VibeThinker reasoning post-train (finetune, 10), Variable-Width + Transformers (pretrain-ablation, 8), OPSD self-distillation (finetune, 8). +- **Radar (5 GB10-feasible):** candidate `fastcontext-long-context-sft`; propose-only + `North-Mini-Code-30.5B` (MoE) and `SANA-WM` (video/world-model) — repo lacks those bases. + +## Proposals awaiting you (3 open) +- `block-text-diffusion` (new-build — DiffusionGemma class; repo has no diffusion-LM base) +- `build-north-mini-code-moe` (new-build — MoE code model) +- `build-sana-wm-world-model` (new-build — diffusion DiT world model) + +## Datasets +- `research/datasets/math-reasoning-openr1-math-220k/` — 125,000,592 SFT tokens (35,512 verified + OpenR1-Math-220k traces, response-masked), held-out reasoning eval + forgetting probe, 0 leakage. + Ledger run `2026-06-17_qwen3-0.6b_openr1-math-220k`; HF sha `e4e141ec…`. + +## Health +- **Two iterations today.** 11:21 deferred (foreign trainer, §C4.5). 19:35 ran full S0→S9 after + the box freed at 16:03. +- **Watcher bug, owned:** the GPU-free watcher's `pgrep -f 'train.*\.py'` self-matched its own + command line, so it never fired and would have timed out — caught on a manual "check it now", + fixed by proceeding. The buggy watcher is killed. +- Box healthy at launch: 0 foreign trainers, 104 GB free, disk 3,173 GB. Trainer + sentinel alive. +- Ledger grew 9→15 techniques, proposals 0→3, runs →7 (1 new in-flight). No dead §C3 endpoints. +- **Open item:** the NorMuon win is §C19-unverified. + +## Tomorrow +- S1 monitors the SFT run (ETA ~4.65 h → completes overnight); S8 evals it (reasoning accuracy + + §C13 forgetting probe vs 28.65). +- Run the **§C19 read-only verifier** on `2026-06-16_qwen3_normuon-vs-adamw` (re-derive headline + + CI from per-seed numbers; assert single-variable + iso-FLOP + seeds≥3) — it's the strongest + result and is currently unverified. +- `vibethinker` GRPO is a separate **queued** follow-on arm once SFT clears the forgetting probe. diff --git a/research/digests/2026-07-14.md b/research/digests/2026-07-14.md new file mode 100644 index 0000000..2159d86 --- /dev/null +++ b/research/digests/2026-07-14.md @@ -0,0 +1,55 @@ +# Research-loop digest — 2026-07-14 + +> **Status: CLOSED (S9-done).** This is the §C9 acceptance artifact — the first +> end-to-end autonomous iteration of the loop to reach S9 (prior digest was hand-made +> 2026-06-17). Scoped **CPU-light**: no S6/S7 trainer this night — the first unattended +> trainer stays behind the human GPU gate + recovery-chain wiring. Drove S0→S9 via the +> tested `loop_state.py` atomic writer; every artifact below verified on disk. + +## Headline +**First autonomous loop iteration closed clean (CPU-light, zero GPU work).** Pulse +surfaced 9 eligible techniques (+4 new ledger candidates); radar surfaced 11 +GB10-feasible models (+2 propose-only build proposals). Selected **zeta-dual-whitening** +— a Muon-whitening optimizer variant, verdict **runnable-now**, a 6-cell / 251.7M-tok +ablation at exact parity with the NorMuon-vs-AdamW run (so its lift ranks directly +against NorMuon's verified +0.4743 bpb). **Launch deliberately deferred** behind your +GPU approval + recovery-chain wiring; no trainer touched the box. + +## Experiments +- **S1** — no in-flight run (`in_flight_run=null`); nothing to resume/adopt. +- **S7** — 1 launch-ready technique (`zeta-dual-whitening`, runnable-now) **declined at the interactive gate** (CPU-light night). Not promoted to `queued`, so no unattended auto-launch is armed. +- **S8** — 0 finished-but-unjudged ablation/finetune runs (the 9 verdict-null runs are `eval`/`dataset-prep`/`scaling-fit` by design). Nothing to judge; no eval launched. + +## Found today +- **S2 community-pulse** → `research/pulse/2026-07-14_pulse.md` (9 eligible). Top-3: KronQ Kronecker-Hessian quantization (arXiv 2607.07964); Scalable Visual Pretraining (2607.09657); Long-Horizon-Terminal-Bench (2607.08964). +4 text-LM candidates → ledger (`kronq-quantization`, `linear-attention-routing`, `trust-region-policy-distillation`, `memorization-generalization-ft`). _[web items agent-reported; the selected technique's paper is the one verified downstream.]_ +- **S3 model-radar** → `research/radar/2026-07-14_radar.md` (11 GB10-feasible). Top-3: `openbmb/MiniCPM5-1B` (1.08B, apache-2.0); `naver/v-splade-quality` (330M SPLADE); `nvidia/nemotron-3.5-asr-streaming-0.6b` (638M). +2 propose-only build proposals (`build-minicpm5-1b`, `build-v-splade-quality`); 0 techniques routed (all in-window releases = repo-lacks-model). + +## Proposals awaiting you +10 open (via `ledger.py`) — the loop authorizes no build/deploy; a human owns those buttons (§C4.2): +- **new-build (5):** `build-minicpm5-1b`, `build-v-splade-quality` _(both filed tonight by radar)_, `build-north-mini-code-moe`, `build-sana-wm-world-model`, `block-text-diffusion` +- **needs-approval (5):** `lifecycle-north-star-2026-07` _(today's lifecycle verdict)_, `next-step-verdict-v2-onbox`, `next-step-verdict-2026-07`, `midtrain-context-extension`, `preference-dpo-plan` + +## Datasets +_(none expected — CPU-light night, S6 skipped)_ + +## Health +- S0 preflight: **OK** — `mem_available=83% disk_free=3114GB load1=1.17 trainers=none` (verified 2026-07-14T00:01Z); box freed after external TRL/DAPO run (PID 2796787) completed ~23:57. +- Prior state: parked at S5 with stale `iteration_date=2026-06-18`, `in_flight_run=null` (diagnosed: never auto-driven, not crashed). + +## Tomorrow +- **Ready to launch on your word:** `zeta-dual-whitening` (6-cell 2×3-seed, 251.7M tok, ~parity with `2026-06-16_qwen3_normuon-vs-adamw`). It answers "does this newer Muon variant beat the NorMuon win we verified?" — the single highest-ROI runnable ablation in the pool. +- **⚠️ Do NOT arm cron before wiring the recovery chain into `ablation-runner`.** Because zeta is `runnable-now` and top-ranked, the *headless* S7 fallback is auto-launch — so the first nightly cron fire would launch it **unattended**. Given the GB10's ~5–10h silent hard-lockup under sustained load, the `ScheduleWakeup`/`boot_resume.sh`/thermal-kill recovery chain (the stack that delivered the 420M sweep) must be wired into `ablation-runner`'s launch path first. +- **Sequencing:** (1) wire recovery chain → (2) you approve the GPU launch → (3) then arm cron for nightly autonomy. +- **Also open:** reconcile the `loop_state.py` schema fork (its `default_state()` still emits the old 7-key schema vs the live §C5 12-key file) — a latent bootstrap-on-corruption bug, harmless tonight (the atomic writer preserves all live keys). + +--- +_stage log (appended live):_ +- S0 `00:01Z` — anchored TODAY=2026-07-14, CUTOFF_3M=2026-04-14, RADAR_STALE=2026-07-07; preflight OK; stub digest written; → S1. +- S1 `00:02Z` — in_flight_run=null → no run to adopt; skip to S2 (no live trainer). +- S2 `00:11Z` — community-pulse: **9 eligible**, 4 new candidates added (kronq-quantization, linear-attention-routing, trust-region-policy-distillation, memorization-generalization-ft); techniques 17→21. Dead/degraded: YC-PaperClub, Reddit 429-recovered. [items agent-reported; selected one verified at S5] +- S3 `00:18Z` — model-radar: **11 GB10-feasible**; top: MiniCPM5-1B(1.08B), v-splade-quality(330M), nemotron-asr-0.6b. 2 propose-only build proposals filed (build-minicpm5-1b, build-v-splade-quality); 0 techniques routed (all in-window releases = repo-lacks-model). last_radar→2026-07-14. +- S4 `00:20Z` — selected **zeta-dual-whitening** (briefed, score 7.0, pretrain-ablation; Muon whitening variant, attaches to Qwen3-0.6B base). Ranked above vibethinker(10,finetune-3B) / kronq(8) — selector prefers briefed+attachable. → S5. +- S5 `00:21Z` — zeta brief exists, verdict **runnable-now** (6-cell, 251.7M tok, parity w/ normuon-vs-adamw; win=3-seed wiki BPB text-lm-v2). No re-brief needed (idempotent). +- S7 `00:21Z` — **launch DECLINED (interactive gate, CPU-light night)**. Zeta ready but deferred behind: (1) human GPU approval, (2) recovery-chain (ScheduleWakeup/boot_resume/thermal-kill) wired into ablation-runner. Technique stays `briefed` (NOT queued → no unattended auto-launch armed). → S8. +- S8 `00:22Z` — 0 finished-but-unjudged ablation/finetune runs; nothing to judge; no eval launched (CPU-light). → S9. +- S9 `00:22Z` — digest finalized (all §C9 sections). Iteration CLOSED. No trainer launched by design. iteration_date=2026-07-14. diff --git a/research/ledger/ledger.json b/research/ledger/ledger.json new file mode 100644 index 0000000..3c205ee --- /dev/null +++ b/research/ledger/ledger.json @@ -0,0 +1,2086 @@ +{ + "schema_version": 1, + "techniques": [ + { + "slug": "block-text-diffusion", + "title": "Block text diffusion LM head (DiffusionGemma-style parallel denoising decode)", + "modality": "text", + "source_url": "https://blog.google/innovation-and-ai/technology/developers-tools/diffusion-gemma-faster-text-generation/", + "paper_date": "2026-06-10", + "first_seen": "2026-06-12", + "status": "proposal", + "score": 10.0, + "brief_path": "research/briefs/block-text-diffusion.md", + "run_ids": [], + "objective": "finetune", + "taxonomy": [ + "architecture: diffusion-LM (block/masked discrete diffusion); training-stage: base->adapted; modality: text; context/position: bidirectional-within-block + RoPE; size-band: SLM/mid" + ], + "taste_score": 3.0 + }, + { + "slug": "minimax-sparse-attention", + "title": "MiniMax Sparse Attention (MSA): per-GQA-group top-k KV-block selection", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2606.13392", + "paper_date": "2026-06-10", + "first_seen": "2026-06-12", + "status": "candidate", + "score": 9.0, + "brief_path": null, + "run_ids": [], + "objective": "any", + "taxonomy": [], + "taste_score": null + }, + { + "slug": "hrm-text-architecture", + "title": "Hierarchical Reasoning Model architecture for text LMs (HRM-Text-1B)", + "modality": "text-lm", + "source_url": "https://huggingface.co/sapientinc/HRM-Text-1B", + "paper_date": "2026-05-17", + "first_seen": "2026-06-12", + "status": "candidate", + "score": 8.0, + "brief_path": null, + "run_ids": [], + "objective": "any", + "taxonomy": [], + "taste_score": null + }, + { + "slug": "opd-sparse-updates", + "title": "Subnetwork-only on-policy distillation (sparse, FFN-heavy OPD updates)", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2606.13657", + "paper_date": "2026-06-11", + "first_seen": "2026-06-12", + "status": "candidate", + "score": 7.0, + "brief_path": null, + "run_ids": [], + "objective": "any", + "taxonomy": [], + "taste_score": null + }, + { + "slug": "influcoder-data-attribution", + "title": "Influcoder: encoder-distilled gradient-influence data filtering", + "modality": "dataset", + "source_url": "https://arxiv.org/abs/2606.13668", + "paper_date": "2026-06-11", + "first_seen": "2026-06-12", + "status": "candidate", + "score": 7.0, + "brief_path": null, + "run_ids": [], + "objective": "any", + "taxonomy": [], + "taste_score": null + }, + { + "slug": "zeta-dual-whitening", + "title": "Zeta: Dual Whitening for Matrix Optimization (Muon variant)", + "modality": "text", + "source_url": "http://arxiv.org/abs/2606.14187", + "paper_date": "2026-06-12", + "first_seen": "2026-06-16", + "status": "briefed", + "objective": "pretrain-ablation", + "taxonomy": [ + "decoder-only", + "optimizer", + "SLM", + "base", + "text" + ], + "score": 7.0, + "taste_score": 7.5, + "brief_path": "research/briefs/zeta-dual-whitening.md", + "run_ids": [] + }, + { + "slug": "pc-layer-preconditioning", + "title": "PC Layer: Polynomial Weight Preconditioning for LLM Pre-Training", + "modality": "text", + "source_url": "http://arxiv.org/abs/2606.06470", + "paper_date": "2026-06-04", + "first_seen": "2026-06-16", + "status": "candidate", + "objective": "pretrain-ablation", + "taxonomy": [ + "decoder-only", + "optimizer", + "SLM" + ], + "score": 7.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "webgraphmix-data-selection", + "title": "WebGraphMix: Pretraining Data Selection via Web Graph Centrality", + "modality": "text", + "source_url": "http://arxiv.org/abs/2606.11499", + "paper_date": "2026-06-09", + "first_seen": "2026-06-16", + "status": "candidate", + "objective": "pretrain-ablation", + "taxonomy": [ + "data-efficiency", + "SLM" + ], + "score": 7.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "normuon-optimizer", + "title": "NorMuon optimizer (Newton-Schulz orthogonalized + per-neuron 2nd-moment) for 2D hidden weights vs AdamW", + "modality": null, + "source_url": "https://arxiv.org/abs/2510.05491", + "paper_date": "2025-10-06", + "first_seen": "2026-06-16", + "status": "briefed", + "objective": "pretrain-ablation", + "taxonomy": [], + "score": null, + "taste_score": null, + "brief_path": null, + "run_ids": [ + "2026-06-16_qwen3_normuon-vs-adamw" + ] + }, + { + "slug": "vibethinker-small-reasoning", + "title": "VibeThinker-3B: Spectrum-to-Signal verifiable-reasoning post-training (curriculum SFT + multi-domain RL + offline self-distillation) for small LMs", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2606.16140", + "paper_date": "2026-06-15", + "first_seen": "2026-06-17", + "status": "briefed", + "objective": "finetune", + "taxonomy": [ + "architecture: decoder-only", + "training-stage: reasoning (RLVR/GRPO)", + "modality: text", + "specialization: code+math", + "size-band: SLM/tiny", + "context/position: RoPE/GQA", + "openness: fully-open" + ], + "score": 10.0, + "taste_score": 6.0, + "brief_path": "research/briefs/vibethinker-small-reasoning.md", + "run_ids": [ + "2026-06-17_qwen3-0.6b_openr1-math-220k", + "2026-06-17_qwen3-0.6b_vibethinker-small-reasoning", + "2026-07-01_qwen3-0.6b_rlvr-phase1-passk", + "2026-07-02_qwen3-0.6b_grpo-prompts-dataprep", + "2026-07-02_qwen3-0.6b_grpo-phase2" + ] + }, + { + "slug": "variable-width-transformers", + "title": "Variable-Width Transformers (width varies across depth as a scaling lever)", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2606.18246", + "paper_date": "2026-06-16", + "first_seen": "2026-06-17", + "status": "candidate", + "objective": "pretrain-ablation", + "taxonomy": [ + "decoder-only", + "tiny-edge", + "base", + "pretrain" + ], + "score": 8.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "opsd-self-distillation", + "title": "OPSD: On-policy self-distillation from the self-future for diffusion LLMs", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2606.18195", + "paper_date": "2026-06-15", + "first_seen": "2026-06-17", + "status": "candidate", + "objective": "finetune", + "taxonomy": [ + "diffusion-lm", + "distillation", + "post-train" + ], + "score": 8.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "ternary-mamba", + "title": "Ternary Mamba: grouped quantization-aware training of W1.58A16 state-space models", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2606.18114", + "paper_date": "2026-06-16", + "first_seen": "2026-06-17", + "status": "candidate", + "objective": "pretrain-ablation", + "taxonomy": [ + "ssm", + "mamba", + "tiny-edge", + "quant-aware-training" + ], + "score": 7.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "hybrid-attention-rethink", + "title": "Rethinking the role of efficient attention in hybrid attention-SSM architectures", + "modality": "text", + "source_url": "https://arxiv.org/abs/2606.15378", + "paper_date": "2026-06-13", + "first_seen": "2026-06-17", + "status": "briefed", + "objective": "pretrain-ablation", + "taxonomy": [ + "architecture:hybrid", + "ssm", + "attention", + "size:tiny-edge", + "stage:base", + "modality:text", + "context:long-context-nope" + ], + "score": 6.0, + "taste_score": 8.0, + "brief_path": "research/briefs/hybrid-attention-rethink.md", + "run_ids": [ + "2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0", + "2026-07-20_hybrid-ssm-0.2b_pretrain-ssm-base-s1", + "2026-07-21_hybrid-ssm-0.2b_arch-ladder" + ], + "linked_proposal": "build-hybrid-ssm-attention-jax" + }, + { + "slug": "fastcontext-long-context-sft", + "title": "FastContext: SFT+RL recipe turning a small base model into a parallel-tool repo-exploration subagent (citations as compact file:line)", + "modality": "text", + "source_url": "https://arxiv.org/abs/2606.14066", + "paper_date": "2026-06-12", + "first_seen": "2026-06-17", + "status": "candidate", + "objective": "finetune", + "taxonomy": [ + "finetune", + "sft", + "rl", + "code", + "decoder-only" + ], + "score": 8.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "arch-subcomponent-drill", + "title": "Phase-2 arch sub-component drill (value-residual / layernorm-scaling / head-gating)", + "modality": "text", + "source_url": null, + "paper_date": null, + "first_seen": "2026-06-21", + "status": "done", + "objective": "pretrain-ablation", + "taxonomy": [ + "architecture", + "decoder-only", + "base", + "text", + "SLM" + ], + "score": null, + "taste_score": 8.0, + "brief_path": "research/decisions/2026-06-21_lifecycle-completion-plan.md", + "run_ids": [ + "2026-06-21_qwen3-0.6b_arch-subdrill-p2" + ], + "note": "Phase-2 follow-on to the attributed arch driver (Phase-1 verdict.json: arch is the sole BPB driver); splits the bundle into its 3 single-variable flags." + }, + { + "slug": "midtrain-anneal-premium-mix", + "title": "Mid-training anneal: premium 50/50 mix cooldown vs iso-token control", + "modality": null, + "source_url": null, + "paper_date": null, + "first_seen": "2026-06-30", + "status": "briefed", + "objective": "pretrain-ablation", + "taxonomy": [ + "training-stage", + "data" + ], + "score": null, + "taste_score": null, + "brief_path": "research/midtraining/plan.md", + "run_ids": [ + "2026-06-30_qwen3-0.6b_midtrain-anneal" + ] + }, + { + "slug": "kronq-quantization", + "title": "KronQ: LLM Quantization via Kronecker-Factored Hessian", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2607.07964", + "paper_date": "2026-07-07", + "first_seen": "2026-07-14", + "status": "candidate", + "objective": "any", + "taxonomy": [ + "quantization", + "efficiency" + ], + "score": 8.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "linear-attention-routing", + "title": "Linear Attention Architectures: Mechanisms, Trade-offs, and Cross-Layer Routing", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2607.07953", + "paper_date": "2026-07-07", + "first_seen": "2026-07-14", + "status": "candidate", + "objective": "pretrain-ablation", + "taxonomy": [ + "architecture", + "linear-attention", + "hybrid" + ], + "score": 7.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "trust-region-policy-distillation", + "title": "Trust Region Policy Distillation", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2607.04751", + "paper_date": "2026-07-05", + "first_seen": "2026-07-14", + "status": "candidate", + "objective": "finetune", + "taxonomy": [ + "distillation", + "rl" + ], + "score": 7.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + }, + { + "slug": "memorization-generalization-ft", + "title": "Towards Mechanistically Understanding Why Memorized Knowledge Fails to Generalize in LLM Finetuning", + "modality": "text-lm", + "source_url": "https://arxiv.org/abs/2607.08393", + "paper_date": "2026-07-09", + "first_seen": "2026-07-14", + "status": "candidate", + "objective": "finetune", + "taxonomy": [ + "finetune", + "interpretability" + ], + "score": 7.0, + "taste_score": null, + "brief_path": null, + "run_ids": [] + } + ], + "runs": [ + { + "run_id": "2026-06-16_qwen3-faithful_eval-first", + "type": "eval", + "model_dir": "Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": null, + "framework": "pytorch", + "eta_hours": null, + "started": "2026-06-16", + "ended": null, + "status": "done", + "verdict": null, + "metrics": { + "suite_version": "text-lm-v2", + "ppl": 37.0101, + "bpb": 1.2256, + "noise_floor_abs": 1.3039, + "wikitext2_val": { + "ppl": 37.0101, + "bpb": 1.2256 + }, + "code_py": { + "ppl": 438.673, + "bpb": 2.1286 + } + }, + "artifacts_dir": null, + "lineage": { + "git_commit": "86e79f3", + "env": null, + "dataset_id": "Salesforce/wikitext + codeparrot/codeparrot-clean-valid", + "dataset_revision": "b08601e0...+4db92d2e...", + "data_hash": null, + "artifact_sha256": "checkpoint_qwen3_baseline2tpp.pt@step18150" + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-16_qwen3-faithful_eval-first.md", + "objective": "pretrain-ablation" + }, + { + "run_id": "2026-06-16_qwen3_normuon-vs-adamw", + "type": "ablation", + "model_dir": "Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b", + "technique_slug": "normuon-optimizer", + "budget": { + "tokens_per_cell": 41943040, + "cells": 6, + "total_tokens": 251658240, + "source": "6-cell cohort (2 arms x 3 seeds), 640 steps x 65536 tok" + }, + "probe": { + "tokens_per_sec": 5944, + "tokens_per_sec_adamw_est": 7000, + "peak_mem_gb": 52.4 + }, + "smoke": null, + "framework": "pytorch", + "eta_hours": 11, + "started": "2026-06-16", + "ended": "2026-06-17", + "status": "done", + "verdict": "win", + "metrics": { + "suite_version": "text-lm-v2", + "headline_metric": "wikitext2_val_bpb", + "verdict": "win", + "wikitext_bpb_adamw_mean": 2.1098, + "wikitext_bpb_normuon_mean": 1.6355, + "wikitext_improvement_bpb": 0.4743, + "wikitext_ci95": [ + 0.4435, + 0.5052 + ], + "wikitext_significant": true, + "code_improvement_bpb": 0.5016, + "code_ci95": [ + 0.456, + 0.5471 + ], + "code_significant": true, + "n_seeds_per_arm": 3, + "systems": { + "mfu": 0.2909, + "mfu_normuon": 0.2907, + "achieved_tflops": 36.36, + "device_peak_tflops": 125.0, + "peak_is_estimated": true, + "n_params_nonembed": 440467456, + "tokens_per_sec_adamw": 6657, + "tokens_per_sec_normuon": 6654 + }, + "caveats": "LR confound RESOLVED via matched-config sweep (NorMuon vs AdamW-at-its-best). Remaining: 42M-tok early-training regime (gap may narrow at scale; no scaling curve); seeds share fixed data split (CI under-estimates full variance); single arch.", + "design": "NorMuon vs AdamW on 2D hidden weights; identical faithful Qwen3-0.6B arch, fixed decontaminated data split (seed 0), 640 steps x 65536 tok = 42M tok/cell, 3 seeds/arm", + "adamw_lr_sweep_wikitext_bpb": { + "1.7e-3": 2.1246, + "2.4e-3": 2.1098, + "3.5e-3": 2.1223, + "4.8e-3": 2.1569 + }, + "adamw_lr_optimum": "2.4e-3 (confirmed best at matched config; spread ~0.047 bpb ~10x < the 0.474 gap)" + }, + "artifacts_dir": null, + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-16_qwen3_normuon-vs-adamw.md", + "objective": "pretrain-ablation", + "confound_check": { + "n_vars": 1, + "iso_flop": true + } + }, + { + "run_id": "2026-06-16_qwen3-0.6b_eval-faithful", + "type": "eval", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": null, + "framework": null, + "eta_hours": null, + "started": "2026-06-16", + "ended": null, + "status": "done", + "verdict": null, + "metrics": {}, + "artifacts_dir": null, + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-faithful.md", + "objective": "pretrain-ablation", + "suite_version": "text-lm-v2", + "self_floor": true, + "wikitext2_ppl": 37.01, + "wikitext2_floor_abs": 1.3, + "code_ppl": 438.67, + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 280.56 + }, + { + "run_id": "2026-06-16_qwen3-0.6b_eval-modernized", + "type": "eval", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": null, + "framework": null, + "eta_hours": null, + "started": "2026-06-16", + "ended": null, + "status": "done", + "verdict": null, + "metrics": {}, + "artifacts_dir": null, + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-modernized.md", + "objective": "pretrain-ablation", + "suite_version": "text-lm-v2", + "self_floor": true, + "wikitext2_ppl": 27.8, + "wikitext2_floor_abs": 1.07, + "code_ppl": 129.42, + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 111.92 + }, + { + "run_id": "2026-06-16_qwen3-0.6b_eval-prope25", + "type": "eval", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": null, + "framework": null, + "eta_hours": null, + "started": "2026-06-16", + "ended": null, + "status": "done", + "verdict": null, + "metrics": {}, + "artifacts_dir": null, + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-prope25.md", + "objective": "pretrain-ablation", + "suite_version": "text-lm-v2", + "self_floor": true, + "wikitext2_ppl": 38.08, + "wikitext2_floor_abs": 1.7, + "code_ppl": 447.3, + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 298.9 + }, + { + "run_id": "2026-06-16_qwen3-0.6b_eval-prope10", + "type": "eval", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": null, + "framework": null, + "eta_hours": null, + "started": "2026-06-16", + "ended": null, + "status": "done", + "verdict": null, + "metrics": {}, + "artifacts_dir": null, + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-prope10.md", + "objective": "pretrain-ablation", + "suite_version": "text-lm-v2", + "self_floor": true, + "wikitext2_ppl": 69.63, + "wikitext2_floor_abs": 3.67, + "code_ppl": 1356.31, + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 690.96, + "note": "step-4000 checkpoint; run stopped early at ~step 5400 (undertrained vs 18150-step peers)" + }, + { + "run_id": "2026-06-17_qwen3-0.6b_openr1-math-220k", + "type": "dataset-prep", + "model_dir": "Qwen3-0.6B", + "technique_slug": "vibethinker-small-reasoning", + "budget": { + "tokens": 125000000, + "source": "research/briefs/vibethinker-small-reasoning.md" + }, + "probe": null, + "smoke": null, + "framework": null, + "eta_hours": null, + "started": "2026-06-17", + "ended": "2026-06-17", + "status": "done", + "verdict": null, + "metrics": { + "train_tokens": 125000592, + "eval_tokens": 232918, + "eval_docs_kept": 66, + "forgetting_tokens": 167242, + "forgetting_docs": 202, + "docs_dropped": 0, + "fertility": 1.92, + "n_train_samples": 35512, + "shards": 3 + }, + "artifacts_dir": "research/datasets/math-reasoning-openr1-math-220k/", + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": "open-r1/OpenR1-Math-220k", + "dataset_revision": "e4e141ec9dea9f8326f4d347be56105859b2bd68", + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-17_qwen3-0.6b_openr1-math-220k.md", + "objective": "finetune" + }, + { + "run_id": "2026-06-17_qwen3-0.6b_vibethinker-small-reasoning", + "type": "finetune", + "model_dir": "Qwen3-0.6B", + "technique_slug": "vibethinker-small-reasoning", + "budget": { + "tokens": 125000000, + "source": "brief vibethinker-small-reasoning \u00a7C5.2/Addendum" + }, + "probe": { + "tokens_per_sec": 7465.3, + "peak_mem_gb": 52.39 + }, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": 4.65, + "started": "2026-06-17", + "ended": "2026-06-18", + "status": "done", + "verdict": "inconclusive", + "metrics": { + "suite_version": "text-lm-v2", + "objective": "finetune", + "forgetting_verdict": "retained", + "forgetting_wikitext_ppl": { + "base": 37.01, + "sft": 37.08, + "delta_pct": 0.2, + "significant": false + }, + "forgetting_code_ppl": { + "base": 438.67, + "sft": 425.68, + "delta_pct": -3.0, + "significant": false + }, + "forgetting_wikitext_bpb": { + "base": 1.2256, + "sft": 1.2263 + }, + "reasoning_openr1_ppl": { + "base": 14.26, + "sft": 11.6, + "delta_pct": -18.7, + "note": "in-loop held-out OpenR1-Math reasoning PPL" + }, + "outcome": "reasoning improved -18.7% PPL with NO significant general-ability regression (retained on wikitext + code); n=1, PPL-based (not benchmark accuracy, not seed-gated)", + "forgetting_fineweb_edu": { + "corpus_id": "fineweb_edu_sample10bt_heldout", + "base_ppl": 24.5514, + "target_ppl": 24.7331, + "delta_abs": 0.1817, + "delta_pct": 0.74, + "floor_abs": 8.0155, + "significant": false, + "label": "retained", + "base_ppl_claim_readme": 28.65, + "note": "\u00a7C13 forgetting probe vs the corpus the base was pretrained on; sub-floor \u2192 retained; SFT PPL below both measured-base 24.55 and README-claim 28.65" + }, + "reasoning_openr1_heldout": { + "corpus_id": "openr1_math_220k_heldout", + "base_ppl": 17.6035, + "target_ppl": 14.2217, + "delta_abs": -3.3818, + "delta_pct": -19.21, + "floor_abs": 4.4291, + "significant": false, + "label": "not significant", + "note": "eval-harness held-out reasoning PPL via text-lm-v2 machinery; -19.2% directional gain but below the single-checkpoint corpus-partition floor 4.43 and n=1 (no seed cohort) \u2192 not a \u00a7C17 win" + }, + "verdict_basis": "inconclusive: n=1 SFT (no seed cohort) cannot clear \u00a7C17 seeds>=3 paired-CI bar; reasoning -19.2% is a promising single-run signal not a significant win; FineWeb-Edu forgetting probe PASSES (retained, sub-floor). Forgetting is not a loss.", + "eval_suite_version": "text-lm-v2", + "eval_brief_probes_path": "Qwen3-0.6B/experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/eval/brief_probes_results.json" + }, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/results", + "lineage": { + "git_commit": "41824a5", + "env": null, + "dataset_id": "open-r1/OpenR1-Math-220k", + "dataset_revision": "e4e141ec9dea9f8326f4d347be56105859b2bd68", + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md", + "objective": "finetune" + }, + { + "run_id": "2026-06-18_qwen3-0.6b_imu1-deconfound-p1", + "type": "ablation", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": { + "steps_per_cell": 2000, + "tokens_per_cell": 131072000, + "cells": 12, + "total_tokens": 1572864000, + "source": "2000-step proxy Phase 1" + }, + "probe": { + "tokens_per_sec": 7444, + "peak_mem_gb": 52.4 + }, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": 58.7, + "started": "2026-06-18", + "ended": "2026-06-21", + "status": "done", + "verdict": "directional", + "metrics": {}, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1", + "lineage": { + "git_commit": "bdc5ec6", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-18_qwen3-0.6b_imu1-deconfound-p1.md", + "objective": "pretrain-ablation", + "arm_plan": { + "arms": [ + "baseline", + "wsd", + "zloss", + "arch" + ], + "seeds": [ + 0, + 1, + 2 + ], + "optimizer": "adamw" + }, + "confound_check": { + "n_vars": 1, + "iso_flop": true + }, + "resume_roundtrip": "pass", + "guards": "verified", + "evidence_path": "Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/c5_evidence.json", + "train_pid": 3720974, + "sentinel_pid": 2276710, + "current_arm": "zloss_seed0", + "resume_cmd": "bash Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/run_arms.sh", + "log_paths": [ + "run_arms.log", + "run_baseline_seed0.log" + ], + "resume_note": "mb4+compile restored after 4 sentinel kills; raised per-cohort sentinel to kill-at 0.83 (safe_cuda 0.85 backstop) since snap-confined Firefox is unfreeable; mb4 spike 80.7% now clears the guard; zero confound (identical to baseline/wsd)", + "note": "closed retroactively 2026-07-01: cohort.done 2026-06-21; verdict.json drivers=[arch] overall=attributed (wsd/zloss n.s.); ledger entry had been left status=running" + }, + { + "run_id": "2026-06-21_qwen3-0.6b_arch-subdrill-p2", + "type": "ablation", + "model_dir": "Qwen3-0.6B", + "technique_slug": "arch-subcomponent-drill", + "budget": { + "steps_per_cell": 2000, + "tokens_per_cell": 131072000, + "new_cells": 9, + "source": "parity with Phase-1 de-confound proxy (2026-06-18)" + }, + "probe": { + "tokens_per_sec": 5940, + "peak_mem_gb": 61.1, + "source": "Phase-1 arch arm measured (identical trainer/config; >= params)" + }, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": 52, + "started": "2026-06-21", + "ended": null, + "status": "done", + "verdict": "directional", + "metrics": {}, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-06-21_qwen3-0.6b_arch-subdrill-p2", + "lineage": { + "git_commit": "59519ca", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-21_qwen3-0.6b_arch-subdrill-p2.md", + "objective": "pretrain-ablation", + "arm_plan": { + "arms": [ + "vr", + "ln", + "hg" + ], + "reused_control": "baseline", + "seeds": [ + 0, + 1, + 2 + ], + "optimizer": "adamw 1.7e-3", + "schedule": "cosine" + }, + "confound_check": { + "n_vars": 1, + "iso_flop": true + }, + "control_reuse": { + "arm": "baseline", + "reused_run_id": "2026-06-18_qwen3-0.6b_imu1-deconfound-p1", + "mechanism": "symlink, not retrained" + }, + "guards": "verified", + "resume_roundtrip": "pass", + "prior_run_id": "2026-06-18_qwen3-0.6b_imu1-deconfound-p1", + "evidence_path": "Qwen3-0.6B/experiments/2026-06-21_qwen3-0.6b_arch-subdrill-p2/c5_evidence.json", + "train_pid": 1629305, + "current_arm": "vr_seed0", + "log_paths": [ + "run_arms.log", + "run_vr_seed0.log", + "sentinel.log", + "post_cohort.log" + ], + "resume_cmd": "bash Qwen3-0.6B/experiments/2026-06-21_qwen3-0.6b_arch-subdrill-p2/run_arms.sh", + "watcher_pid": 1629306, + "lifecycle_stage": "architecture", + "eval_items_present": [ + "iso_flop", + "seed_ci_bpb", + "single_axis_isolation", + "suite_version", + "downstream_battery", + "per_task_bpb_gold" + ], + "incomplete_eval": [ + "ladder_3rung_trend (rented multi-scale)", + "cross_data_mix (rented)", + "kv_ttft_itl_pareto (hg touches attention)" + ], + "downstream_confirmed": "v3 LAMBADA+BPB-gold confirm vr247). Not more-code (dclm has ~none); FineWeb-Edu prose-filter is unusually bad at code structure.", + "eval_items_present": [ + "single_axis_isolation", + "seed_ci_bpb", + "decontam_report", + "downstream_or_bpb", + "figure", + "provenance_sha_license", + "doc_disjoint_split" + ], + "incomplete_eval": [ + "second_lr_recheck (single budget)", + "tokenizer_fertility (not measured)" + ] + }, + { + "run_id": "2026-06-26_qwen3-0.6b_data-mix-composition", + "type": "ablation", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": null, + "started": "2026-06-26", + "ended": null, + "status": "done", + "verdict": "directional", + "metrics": {}, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-06-26_qwen3-0.6b_data-mix-composition", + "lineage": { + "git_commit": "61c9b51", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-26_qwen3-0.6b_data-mix-composition.md", + "objective": "pretrain-ablation", + "lifecycle_stage": "data", + "confound_check": { + "n_vars": 1, + "iso_flop": true + }, + "arm_plan": { + "arms": [ + "fineweb(reused)", + "dclm(reused)", + "mix(new 50/50)" + ], + "seeds": [ + 0, + 1, + 2 + ], + "new_cells": 3 + }, + "purpose": "data-composition curve: does a 50/50 FineWeb-Edu+dclm-edu mix keep English AND gain code (no tradeoff)?", + "train_pid": 2523752, + "note": "mix re-run after fixing token-level-shuffle bug (was destroying sequence structure)", + "headline": "50/50 mix = BEST OF BOTH: English on par with FineWeb (\u0394+0.016 n.s.), code recovers +0.59 of dclm s +0.70 win (mix 2.05 between FineWeb 2.64 and dclm 1.94). No English tradeoff; ~84% of the code gain." + }, + { + "run_id": "2026-06-27_qwen3-0.6b_sft-3seed", + "type": "finetune", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": 30, + "started": "2026-06-27", + "ended": null, + "status": "done", + "verdict": "directional", + "metrics": { + "suite_version": "text-lm-v2", + "eval_harness_note": "1 representative seed/arm scored on the standard suite (held-out \u03c3\u22480.001 across seeds; full 3-seed CI is the in-domain reasoning_verdict.json)", + "wikitext2_ppl": { + "base": 37.01, + "sft_seed0": 37.082, + "ctrl_seed0": 37.084, + "floor_abs": 1.304, + "label": "retained (\u0394+0.07 sub-floor)" + }, + "code_ppl": { + "base": 438.673, + "sft_seed0": 425.689, + "ctrl_seed0": 425.594, + "floor_abs": 280.564, + "label": "retained (sub-floor)" + }, + "forgetting_wikitext2": "retained", + "forgetting_code": "retained", + "forgetting_fineweb_edu_reasoning_eval": { + "base": 21.495, + "sft_mean": 21.652, + "ctrl_mean": 21.66, + "label": "retained (~0.7%)" + }, + "heldout_reasoning_ppl_masked": { + "base": 14.127, + "sft_mean": 11.573, + "ctrl_mean": 11.582 + }, + "sft_vs_control_masked": { + "improvement_ppl": 0.0092, + "ci95": [ + 0.0045, + 0.0139 + ], + "significant": true + }, + "sft_vs_control_fullseq": { + "improvement_ppl": -0.0064, + "ci95": [ + -0.011, + -0.0015 + ], + "significant": false + }, + "sft_vs_base_masked": { + "improvement_ppl": 2.554, + "pct": 18.1 + }, + "headline": "SFT (response-masked) does NOT separate from the iso-FLOP --no_mask control on held-out reasoning (masked +0.009 sig / full-seq -0.006 nonsig -> directional). The ~18% reasoning-PPL gain over base is from the extra math tokens, not the masking. No catastrophic forgetting (wikitext/code/FineWeb-Edu all retained)." + }, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed", + "lineage": { + "git_commit": "b95c019", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-06-27_qwen3-0.6b_sft-3seed.md", + "objective": "finetune", + "lifecycle_stage": "sft", + "arm_plan": { + "arms": [ + "sft-masked", + "ctrl-no_mask" + ], + "seeds": [ + 0, + 1, + 2 + ], + "new_cells": 6 + }, + "purpose": "n=1 SFT -> 3-seed CI verdict + iso-FLOP control", + "train_pid": 3857373, + "current_cell": "sft_seed0", + "note": "stepping confirmed (step10 loss2.59 7294 tok/s ETA4.55h/cell); fixed scorer armed" + }, + { + "run_id": "2026-06-30_qwen3-0.6b_midtrain-anneal", + "type": "ablation", + "model_dir": "Qwen3-0.6B", + "technique_slug": "midtrain-anneal-premium-mix", + "budget": null, + "probe": { + "tokens_per_sec": 6885, + "peak_mem_gb": 57.4, + "pct_pool": 48, + "source": "mb4+compile data-mix ref; mb6 measured slower+89.7GB=75% (unified-mem thrash), mb2 5613" + }, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": 36, + "started": "2026-06-30", + "ended": "2026-07-01", + "status": "done", + "verdict": "win", + "metrics": { + "suite_version": "text-lm-v2", + "headline": "code_py_bpb_treatment_vs_iso_token_control", + "code_py_bpb": { + "control_mean": 2.1236, + "treatment_mean": 1.852, + "improvement": 0.2716, + "ci95": [ + 0.2641, + 0.2792 + ], + "significant": true, + "n_seeds": 3 + }, + "wikitext2_bpb": { + "control_mean": 1.222, + "treatment_mean": 1.2062, + "improvement": 0.0157, + "ci95": [ + 0.014, + 0.0174 + ], + "significant": true, + "n_seeds": 3 + }, + "short_ctx_non_regression": "pass \u2014 all 4 arm/corpus cells improved vs un-annealed base", + "c25_cap": "directional (effective_context_length_ruler missing \u2014 loader build-backlog)" + }, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-06-30_qwen3-0.6b_midtrain-anneal", + "lineage": { + "git_commit": "ac8e332", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": 2220, + "gpu_hours": 36.7 + }, + "detail_md": "research/ledger/runs/2026-06-30_qwen3-0.6b_midtrain-anneal.md", + "objective": "pretrain-ablation", + "lifecycle_stage": "mid-training", + "arm_plan": { + "arms": [ + "mix-treatment", + "fineweb-control" + ], + "seeds": [ + 0, + 1, + 2 + ], + "new_cells": 6 + }, + "note": "anneal cooldown from faithful base; headline code_py BPB treatment-vs-control 3-seed CI; honest null expected English/downstream", + "confound_check": { + "n_vars": 1, + "iso_flop": true, + "evidence": "single variable = DATA (mix vs fineweb), identical base ckpt + cooldown schedule + 2300 steps x 65,536 tok = 150.7M consumed tokens per cell (iso-FLOP exact by construction); \u00a7C5 evidence in c5_evidence.json" + } + }, + { + "run_id": "2026-07-01_qwen3-0.6b_rlvr-phase1-passk", + "type": "eval", + "model_dir": "Qwen3-0.6B", + "technique_slug": "vibethinker-small-reasoning", + "budget": null, + "probe": null, + "smoke": null, + "framework": "pytorch", + "eta_hours": null, + "started": "2026-07-01", + "ended": "2026-07-02", + "status": "done", + "verdict": null, + "metrics": { + "extractor": "math-acc-v1", + "band": "100 gsm8k_test_clean + 50 math500_clean L<=3, seed 20260701, n=8 @ T=0.8, max_new=256", + "sft_seed0": { + "gsm8k": { + "pass1": { + "acc": 0.01125, + "ci_low": 0.0059297687461997645, + "ci_high": 0.021241582744848563 + }, + "pass8": 0.07, + "solved": 7 + }, + "math500_l13": { + "pass1": { + "acc": 0.015, + "ci_low": 0.006892291919440893, + "ci_high": 0.03233463358482672 + }, + "pass8": 0.1, + "solved": 5 + } + }, + "base": { + "gsm8k": { + "pass1": { + "acc": 0.00625, + "ci_low": 0.0026724940266688486, + "ci_high": 0.014546646226436234 + }, + "solved": 5 + }, + "math500_l13": { + "pass1": { + "acc": 0.0225, + "ci_low": 0.011881583530495034, + "ci_high": 0.04220265755875813 + }, + "solved": 7 + } + }, + "decision": "GO", + "go_rule": "GO iff SFT solved_items >= 3 across the band (pre-registered)", + "honest_read": "GO fires on absolute solvability (12 items) \u2014 NOT on SFT>base: CIs overlap everywhere and base BEATS sft on math500_l13 (14% vs 10% pass@8). GRPO ceiling = sharpening pass@1 toward the ~7-10% pass@8 band; random-reward control mandatory." + }, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-07-01_qwen3-0.6b_rlvr-phase1-passk", + "lineage": { + "git_commit": "ac8e332", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-01_qwen3-0.6b_rlvr-phase1-passk.md", + "objective": "finetune", + "lifecycle_stage": "rlvr", + "note": "rlvr Phase-1 pass@k go/no-go (plan.md Phase 1, pre-registered rule in run_phase1_passk.py). DECISION: GO -> P1 prompt-set prep unlocked; Phase-2 GRPO training remains behind the needs-approval rlvr-method-plan proposal." + }, + { + "run_id": "2026-07-02_qwen3-0.6b_grpo-prompts-dataprep", + "type": "dataset-prep", + "model_dir": "Qwen3-0.6B", + "technique_slug": "vibethinker-small-reasoning", + "budget": null, + "probe": null, + "smoke": null, + "framework": null, + "eta_hours": null, + "started": "2026-07-02", + "ended": "2026-07-02", + "status": "done", + "verdict": null, + "metrics": { + "gsm8k_train_clean": 7471, + "math_l13_train_clean": 3490, + "eval_overlap_dropped": 14, + "sft_overlap_flagged": 7, + "extractor": "math-acc-v1" + }, + "artifacts_dir": "research/datasets/grpo-math-prompts-v1", + "lineage": { + "git_commit": "ac8e332", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-02_qwen3-0.6b_grpo-prompts-dataprep.md", + "objective": "finetune", + "lifecycle_stage": "rlvr", + "note": "P1 of rlvr plan (unlocked by Phase-1 GO): GRPO training prompts, eval-decontaminated (drops) + SFT-flagged (kept). Phase-2 GRPO training still needs-approval." + }, + { + "run_id": "2026-07-02_qwen3-0.6b_grpo-phase2", + "type": "finetune", + "model_dir": "Qwen3-0.6B", + "technique_slug": "vibethinker-small-reasoning", + "budget": null, + "probe": null, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": 88, + "started": "2026-07-02", + "ended": null, + "status": "done", + "verdict": "directional", + "metrics": { + "proceed_to_phase3": false, + "conclusion": "PREDICTED NULL CONFIRMED: GRPO does not beat the SFT floor; GRPO does not beat the random-reward gate. At ~1% base pass@1 there is nothing for RL to sharpen (GRPO training reward flat at ~0.9% correct with no trend [first-50 0.0091 vs last-50 0.0080]; RFT's 352 collected completions, ~0.9% of 38,400 rollouts, were too few to separate from the floor). Do NOT spend the multi-seed cohort; the reasoning gain lives in SFT/distillation, not RL at this scale.", + "comparison": [ + { + "set": "gsm8k", + "sft_pass1": 0.0112, + "grpo_pass1": 0.01, + "random_pass1": 0.01, + "rft_pass1": 0.015, + "sft_pass8": 0.07, + "grpo_pass8": 0.07, + "random_pass8": 0.06, + "rft_pass8": 0.11, + "grpo_pass1_ci": [ + 0.0051, + 0.0196 + ], + "sft_pass1_ci": [ + 0.0059, + 0.0212 + ], + "grpo_beats_sft_floor": false, + "grpo_beats_random_gate": false + }, + { + "set": "math500_l13", + "sft_pass1": 0.015, + "grpo_pass1": 0.0225, + "random_pass1": 0.015, + "rft_pass1": 0.0175, + "sft_pass8": 0.1, + "grpo_pass8": 0.16, + "random_pass8": 0.1, + "rft_pass8": 0.12, + "grpo_pass1_ci": [ + 0.0119, + 0.0422 + ], + "sft_pass1_ci": [ + 0.0069, + 0.0323 + ], + "grpo_beats_sft_floor": false, + "grpo_beats_random_gate": false + } + ] + }, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-07-02_qwen3-0.6b_grpo-phase2", + "lineage": { + "git_commit": "ac8e332", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-02_qwen3-0.6b_grpo-phase2.md", + "objective": "finetune", + "lifecycle_stage": "rlvr", + "note": "measured pace 362s/step (no-KV-cache decode dominates) -> revised ETA ~88h total (grpo ~30h + random ~30h + rft ~26h + evals); launched 2026-07-02 07:39" + }, + { + "run_id": "2026-07-05_qwen3-0.6b_scaling-persistence", + "type": "scaling-fit", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": null, + "started": "2026-07-05", + "ended": "2026-07-12", + "status": "done", + "verdict": "directional", + "metrics": { + "trend_verdict_wikitext": "CONVERGES", + "trend_code_py": "CONVERGES", + "budgets": [ + "42M", + "168M", + "420M" + ], + "top_budget_seeds": [ + 2, + 2 + ], + "headline_capped": true, + "corpora_agree": true, + "conclusion": "DIRECTIONAL (CONVERGES shape on wikitext-2): the top budget (420M) rung is n=2 seeds (<3, \u00a7C17), so this is a directional trend, NOT a headline win \u2014 add a 3rd 420M seed (and the 840M rung) to earn more. Shape: The NorMuon advantage CONVERGES toward 0 with budget \u2014 an early-training speedup, as the IMU-1 RESULT.md Limitation #3 predicted it might." + }, + "artifacts_dir": "Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence", + "lineage": { + "git_commit": "cc07070", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-05_qwen3-0.6b_scaling-persistence.md", + "objective": "pretrain-ablation", + "lifecycle_stage": "scaling", + "suite_version": "text-lm-v2" + }, + { + "run_id": "2026-07-19_qwen3-0.6b_interp-cka-repconvergence", + "type": "eval", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": "pass", + "framework": "pytorch", + "eta_hours": null, + "started": "2026-07-19", + "ended": null, + "status": "done", + "verdict": null, + "metrics": {}, + "artifacts_dir": "research/interp", + "lineage": { + "git_commit": "cc07070", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-19_qwen3-0.6b_interp-cka-repconvergence.md", + "lifecycle_stage": "interpretability", + "metric": "linear_cka", + "headline": "Pre-registered CKA null: NorMuon does NOT reach AdamW@420M representation earlier by more than the across-seed band; directional-but-sub-noise at 42M, reverses by 168M. A caught fp16-overflow confound faked an earlier null.", + "prereg": "research/interp/prereg_2026-07-19_repconvergence.md", + "result_doc": "research/interp/cka_result_2026-07-19.md", + "verdict_json": "research/interp/cka_verdict.json" + }, + { + "run_id": "2026-07-19_qwen3-0.6b_cce-fused-ce", + "type": "serving-bench", + "model_dir": "Qwen3-0.6B", + "technique_slug": null, + "budget": null, + "probe": null, + "smoke": "pass", + "framework": "triton", + "eta_hours": null, + "started": "2026-07-19", + "ended": null, + "status": "done", + "verdict": "directional", + "metrics": { + "correctness": "pass", + "d_loss": 9.5e-07, + "mem_gb_naive_32768": 61.4, + "mem_gb_cce_32768": 2.04, + "mem_gb_cce_65536": 2.17, + "naive_oom_at": 65536, + "tput_ms_cce_32768": 298.27, + "tput_ms_naive_32768": 2014.53 + }, + "artifacts_dir": "research/kernel", + "lineage": { + "git_commit": "cc07070", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-19_qwen3-0.6b_cce-fused-ce.md", + "lifecycle_stage": "systems", + "headline": "CCE fused linear CE: correctness PASS (torch+Triton vs naive fp32 oracle, re-verified GB10 2026-07-19, |Dloss|<1e-6); MEMORY win validated on GB10 (naive 61.4GB->CCE 2.04GB at 32768 tok, naive OOMs at 65536); throughput wall-clock CCE fastest (298ms vs 1038 torch.compile). Roofline off-box (propose-only). Convergence@pretrain UNVALIDATED (needs iso-FLOP A/B).", + "spec": "research/kernel/SPEC_cce_fused_linear_ce.md", + "result_doc": "research/kernel/RESULT_cce_2026-07-19.md", + "roofline": "off-box-propose-only" + }, + { + "run_id": "2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0", + "type": "ablation", + "model_dir": "HybridSSM-0.2B", + "technique_slug": "hybrid-attention-rethink", + "budget": { + "tokens": 170034304, + "seq": 2048, + "batch": 8, + "steps": 10378 + }, + "probe": { + "tokens_per_sec": 2837, + "steps_per_hour": 1247, + "peak_mem_gb": 16.6, + "peak_frac_of_pool": 0.139, + "fits": true, + "note": "DERIVED from the production run (7080 steps x 8192 tok in 5.68h), not a pre-launch probe: the probe logs record no timing. peak_mem from live nvidia-smi 16635 MiB, concurs with memory_fix." + }, + "smoke": "pass", + "framework": "jax", + "eta_hours": 16.6, + "started": "2026-07-19", + "ended": "2026-07-20", + "status": "done", + "verdict": "directional", + "metrics": { + "ppl_wikitext2_val": 133.4628, + "ppl_code_py": 5142.6426, + "floor_wikitext2_abs": 18.8143, + "floor_code_py_abs": 1367.9534, + "suite_version": "text-lm-v2", + "self_floor": true, + "corpus_wikitext": "Salesforce/wikitext:wikitext-2-raw-v1:validation@b08601e", + "corpus_code": "codeparrot/codeparrot-clean-valid:train@4db92d2 (streaming first-N)" + }, + "artifacts_dir": "HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build", + "lineage": { + "git_commit": "cc07070", + "env": "jax/flax on GB10; jax_safe_env guard active", + "dataset_id": "HuggingFaceFW/fineweb-edu", + "dataset_revision": "sample-10BT (config, not a pinned sha - the upstream revision was not recorded when the tokcache was built on 2026-07-05)", + "data_hash": "sha256:c83b7d608a0ca320ae7b7e41dbee05282f074a004a87a9a90f2f4fd0f5032491", + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": 990, + "gpu_hours": 16.5, + "note": "final process only (22:34:29Z 07-19 -> 15:04:24Z 07-20). EXCLUDES the killed first attempt, whose start time is not on disk, so total GPU time is a LOWER BOUND." + }, + "detail_md": "research/ledger/runs/2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md", + "objective": "pretrain-ablation", + "lifecycle_stage": "architecture", + "arm": "ssm_base_s0", + "probe_fits": true, + "brief": "research/briefs/hybrid-attention-rethink.md", + "sentinel_pid": 3167084, + "train_pid": 3164922, + "launch_config": { + "batch": 4, + "seq": 2048, + "steps": 20756, + "tok_per_step": 8192, + "note": "batch reduced 8->4 after OOM; CE-under-autodiff memory follow-up recorded" + }, + "step0_loss": 12.4317, + "kill_reason": "sentinel memory kill at step 580 (pool 81.3% >= 0.80); reached loss 12.4->6.6 on real FineWeb-Edu; ckpt@step400 resumable but needs a memory fix before resume (SSM-scan + chunked-CE hold ~61GB)", + "resume_needs": "remat/reduce-batch/reduce-SSM-state, then clear marker", + "memory_fix": "nn.remat blocks: 61.5GB->16.6GB alloc, pool 81%->~42%; resumed from step-400 ckpt", + "evidence_path": "HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json", + "resume_roundtrip": "pass (proven in production: resumed from step-400 ckpt after the step-580 sentinel kill, 7080 clean steps since)", + "guards": "verified 2026-07-20: jax_safe_env imported before jax (train_hybrid.py:10 vs :15); chunked CE streaming max+sumexp (model.py:136), never materializes (N,151936)", + "c5_evidence_provenance": "RECONSTRUCTED 2026-07-20 AFTER launch - the 2026-07-19 launch created this ledger entry but never wrote c5_evidence.json, so the SS-C5 \"evidence recorded BEFORE launch\" requirement was NOT met for this run. Per-item src tags (log/derived/attestation/not-captured) are in the evidence file.", + "open_gate_gap": "verify.py last ran 2026-07-19 12:52, model.py modified 22:27 to add nn.remat -> the verify gate has NOT been re-run against the training model. Re-run + capture verify.log before scoring (GPU work; waits for the arm to finish per SS-C4.5).", + "completed": "2026-07-20", + "final_train_loss": 3.8149, + "final_val_loss": 3.9245, + "best_val_loss": 3.7839, + "steps": 21156, + "incomplete_eval": [ + "iso-flop sibling arms (none yet)", + "seed CI (n=1)", + "scaling-ladder fit", + "long-context retrieval probe" + ], + "eval_note": "text-lm-v2 JAX port (eval_suite_jax.py), self-floor mode; code_py is far-OOD for a FineWeb-Edu-only 170M-token model; generations show no capability signal yet (expected at this budget). Baseline datum only \u2014 no cross-arm claim until s1/s2 + comparand arms.", + "budget_overshoot": "train_hybrid.py:129 uses range(start_step, start_step+steps), so a RESUMED run repeats the full step budget from the resume point: resumed at 400 -> ran 21156 steps = 173,309,952 tok vs the declared 170,034,304 (+1.93%, ~1.9% into a 2nd epoch). Harmless at n=1, but it silently breaks SS-C18 iso-FLOP across arms: any arm that crashes and resumes gets MORE compute than one that does not, scaling with the resume point (a resume at step 5000 would be +24%). FIX before the ladder: range(start_step, steps)." + }, + { + "run_id": "2026-07-20_hybrid-ssm-0.2b_pretrain-ssm-base-s1", + "type": "ablation", + "model_dir": "HybridSSM-0.2B", + "technique_slug": "hybrid-attention-rethink", + "budget": { + "tokens": 170034304, + "seq": 2048, + "batch": 4, + "steps": 20756, + "lr": 0.003, + "warmup": 200, + "source": "brief hybrid-attention-rethink + parity with s0 (paired-seed design)" + }, + "probe": { + "tokens_per_sec": 2837, + "steps_per_hour": 1247, + "peak_mem_gb": 16.6, + "peak_frac_of_pool": 0.139, + "fits": true, + "source": "s0 production run, identical config" + }, + "smoke": "pass", + "framework": "jax", + "eta_hours": 16.6, + "started": "2026-07-21", + "ended": null, + "status": "launched", + "verdict": null, + "metrics": {}, + "artifacts_dir": null, + "lineage": { + "git_commit": "266487a", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-20_hybrid-ssm-0.2b_pretrain-ssm-base-s1.md", + "objective": "pretrain-ablation", + "lifecycle_stage": "architecture", + "arm": "ssm_base_s1", + "brief": "research/briefs/hybrid-attention-rethink.md", + "paired_with": "2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0", + "smoke_evidence": "train_hybrid.py --smoke PASS 2026-07-19 (identical script+config; only --seed changes); fresh smoke re-run at launch", + "design": "seed ladder base cell (mixer=ssm, attn_every=2, nope off): identical to s0, only --seed 0->1. C17 paired-seed; same tokcache.", + "resume_roundtrip": "pass (proven on s0: step-400 ckpt resume after sentinel kill)", + "guards": "jax_safe_env before jax (train_hybrid.py:10); chunked CE (model.py:136); verified 2026-07-20", + "planned_launch": "after s0 eval-suite completes (C4.5); fresh smoke + sentinel watch at launch; status set to launched then" + }, + { + "run_id": "2026-07-21_hybrid-ssm-0.2b_arch-ladder", + "type": "ablation", + "model_dir": "HybridSSM-0.2B", + "technique_slug": "hybrid-attention-rethink", + "budget": { + "tokens_total": 1191000000, + "cells": 15, + "rungs": [ + 42000000, + 85000000, + 150000000 + ], + "seq": 2048, + "batch": 4 + }, + "probe": { + "tokens_per_sec": 2863, + "peak_mem_gb": 16.6, + "fits": true, + "note": "from the completed pilot arm (20756 steps / 16.50h), not a short probe" + }, + "smoke": "pass", + "framework": "jax", + "eta_hours": 141.7, + "started": "2026-07-21", + "ended": null, + "status": "launched", + "verdict": null, + "metrics": {}, + "artifacts_dir": "HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder", + "lineage": { + "git_commit": "c43d022", + "env": null, + "dataset_id": null, + "dataset_revision": null, + "data_hash": null, + "artifact_sha256": null + }, + "cost": { + "wall_clock_min": null, + "gpu_hours": null + }, + "detail_md": "research/ledger/runs/2026-07-21_hybrid-ssm-0.2b_arch-ladder.md", + "objective": "pretrain-ablation", + "lifecycle_stage": "architecture", + "confound_check": { + "n_vars": 1, + "iso_flop": true + }, + "arm_plan": { + "arms": [ + "ssm_base", + "swa128", + "swa128_nope", + "attn1to3", + "fullattn" + ], + "seeds": [ + 0 + ], + "new_cells": 15, + "phase": "1 (scout n=1; seeds 1-2 appended only for arms that separate)" + }, + "iso_flop_detail": { + "method": "per-arm token budget matched on total train FLOPs (6N + 12*L_full*H*Dh*T + 12*L_swa*H*Dh*w)", + "worst_mismatch_pct": 0.17, + "tolerance_pct": 5.0, + "why": "arms differ 10-20% in non-embed params, so iso-token would violate C18" + }, + "resume_roundtrip": "pass", + "guards": "verified", + "evidence_path": "HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json", + "driver": "HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh (continuous: .done markers, cooldown gate, per-cell sentinel, loop-until-done, hot-spell backoff, completion gate)", + "ladder_progress": { + "cells_done": 5, + "of": 15, + "rung_42M_done": 5, + "of_5": 5, + "running": "ssm_base_85M_s0" + }, + "cell_ssm_base_42M_s0": { + "final_train_loss": 4.8099, + "steps": 5126, + "tokens": 42000000, + "bpb": "deferred \u2014 GPU busy (\u00a7C4.5); scored at rung/ladder completion" + }, + "cell_swa128_42M_s0": { + "final_train_loss": 6.0373, + "steps": 5859, + "tokens": 48000000, + "bpb": "deferred (\u00a7C4.5, GPU busy)" + }, + "cell_swa128_nope_42M_s0": { + "final_train_loss": 6.3001, + "steps": 5859, + "tokens": 48000000, + "bpb": "deferred (\u00a7C4.5, GPU busy)" + }, + "cell_attn1to3_42M_s0": { + "final_train_loss": 4.8104, + "steps": 5126, + "tokens": 42000000, + "bpb": "deferred (\u00a7C4.5)" + }, + "cell_fullattn_42M_s0": { + "final_train_loss": 5.6667, + "steps": 5126, + "tokens": 42000000 + }, + "rung_42M_result": { + "metric": "val_CE best (n=1, iso-FLOP, DIRECTIONAL)", + "ssm_base": 4.7353, + "ssm_1to3": 4.724, + "all_attn": 5.4223, + "swa128": 5.6805, + "swa128_nope": 5.8731, + "finding": "SSM mixer >> SWA/full-attn (but LR-not-retuned-per-arm confound); attention fraction 1:1~1:3 negligible; NoPE worse not better", + "artifact": "rung_42M_comparison.md", + "bpb": "deferred \u00a7C4.5" + } + } + ], + "proposals": [ + { + "slug": "build-north-mini-code-moe", + "kind": "new-build", + "md_path": "research/proposals/build-north-mini-code-moe.md", + "created": "2026-06-17", + "status": "open" + }, + { + "slug": "build-sana-wm-world-model", + "kind": "new-build", + "md_path": "research/proposals/build-sana-wm-world-model.md", + "created": "2026-06-17", + "status": "open" + }, + { + "slug": "block-text-diffusion", + "kind": "new-build", + "md_path": "research/proposals/block-text-diffusion.md", + "created": "2026-06-17", + "status": "open" + }, + { + "slug": "midtrain-context-extension", + "kind": "needs-approval", + "md_path": "research/midtraining/plan.md", + "created": "2026-06-27", + "status": "open", + "reason": "Step-0 diagnostic GATE FAILED: faithful base shows no clean passkey retrieval up to its trained 4096 (acc 0.08-0.33, noisy) -> context-extension MOOT/propose-only at 596M/1.19B (research predicted this). The ~0.3-GPU-day diagnostic saved ~1.5-2 GPU-days." + }, + { + "slug": "rlvr-method-plan", + "kind": "needs-approval", + "md_path": "research/rlvr/plan.md", + "created": "2026-07-01", + "status": "accepted", + "researched_on": "2026-07-01", + "stage": "rlvr", + "reason": "\u00a7C27 rlvr method-research (stage-method-research wf_5cf3b0d7-5f1, 3-persona adversarially verified). HONEST verdict: at 0.6B on a ~10x-undertrained SFT base, RLVR is a methodology/negative-result artifact, not a reasoning win \u2014 RL only sharpens pass@k the base already has, and ours is near-floor; the reasoning lives in SFT/distillation. NOT runnable today: decision metric (held-out GSM8K/MATH-500 exact-match pass@1+pass@k) does NOT exist + GRPO prompt/eval sets not on disk. Prereqs P1-P4 (dataset-forge GSM8K+MATH bands, MATH-500 eval, build math-acc-v1 exact-match/passk in eval-harness, pin+fuzz extractor) then Phase-1 pass@k go/no-go (~0.3 GPU-day, STOP if pass@k~0). Matched control = iso-compute rejection-sampling SFT + REQUIRED random-reward gate (Spurious Rewards, Qwen lineage). On-box ~3-5 GPU-days; multi-seed CI cohort off-box/propose-only.", + "note": "user accepted Phase 2 (exploratory Dr.GRPO + iso-compute RFT + random-reward gate) 2026-07-02" + }, + { + "slug": "preference-dpo-plan", + "kind": "needs-approval", + "md_path": "research/preference/plan.md", + "created": "2026-07-01", + "status": "open", + "note": "\u00a7C27 preference-stage plan (wf_5b68bd62-487, 3-persona + adversarial): DPO-vs-chosen-SFT on UltraFeedback in SFT plain format; ~2.5 GPU-days; claim ceiling = narrow win on pref-acc + KL-frontier; judge item off-box or verdict capped directional", + "prediction_md": "research/preference/prediction.md", + "terminal_verdict": "predicted-null" + }, + { + "slug": "next-step-verdict-2026-07", + "kind": "needs-approval", + "md_path": "research/decisions/2026-07-05_next-step-verdict.md", + "created": "2026-07-05", + "status": "open", + "reason": "Honest strategic verdict (3-persona adversarial, all cores survived red-team): STOP post-training on the ~2-tok/param base (it IS the ceiling; SFT-directional + RLVR-null are the same ~1% capability). Rank: (1) ship loop end-to-end + honest 2-wins paper [cheap/CPU], (2) RENT multi-GPU scaling ladder = NorMuon/IMU-1 persistence + correctness-gated MFU report [THE hinge, ~2-8k, the un-fakeable Scaling credential], (3) write the CCE Triton kernel. Make compute-vs-loss the object of study." + }, + { + "slug": "next-step-verdict-v2-onbox", + "kind": "needs-approval", + "md_path": "research/decisions/2026-07-05_next-step-verdict-v2-onbox.md", + "created": "2026-07-05", + "status": "open", + "reason": "ON-BOX no-rent verdict (4-persona Anthropic+GDM, all cores survived). Frontier-Pre-training-RE(Scaling) does NOT survive no-rent -> PIVOT to RS/RE-via-Fellows (small honest scaling+muP study, spine=RLVR-null+SFT-confound) + research-infra ML-Eng (ship the loop) + kernels hedge. Ranked on-box: #1 SHIP THE LOOP end-to-end + 2-wins paper (loop has NEVER run: 1 hand-made digest, stale PID); #2 persistence-first fixed-N token sweep (does IMU-1 +0.474 persist?) + 1-D L(C) frontier (NOT 2-D, unidentifiable on a ray); #3 CCE kernel (correctness-first, win=memory). STOP all post-training on the 596M base." + }, + { + "slug": "scaling-persistence-ladder", + "kind": "big-run", + "md_path": "research/scaling/plan.md", + "created": "2026-07-05", + "status": "accepted", + "reason": "QUEUED on-box scaling stage (persistence-first). Closes the IMU-1 RESULT.md Limitation #3 (no >=3-budget scaling curve): does NorMuon +0.474 BPB win persist or vanish with budget? Reuse train_ablation.py at fixed N=596M, sweep tokens 42M(reuse)/168M/420M/840M x {adamw,normuon} x seeds, fit_gap_trend. ~10.5 GPU-days full (or ~4 for 3-budget). Fires after RLVR closes + explicit go; NOT auto-armed (10-day commitment). Cells: research/scaling/ladder_plan.json.", + "note": "COMPLETED 2026-07-12: recorded as run 2026-07-05_qwen3-0.6b_scaling-persistence (type=scaling-fit, verdict=directional, trend=CONVERGES on wikitext-2 + code_py). Top rung 420M is n=2 -> capped to directional. See experiments/.../verdict.json.", + "completed_run": "2026-07-05_qwen3-0.6b_scaling-persistence" + }, + { + "slug": "lifecycle-north-star-2026-07", + "kind": "needs-approval", + "md_path": "research/decisions/2026-07-13_lifecycle-north-star-verdict.md", + "created": "2026-07-13", + "status": "open" + }, + { + "slug": "build-minicpm5-1b", + "kind": "new-build", + "md_path": "research/proposals/build-minicpm5-1b.md", + "created": "2026-07-14", + "status": "open" + }, + { + "slug": "build-v-splade-quality", + "kind": "new-build", + "md_path": "research/proposals/build-v-splade-quality.md", + "created": "2026-07-14", + "status": "open" + }, + { + "slug": "safety-stage-plan", + "kind": "needs-approval", + "md_path": "research/safety/plan.md", + "created": "2026-07-19", + "status": "open" + }, + { + "slug": "build-hybrid-ssm-attention-jax", + "kind": "new-build", + "md_path": "research/proposals/build-hybrid-ssm-attention-jax.md", + "created": "2026-07-19", + "status": "open" + } + ], + "never_repeat": [], + "papers": [ + { + "slug": "qwen3-imu1-matched-compute", + "title": "Reproduce, Then Modernize: A Matched-Compute Study of a 2026 Architecture-and-Optimizer Recipe on a Bit-Exact Qwen3-0.6B", + "status": "abandoned", + "venue": "arxiv", + "arxiv_id": null, + "run_ids": [ + "2026-06-08_qwen3-0.6b_faithful-baseline", + "2026-06-08_qwen3-0.6b_imu1-modernized" + ], + "md_path": "research/papers/qwen3-imu1-matched-compute/", + "created": "2026-06-15", + "updated": "2026-06-19", + "notes": "SUPERSEDED by qwen3-0.6b-study (consolidated single-model study)" + }, + { + "slug": "qwen3-0.6b-study", + "title": "Reproduce, Then Attribute: A Controlled Study of the LLM Training Lifecycle on a Bit-Exact Qwen3-0.6B Reproduction", + "status": "packaged", + "venue": "arxiv", + "arxiv_id": null, + "run_ids": [ + "2026-06-16_qwen3_normuon-vs-adamw", + "2026-06-17_qwen3-0.6b_vibethinker-small-reasoning", + "2026-06-18_qwen3-0.6b_imu1-deconfound-p1" + ], + "md_path": "research/papers/qwen3-0.6b-study/", + "created": "2026-06-19", + "updated": "2026-07-07", + "notes": "Full-lifecycle refresh finalized 2026-07-07: 34-page PDF (tectonic, clean glyphs), 26 verified refs, arxiv_package.tar.gz (35 files, standalone-compile verified), SUBMISSION.md dossier, report page published as Artifact https://claude.ai/code/artifact/17f80f4a-bb77-43d4-a11f-55fa8d07a41e (18/18 fragments adversarially verified vs LaTeX source), distribution kit slots filled. Human steps remaining: share artifact, push repo, arXiv endorsement+submit, post HN/r-ML/X." + } + ] +} diff --git a/research/ledger/runs/2026-06-16_qwen3-faithful_eval-first.md b/research/ledger/runs/2026-06-16_qwen3-faithful_eval-first.md new file mode 100644 index 0000000..fbf4ee5 --- /dev/null +++ b/research/ledger/runs/2026-06-16_qwen3-faithful_eval-first.md @@ -0,0 +1,30 @@ +# Run 2026-06-16_qwen3-faithful_eval-first + +**Type:** eval · **Suite:** text-lm-v2 · **Objective:** pretrain-ablation · **Status:** done +**Model:** Qwen3-0.6B faithful reproduction — `checkpoint_qwen3_baseline2tpp.pt` (step 18,150, the 2-TPP baseline) +**Significance:** this is the **first checkpoint ever scored through the standing `/eval-harness` suite** (previously the suite had produced zero result files). It turns the eval-harness from *tested-but-unexecuted* into *executed*. + +## Result (single checkpoint, self-floor mode) + +| Corpus | PPL | BPB (bits/byte) | Noise floor (PPL) | tokens scored | +|---|---|---|---|---| +| `wikitext2_raw_v1_val` (headline) | 37.01 | **1.2256** | ±1.30 | 204,600 | +| `codeparrot_clean_valid` | 438.67 | 2.1286 | ±280.6 | 204,600 | + +- **Headline cross-tokenizer metric: BPB = 1.2256** on wikitext-2. This is the comparable number future runs (and other tokenizers, e.g. SmolLM2) are measured against. +- The high code PPL (438) is honest and expected: this faithful baseline was trained on FineWeb-Edu with little code, so it is poor at code — the suite reports it without flattering. +- wikitext PPL 37.01 is higher than the build's own FineWeb-Edu val PPL of 28.65 because **it is a different corpus** (wikitext is out-of-distribution for a FineWeb-trained model); the numbers are not directly comparable, which is exactly why a *standing* suite with pinned corpora matters. + +## A real bug this run surfaced (and fixed) +The constructed eval script crashed on first smoke with `AttributeError: 'NoneType' object has no attribute '__dict__'` — `load_model_module()` exec'd `model.py` (which uses `@dataclass`) via `importlib` **without registering the module in `sys.modules` first**, so `@dataclass`'s `cls.__module__` lookup returned `None`. Fixed in `eval_suite_template.py` (register before exec). This is exactly the class of defect that only a real run surfaces — it would have crashed the first eval-harness invocation in production. + +## Honest caveats — what this is NOT +- **n = 1, single checkpoint, single seed.** This is NOT a 9+/10 result and makes NO capability or comparison claim. It is the first *executed* eval, full stop. +- No baseline comparison (self-floor mode), so no `win|loss|inconclusive` verdict — that requires a multi-seed iso-FLOP cohort (the next step on the path to 9+). +- The noise floor here is the single-checkpoint subsample floor (corpus jitter), explicitly NOT a seed CI. + +## Provenance +- Constructed script: `Qwen3-0.6B/experiments/2026-06-16_qwen3-faithful_eval-first/eval_suite.py` +- Raw output: `.../eval/suite_results.json` (+ `generations.md`) +- Corpora pinned: wikitext-2 `b08601e0…`, codeparrot `4db92d2e…` +- Deploy gate (`research/ci/eval_gate.py`) verified it can read this artifact (exit 0). diff --git a/research/ledger/runs/2026-06-16_qwen3_normuon-vs-adamw.md b/research/ledger/runs/2026-06-16_qwen3_normuon-vs-adamw.md new file mode 100644 index 0000000..da52615 --- /dev/null +++ b/research/ledger/runs/2026-06-16_qwen3_normuon-vs-adamw.md @@ -0,0 +1,75 @@ +# NorMuon vs AdamW on Qwen3-0.6B 2D Weights — Single-Variable Iso-FLOP Ablation (42M tokens) + +**Run ID:** `2026-06-16_qwen3_normuon-vs-adamw` · **Suite:** text-lm-v2 · **Verdict:** WIN (scoped) · **Status:** verified against on-disk evidence (all 6 logs, both result JSONs, scorer, stats gate, NorMuon impl re-read; CI reproduced bit-for-bit). + +## 1. Headline (correctly scoped) + +> At a **fixed 42M-token, iso-FLOP budget** (640 steps × 65,536 tok) on an **identical faithful Qwen3-0.6B** (440M non-embedding params), swapping **AdamW → NorMuon on only the 2D hidden weights** — everything else (architecture, data, fixed split seed 0, schedule shape, embedding/1D treatment, weight decay) held byte-identical — improves text-LM **bits-per-byte by +0.474 on wikitext-2 (95% CI [+0.443, +0.505]) and +0.502 on code ([+0.456, +0.547])**, 3 seeds/arm, fully disjoint arms, significant. A **matched-config AdamW LR sweep (1.7/2.4/3.5/4.8e-3)** shows AdamW's BPB is **flat within seed noise across 1.7–3.5e-3** and **no AdamW LR comes within ~10× of closing the gap** (full LR spread ~0.047 bpb vs the 0.474 gap), so this is **NorMuon vs AdamW-anywhere-in-a-reasonable-LR-range**, not an undertuned baseline. **This is an early-training optimization-speed signal at one architecture and one budget; we do NOT claim it holds at scale.** + +The qualifiers are load-bearing — see Limitations. The unqualified claim "NorMuon gives a 22% BPB improvement" would be an over-claim and is not what this run shows. + +## 2. Results + +**wikitext-2-raw-v1 (val, pinned rev `b08601e…`)** — lower is better + +| Arm | Seed BPB | Mean | ±SEM | fineweb-val PPL | +|---|---|---|---|---| +| AdamW @ 2.4e-3 | 2.1050, 2.1024, 2.1221 | **2.1098** | 0.0062 | 147, 145, 156 | +| NorMuon @ 0.011 | 1.6499, 1.6248, 1.6317 | **1.6355** | 0.0075 | 61, 60, 60 | +| **Improvement (AdamW − NorMuon)** | | **+0.4743** | | | +| **95% CI (Welch-t, df 3.86→3, t=3.182)** | | **[+0.4435, +0.5052]** | significant ✓ | | + +**codeparrot-clean-valid (pinned rev `4db92d2…`, 500k chars)** + +| Arm | Mean BPB | ±SEM | Improvement | 95% CI | +|---|---|---|---|---| +| AdamW | 3.3847 | 0.0099 | — | — | +| NorMuon | 2.8831 | 0.0104 | **+0.5016** | **[+0.4560, +0.5471]** ✓ | + +Arms are **fully separated**: worst NorMuon (1.6499) ≪ best AdamW (2.1024). Within-arm SD ≈ 0.011–0.013; the gap is ≈40 within-arm SD. I re-ran `seed_delta_significant` on the raw seeds and it reproduces the recorded CI, df, and verdict **exactly**. + +**Throughput / MFU.** Both arms trained cleanly at ~52.4 GB peak (under the 0.85 unified-memory guard), AdamW ~7,340 tok/s, NorMuon ~6,664 tok/s (NorMuon's Newton–Schulz orthogonalization is the ~9% overhead). Reported **MFU ≈ 29%** uses a **GB10 bf16-dense peak of 125 TFLOP/s that is an *estimated* spec number, not a measured device roofline** — treat the MFU figure as approximate. + +**Training health (verified all 6 logs).** Every cell trained monotone-down with no NaN/spike/plateau-from-instability. AdamW seed0 loss 8.18→5.15, grad-norm decaying smoothly 1.0→0.22; NorMuon seed0 loss 7.94→4.25, grad-norm 1.9→0.17. The win is **not** an artifact of a broken or divergent baseline — AdamW was a healthy run that simply converged slower at this budget. + +## 3. Why this matters + +The project's IMU-1 matched-compute result (NorMuon **bundled with ~5 other changes** — partial-RoPE, WSD-to-zero, etc. — at 1.19B tokens, −17.9% PPL) had to mark optimizer attribution as an explicit **limitation**: it could not say how much of the gain came from the optimizer versus the architecture/schedule changes. This run **isolates the optimizer as a single variable** (`train_ablation.py` changes only how the 196 2D matrices are stepped; the 114 embedding/1D params are AdamW@2.4e-3 wd=0 in *both* arms; 2D wd=0.1 in both; data split seed 0 fixed). It directly answers that open question at this budget: at 42M tokens, the NorMuon update rule **alone** moves BPB substantially. That clean attribution — not the magnitude — is the result. + +## 4. Limitations (every surviving red-team caveat, stated plainly) + +1. **AdamW LR — RESOLVED by a matched-config sweep (no longer an open caveat).** The original concern was that AdamW's 2.4e-3 was tuned at a 28×-longer budget and might be undertuned for this 42M-token horizon. We ran a **confirmatory AdamW LR sweep at the EXACT ablation config** (42M tok, wd=0.1, 2D-AdamW split, seed 0, same cached data): wikitext BPB = 1.7e-3 → **2.1246**, 2.4e-3 → **2.1098** (the 3-seed cohort mean), 3.5e-3 → **2.1223**, 4.8e-3 → **2.1569**. **Honest reading (not a clean U-shape):** AdamW is **flat within seed noise across 1.7–3.5e-3** — those three span only **0.015 bpb**, inside the cohort's ±0.011 seed band, so 2.4e-3 vs 3.5e-3 is a statistical tie and we do **not** claim a single exact optimum — and degrades only at 4.8e-3. The conclusion is **robust to which LR is best**: the **full AdamW LR spread is ~0.047 bpb, ~10× smaller than NorMuon's +0.474 advantage**, so **no AdamW LR in the swept range comes close to closing the gap**. The headline is therefore **NorMuon vs AdamW-anywhere-in-a-reasonable-LR-range**, not an undertuned-baseline artifact. **Caveat:** the off-baseline LRs are **single-seed** (only the 2.4e-3 point has 3 seeds), so the within-band ordering is unresolved — but the load-bearing conclusion (no LR closes the gap) does not depend on it. wd is held at 0.1 for single-variable isolation rather than AdamW's own 0.01 — a deliberate design choice (see #2). Evidence: `results/lr_sweep_bpb.json` (3 off-baseline points) + `results/verdict.json` (2.4e-3, 3 seeds), `results/adamw_lr{17,35,48}_seed0.log`. + +2. **Cross-arm weight-decay provenance.** Both arms use 2D wd=0.1 (NorMuon's/IMU-1's tuned value). AdamW's faithful recipe was tuned at wd=0.01. So AdamW runs at a 10× wd it was never tuned for, on a budget it was never tuned for — part of the gap may be baseline handicap rather than genuine optimizer advantage. (wd is *held equal* across arms, which is correct for single-variable isolation, but it is not AdamW's own tuned wd.) + +3. **42M tokens is deep in the early-training regime where Muon-family optimizers are most flattered.** Both arms are ~28× under-trained versus the faithful 1.19B baseline (wikitext BPB **1.2256**) — both 42M models (2.11 / 1.64) are far worse than that baseline. Orthogonalized/RMS-matched updates help most exactly here; the same NorMuon inside the full IMU-1 bundle at 1.19B gave only −17.9% PPL, so the 22–75% relative gap here is **expected to compress substantially with budget and may largely vanish**. There is **no scaling curve** (≥3 budgets) — nothing licenses extrapolation. + +4. **Seeds vary init + DataLoader shuffle on a FIXED data split (seed 0).** Corpus-resampling variance is structurally excluded, so the reported ±0.006–0.008 SEM **under-estimates true end-to-end seed variance**; the CI is narrower than a fully-randomized design would give. The verdict is robust by a wide margin (breaking significance needs ~15× SEM inflation at the actual df), but the claim must be stated as **"significant under init+shuffle variance on a fixed split,"** not as a fully-randomized 3-seed result. n=3 is also exactly the gate minimum (df floored to 3, t=3.182 — honestly wide, not "comfortably large"). + +5. **Scope.** One architecture (Qwen3-0.6B), two corpora, one optimizer pair, one budget, one seed-triple per arm. The BPB *scoring* is clean (sum_nll/ln2/bytes; byte denominator and SEQ=1024/STRIDE=512/pinned corpora bit-identical across all 6; all loads `strict=True`; checkpoints identical size; an independent in-training fineweb-val PPL on a different corpus reproduces the same ~2.4× gap) — the caveats above are about the **comparison**, not the metric. + +## 5. Verdict + +**Yes — this is a genuine, publishable miniature result, scoped tightly.** It is a clean single-variable, iso-FLOP, multi-seed ablation with a correctly-computed conservative Student-t CI, fully disjoint arms, healthy baseline training, and an independent corpus corroborating the gap. It legitimately isolates what the bundled IMU-1 paper could not attribute. But it is **a convergence-speed result at 42M tokens with an un-re-tuned, foreign-wd AdamW baseline and a fixed-split seed design — not a steady-state quality claim and not evidence the gap survives at scale.** Reported with all five caveats, an Anthropic RS would sign it as "NorMuon wins the early-training optimizer race on Qwen3-0.6B's 2D weights at 42M tokens, single variable, n=3 — needs a scaling curve and a per-horizon AdamW LR sweep before any general claim." Without those caveats, it would be an over-claim and should not ship. + +## 6. Reproducibility + +**Commands** (sequential on one GB10; each cell ~95–106 min): +``` +# 6 training cells — only --optimizer and --seed vary +for opt in adamw normuon; do for s in 0 1 2; do + python train_ablation.py --optimizer $opt --seed $s --steps 640 +done; done +# defaults: --peak_lr 2.4e-3 --normuon_lr 0.011 --weight_decay 0.1 --grad_clip 1.0 --mem_fraction 0.85 +# then score all 6 checkpoints through text-lm-v2: +python score_cohort.py # -> results/cohort_bpb.json +# verdict (Welch-t via research/eval_stats.seed_delta_significant) -> results/verdict.json +``` +- **Budget:** 640 steps × (SEQ 4096 × micro-batch 4 × grad-accum 4 = 65,536 tok) = **41,943,040 tok/cell**, 6 cells. +- **Single variable:** 196 2D non-embed matrices → {AdamW@2.4e-3 | NorMuon@0.011}; 114 embedding/1D params → AdamW@2.4e-3 wd=0 (both arms); 2D wd=0.1 (both arms); shared cosine schedule, warmup 50, end-ratio 0.1, global grad-clip 1.0; bf16; `torch.compile`. +- **Seeds:** per-cell `--seed` sets init + shuffle only; **data split `SPLIT_SEED=0` fixed across all 6 cells** (identical corpus, token-cached). +- **Decontam:** FineWeb-Edu stream, decontam dropped **0/451** val docs (`results/decontam_report.json`); eval corpora (wikitext-2, codeparrot) are independent public sources, not FineWeb-Edu — no train→eval leak path. +- **Pinned eval corpora:** `Salesforce/wikitext` wikitext-2-raw-v1 rev `b08601e04326c79dfdd32d625aee71d232d685c3`; `codeparrot/codeparrot-clean-valid` rev `4db92d2ec0c1b4c41eeb439cfae16854511d9dcd` (500k chars). SEQ 1024 / STRIDE 512 / 200 windows; tokenized once, reused for all 6 → n_bytes 869,710 (wikitext) / 843,643 (code) identical across cells. +- **Checkpoints:** `results/checkpoint_{adamw,normuon}_seed{0,1,2}.pt` (AdamW 1,192,229,527 B; NorMuon 1,192,230,159 B — identical `Qwen3Config()`, loaded `strict=True`). + +**Evidence files** (all under `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/`): `results/{adamw,normuon}_seed{0,1,2}.log`, `results/cohort_bpb.json`, `results/verdict.json`, `train_ablation.py`, `score_cohort.py`, `normuon.py`; stats gate `research/eval_stats.py`; faithful baseline `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/` (wikitext BPB 1.2256 @ 1.19B tok). \ No newline at end of file diff --git a/research/ledger/runs/2026-06-17_qwen3-0.6b_openr1-math-220k.md b/research/ledger/runs/2026-06-17_qwen3-0.6b_openr1-math-220k.md new file mode 100644 index 0000000..b9976b2 --- /dev/null +++ b/research/ledger/runs/2026-06-17_qwen3-0.6b_openr1-math-220k.md @@ -0,0 +1,65 @@ +# Run 2026-06-17_qwen3-0.6b_openr1-math-220k — dataset-prep + +- type: **dataset-prep** · objective: **finetune** (§C13) · framework: null · smoke: null (forge never trains, §C5.0 is a trainer gate) +- model_dir: `Qwen3-0.6B` · technique_slug: `vibethinker-small-reasoning` +- artifacts_dir: `research/datasets/math-reasoning-openr1-math-220k/` +- prepared: 2026-06-17 by dataset-forge (headless S6 of the nightly research-loop) + +## What was prepared +Training-ready **SFT shards of verified math reasoning traces** + a hygienic +held-out eval split, for the VibeThinker-small-reasoning brief's **first run +(SFT-only)** on the Qwen3-0.6B base. + +- Source (LIVE HF API, fetch-verified 2026-06-17): `open-r1/OpenR1-Math-220k`, + config `default`, split `train` (93,733 rows), **revision sha + `e4e141ec9dea9f8326f4d347be56105859b2bd68`** (matches the brief's pin), + ungated, apache-2.0. +- Tokenized with the model's OWN tokenizer `Qwen/Qwen3-0.6B-Base` + (len 151,669; config vocab 151,936; eos 151,643) per §C10 — PPL/fertility + under any other tokenizer is meaningless. +- SFT sample = `problem` prompt + blank line + the FIRST generation whose + `correctness_math_verify` is True (verified, answer-checked reasoning trace); + rows with no verified trace are dropped (the brief's data-hygiene / answer-check + filter). Each sample truncated to the build seq_len 4096; a per-sample + `prompt_len` is written to `train_meta.jsonl` so `/ablation-runner`'s + masked-completion CE (`research/posttrain_losses.py`) masks the prompt span. + +## Why this domain + dataset +- Domain (math reasoning) is FIXED by the consuming brief + `research/briefs/vibethinker-small-reasoning.md` (verdict `needs-dataset`, + objective `finetune`). The brief pins OpenR1-Math-220k as the SFT set; forge + recorded the evidence (see `selection.md`) rather than re-competing the choice. +- Model-fit (§C12): the base is a text-LM decoder-only Qwen3-0.6B repro; the + brief adapts it toward code+math **reasoning** (RLVR/GRPO training-stage), + SFT-first. OpenR1-Math carries long-CoT `` traces with per-generation + verification flags — exactly the curriculum-SFT-on-verified-traces input the + VibeThinker Spectrum-to-Signal Stage-1 recipe needs. + +## Splits + hygiene (finance contracts §6 / recipes §R5) +- Document-level seeded split FIRST (sha256 mod, seed `forge-v1`, eval frac + 0.005), then streaming **13-gram Jaccard dedup** of eval docs vs the streamed + train side, dropping eval docs > 0.8 (docs_dropped reported in stats.json). + No eval doc enters train (its uuid is excluded from the train pass). +- **Forgetting probe (§C13, Phase 6 step 4):** held-out general-distribution + slice from `HuggingFaceFW/fineweb-edu` config `sample-10BT` @ + `87f09149ef4734204d70ed1d046ddc9ca3f2b8f9` (ungated, odc-by) — the FineWeb-Edu + distribution the base was pretrained on (train_qwen3.py line 151) and whose + held-out PPL (base 28.65) the brief names as the catastrophic-forgetting + guard. Same doc-level seeded split. Written to `eval/forgetting_*`. + eval-harness owns scoring it against the base checkpoint (§C10/§C11). + +## Linkage (for /ablation-runner via /post-train) +- Base checkpoint: + `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` + (596,049,920 params; FineWeb-Edu val PPL 28.65). +- Adaptation: curriculum SFT (masked-completion CE on verified traces) — the + brief's first single-variable arm (§C18). GRPO is a separate queued follow-on + (needs the DeepScaleR prompt set + MATH-500 anchor, not prepared here). +- TOKEN_TARGET = 125M tokens = 1.25 × the brief's recommended ~100M SFT budget + (SKILL Phase 5 rule; budget source = the brief). + +## Evidence +- selection.md (domain evidence + candidate table + pick rationale) +- card.md (sources, license verbatim, sha, prep commands, split recipe) +- stats.json (train/eval/forgetting tokens, shards, docs_dropped, fertility) +- prep.log (verbatim decoded 256-token readback window) diff --git a/research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md b/research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md new file mode 100644 index 0000000..989bb39 --- /dev/null +++ b/research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md @@ -0,0 +1,140 @@ +# Run 2026-06-17_qwen3-0.6b_vibethinker-small-reasoning + +- type: **finetune** · objective: **finetune** (§C13) · framework: pytorch (§C14) +- technique: `vibethinker-small-reasoning` (brief `research/briefs/vibethinker-small-reasoning.md`, taste 6.0) +- method: **SFT-only**, FIRST run = single variable (§C18); GRPO is a separate queued arm. +- model_dir: `Qwen3-0.6B` +- base checkpoint: `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt` + (596,049,920 params, FineWeb-Edu val PPL 28.65, vocab 151,936, seq 4096) — loaded `strict=True`. +- experiment dir: `Qwen3-0.6B/experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/` + (minimal diff, §C4.4 — no canonical file edited). + +## Data +`research/datasets/math-reasoning-openr1-math-220k/` — 125,000,592 SFT tokens +(35,512 verified OpenR1-Math-220k reasoning traces, Qwen3 tokenizer, EOS-separated +flat uint32 shards). Response-masked: per-sample `prompt_len` from `train_meta.jsonl` +marks the prompt span as label `-100` (ignored); only the verified reasoning trace +is trained. dataset HF revision `e4e141ec9dea9f8326f4d347be56105859b2bd68`. + +## Recipe (brief "Exact recipe") +- global batch **128** (reported) = micro_batch 4 × grad_accum 32; seq_len 4096. +- peak LR **5e-5** → cosine → **8e-8** (reported); **5%** linear warmup (reported). +- AdamW(0.9, 0.95) eps 1e-8, weight_decay 0.01, grad_clip 1.0, bf16, smoothing 0. +- loss = `posttrain_losses.masked_sft_nll` (eps=0 → per-resp-token `-logp`), realized + as a **chunked masked CE** (§C1 — vocab 151,936 > 64k); startup asserts the GPU + loss == the unit-tested `masked_sft_nll` oracle on a tiny tensor. + +## §C5 bounded-auto-run evidence (recorded BEFORE launch) +- **§C5.0 smoke = pass** — `results/smoke.log`: oracle equivalence (4.414680 == + 4.414680), base load strict=True, 4 finite SFT steps (loss 2.82→2.43), checkpoint + save→reload **byte-identical** (model max|Δ|=0.0), resume continues finite. The real + launch script also ran 2 steps + a resume (step 2 → cursor 8 → step 4), exit 0. +- **§C5.1 concurrency = clear** — pgrep/nvidia-smi: no trainer; `sentinel.py preflight` + OK (mem_available 87%, disk_free 3406 GB, load1 0.23, trainers=none). +- **§C5.2 budget = 125,000,000 tokens** — source: brief §C5.2 / Addendum (~100–125M, + one pass over the prepared 125,000,592-tok set). total_steps = 238. +- **§C5.3 probe** (`results/probe.json`, exact config seq4096×mb4×ga32) = + **7,465 tok/s**, **peak 52.39 GB** < 71.4 GB cap (60% of 119 GB pool) → fits. +- **§C5.4 ETA = 4.65 h** (125M / 7,465 tok/s). +- **§C5.5 resume = proven** — real-script resume exit 0 + byte-identical save→reload. +- **§C5.6 sentinel** — `sentinel.py watch --pid ` armed at launch (detached). +- **§C5.7 safe_cuda + chunked CE = present** — grep verified `import safe_cuda` + + `safe_cuda.guard()` and `masked_chunked_ce`; no full-vocab `logits.float()`. + +## Win condition (§C13, two-part — scored tomorrow by /eval-harness, S8) +1. Held-out reasoning gain clearing the §C17 noise floor (seeds ≥ 3, paired CI + excludes 0) on the in-domain reasoning eval split. +2. **§C13 forgetting probe**: FineWeb-Edu PPL under the Qwen3 tokenizer must NOT + regress past 28.65 beyond the noise floor. A reasoning gain bought with PPL + regression is a `loss`. + +## Abort criteria (brief) +forgetting PPL > floor above 28.65 for 2 consecutive evals · grad-norm > 5× the SFT +median or NaN/Inf · no reasoning-probe signal after 25% of budget. + +--- + +## Post-training scoring (S8 — completed 2026-06-18, by /eval-harness) + +The SFT finished cleanly (238/238 steps, 297 min, no sentinel kill). Scored against +the base through the standing eval-harness in finetune mode (text-lm-v2). + +| Probe | base → SFT | reading | +|---|---|---| +| **Reasoning** (in-loop held-out OpenR1-Math PPL) | 14.26 → **11.60** (−18.7%) | positive signal | +| **Forgetting — wikitext-2** (BPB / PPL) | BPB 1.2256 → 1.2263 · PPL 37.01 → 37.08 (+0.2%, **not significant**) | **retained** | +| **Forgetting — code** | PPL 438.7 → 425.7 (−3.0%, **not significant**) | **retained** | + +### Honest verdict against the brief's OWN win condition — NOT a `win` (yet) +The brief (§"Win condition") requires **two** things this single run does **not** satisfy: +1. **Reasoning gain needs ≥3 seeds + a paired CI excluding 0** (§C17). This is **n=1** — + the −18.7% is a *single-run* signal, **not** significance-gated. Per the brief's own + bar it cannot be called a confirmed reasoning gain until the multi-seed run exists. +2. **The brief's forgetting probe is FineWeb-Edu PPL vs the base's 28.65.** I ran the + standing suite's general corpora (**wikitext-2 + code**) instead — strong evidence of + no catastrophic forgetting (both retained, sub-floor), but **not** the exact + FineWeb-Edu number the brief specified; that probe was not run. + +**So:** the honest outcome is **"no catastrophic forgetting (retained on wikitext+code) + a positive single-run reasoning signal (−18.7% PPL)"** — a *successful SFT*, but the formal §C13 `win` is **unclaimed**, pending (a) the ≥3-seed reasoning CI and (b) the FineWeb-Edu forgetting probe. The run's `verdict` is left null by design (not win/loss). The queued **GRPO arm** (per line 5) is the natural next step. + +### Caveats +Reasoning is held-out PPL on the *training distribution* (OpenR1-Math traces), **not** +problem-solving accuracy on an independent benchmark (e.g. GSM8K) — a real but limited +signal. Small (0.6B), under-trained base. Eval: `…/eval/suite_results.json`. + +--- + +## §C13 brief-specified probes + final verdict (S8 completed 2026-06-18, /eval-harness) + +The 2026-06-18 01:34 pass above ran the *generic* text-lm-v2 Section-4 forgetting probe +(wikitext-2 + code) and **explicitly did NOT run the brief's named §C13 corpus** +(FineWeb-Edu) nor score the held-out reasoning split through eval-harness, so it left +`verdict: null`. This section closes both gaps and judges the run. + +Scored via `eval_brief_probes.py` (reuses the **frozen text-lm-v2** window/floor/ +significance machinery — `SEQ=1024 STRIDE=512 MAX_WINDOWS=200 FLOOR_WINDOWS=66`, imported +verbatim from the constructed `eval_suite.py`; constants NOT tuned per run, Hard rule 3), +on the dataset-forge pre-tokenized held-out splits, Qwen3 own tokenizer (§C10), both +checkpoints `strict=True`. Base = the pre-SFT faithful build checkpoint. + +| Probe (corpus_id) | base PPL | SFT PPL | Δ | floor_abs | label | +|---|---|---|---|---|---| +| **§C13 forgetting — FineWeb-Edu** (`fineweb_edu_sample10bt_heldout`, 202 docs / 204,600 scored tok) | 24.5514 | 24.7331 | **+0.18 (+0.74%)** | 8.0155 | **retained — not significant** | +| **held-out reasoning — OpenR1-Math** (`openr1_math_220k_heldout`, 66 docs / 204,600 scored tok) | 17.6035 | 14.2217 | **−3.38 (−19.2%)** | 4.4291 | not significant (below corpus-partition floor) | + +Artifacts: `…/eval/brief_probes_results.json`, `…/eval/brief_probes.log`. +Suite methodology stamp: **text-lm-v2** (corpora are brief-specified, not the frozen +Section-1 corpora — comparable only to other runs on the SAME corpus_id + tokenizer). + +### §C13 FineWeb-Edu forgetting probe — **PASS** +SFT FineWeb-Edu PPL (24.73) regressed only **+0.18 (+0.74%)** vs the measured base +(24.55), far **below** the 3-disjoint-subsample noise floor (8.02) → **retained, not +significant**. The SFT model is also **below** the brief's stated base claim of **28.65**. +(The measured base 24.55 ≠ the README's 28.65: the 28.65 is the build's own +`eval_original_vs_repro.py` 300k slice; this is the dataset-forge 202-doc held-out +FineWeb-Edu split under the v2 window config — a different slice/windowing, so a different +absolute number. The contract test — "did the SFT regress past base beyond the floor" — is +PASS either way.) **No catastrophic forgetting.** + +### Reasoning signal — promising but NOT a §C17 win +eval-harness held-out reasoning PPL **17.60 → 14.22 (−19.2%)**, directionally consistent +with the in-loop trainer log (14.262 → 11.596 on its own slice). But: (a) the −3.38 gain +is **below even the single-checkpoint corpus-partition floor (4.43)** on this noisy split, +and (b) this is **n=1** — no seed cohort, so it categorically **cannot** clear the §C17 +`seeds ≥ 3 / paired-CI-excludes-0` bar required for a `win`. + +### Verdict: **inconclusive** (honest n=1) +- A `win` is **not permitted** at `n_seeds < 2` (§C17 / §C7 ablation-runner Phase 6.3) — + this single SFT has no seed cohort, so the reasoning gain is a **promising single-run + signal, not a significance-gated win**. +- It is **not a `loss`**: the §C13 forgetting probe PASSES (FineWeb-Edu retained, + sub-floor; no general-ability regression on wikitext/code either), so the in-domain + gain was **not** bought with forgetting. Slug NOT added to `never_repeat`; technique + stays `briefed` (the **GRPO arm** remains the queued single-variable follow-on, §C18). +- Recorded `verdict: inconclusive`; FineWeb-Edu probe is now in the run `metrics` + (`forgetting_fineweb_edu`, §C8 requirement for a finetune run). + +### Reproduce +``` +python3 Qwen3-0.6B/experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/eval_brief_probes.py +``` diff --git a/research/ledger/runs/2026-06-27_qwen3-0.6b_sft-3seed.md b/research/ledger/runs/2026-06-27_qwen3-0.6b_sft-3seed.md new file mode 100644 index 0000000..9f95678 --- /dev/null +++ b/research/ledger/runs/2026-06-27_qwen3-0.6b_sft-3seed.md @@ -0,0 +1,76 @@ +# 2026-06-27_qwen3-0.6b_sft-3seed + +**Objective:** finetune (§C13) · **Stage:** sft · **Framework:** pytorch · **Status:** done · **Verdict:** directional + +Converts the n=1 SFT (inconclusive) into a ≥3-seed paired-CI verdict with an iso-FLOP `--no_mask` +control. Single variable = **response-masking** (sft masks the prompt span / trains on the response; +ctrl trains on all tokens) on the SAME data / budget / config. Base = faithful Qwen3-0.6B +(`builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt`), loaded +strict=True. Recipe: peak_lr 5e-5 cosine→8e-8, 5% warmup, mb4×ga32 (global 128), ~125M tok, +AdamW(0.9,0.95) wd 0.01, bf16, seq 4096. MFU 32% (40.1 TFLOP/s, GB10 peak estimated) — see +`c5_evidence.json`. + +## Headline + +**Response-masking does NOT separate from the iso-FLOP `--no_mask` control at this budget.** On the +held-out reasoning set the two arms are within ~0.01 PPL and the masked vs full-sequence comparisons +disagree on significance → **directional, not a win**. The ~18% reasoning-PPL improvement over base is +delivered by *the extra in-domain math tokens*, which the control gets too — not by the masking. No +catastrophic forgetting on any general corpus. + +> The in-loop advisory verdict (`verdict.json`) showed "SFT beats control by 0.68 PPL (significant)", +> but that was an **eval-token confound** (SFT scored response-only, control scored all-tokens). Once +> both arms are re-scored on ONE fixed response-masked held-out set (`reasoning_eval.py`), the gap +> collapses 0.68 → ~0.01. The corrected verdict is the one of record. + +## Held-out in-domain reasoning (reasoning_eval.py, 3 seeds, masked PPL) + +Held-out OpenR1-Math eval split (66 docs, seeded split + 13-gram dedup vs train), response-masked, +identical token set for every arm. + +| arm | masked reasoning PPL (mean) | vs base | vs control | +|-----|-----:|-----:|-----| +| base | 14.127 | — | — | +| SFT (masked) | 11.573 | −2.554 (−18.1%) | — | +| control (--no_mask) | 11.582 | −2.545 (−18.0%) | — | +| **SFT − control** | | | **masked +0.009** (CI [0.0045, 0.0139], *sig*) · **full-seq −0.006** (CI [−0.011, −0.0015], *n.s.*) → **directional** | + +Seed spread within each arm ≈ 0.001 PPL (n=3). Full plot: `plots/reasoning_verdict.png`; data: +`reasoning_verdict.json`. + +## Eval — suite text-lm-v2 (2026-06-30) + +Standard §C10 suite (own tokenizer `Qwen/Qwen3-0.6B-Base`). One representative seed per arm scored +(held-out σ≈0.001 across seeds; the 3-seed CI lives in the reasoning verdict above). + +| corpus | corpus_id | base | SFT seed0 | control seed0 | floor_abs | label | +|--------|-----------|-----:|-----:|-----:|-----:|-------| +| wikitext2_val | wikitext2_raw_v1_val | 37.010 | 37.082 | 37.084 | 1.304 | not significant | +| code_py | codeparrot_clean_valid | 438.673 | 425.689 | 425.594 | 280.564 | not significant | + +SFT ≈ control on general English and code (both within noise) — consistent with the held-out reasoning +result. + +### Catastrophic-forgetting / retention (§C13, finetune) + +| corpus | base_ppl | target_ppl (SFT) | retention Δabs | floor_abs | label | +|--------|-----:|-----:|-----:|-----:|-------| +| wikitext2_val | 37.010 | 37.082 | +0.072 | 1.304 | retained | +| code_py | 438.673 | 425.689 | −12.984 | 280.564 | retained | +| FineWeb-Edu (reasoning_eval) | 21.495 | 21.652 | +0.157 (~0.7%) | — | retained | + +**Forgetting summary: retained** (both arms). The fine-tune did not damage general-domain capability. + +- tokenizer: `Qwen/Qwen3-0.6B-Base` · base ckpt: `…/checkpoint_qwen3_baseline2tpp.pt` +- generation probes: `eval/sft_seed0/generations.md`, `eval/ctrl_seed0/generations.md` (8 fixed prompts, seed 42) +- per-arm results: `eval/sft_seed0/suite_results.json`, `eval/ctrl_seed0/suite_results.json` +- not run: ctrl/sft seeds 1–2 on the standard suite (representative-seed policy; in-domain 3-seed CI is in `reasoning_verdict.json`) + +## Interpretation (for the caller / §C7 win criteria) + +eval-harness reports numbers; the verdict is **directional** because the single-variable contrast +(masking) is not separable from the iso-FLOP control. A defensible "win" for response-masking would +need a larger effect or a budget/scale where masking matters; at ~125M SFT tokens on a 0.6B base it +does not beat just training on the same math tokens unmasked. This is a clean **negative/null result on +the attribution question** and validates the de-confounding (the apparent in-loop win was an eval +artifact). diff --git a/research/ledger/runs/2026-06-30_qwen3-0.6b_midtrain-anneal.md b/research/ledger/runs/2026-06-30_qwen3-0.6b_midtrain-anneal.md new file mode 100644 index 0000000..4ae8f79 --- /dev/null +++ b/research/ledger/runs/2026-06-30_qwen3-0.6b_midtrain-anneal.md @@ -0,0 +1,96 @@ +# 2026-06-30_qwen3-0.6b_midtrain-anneal + +**Objective:** pretrain-ablation (§C13) · **Stage:** mid-training · **Framework:** pytorch · **Status:** done · **Verdict:** **win** (§C25 HARD-complete 2026-07-01; §C18 confound_check recorded) + +Mid-training ANNEAL A/B (`research/midtraining/plan.md` §2): a low-LR **1-sqrt cooldown** +(2.5e-4 → 2.5e-5, 10% floor, warmup 46) from the faithful base +(`builds/2026-06-08_reproduce-faithful_qwen3-0.6b/checkpoint_qwen3_baseline2tpp.pt`, strict load) +for **2300 steps × 65,536 tok = 150.7M tok/cell (~13% of the 1.19B pretrain, WSD band)**, +mb4×ga4, seq 4096, AdamW(0.9,0.95) wd 0.01, bf16, chunked CE, compile on. Single variable = **DATA**: +treatment = premium **50/50 dclm-edu + FineWeb-Edu mix**, control = **iso-token FineWeb-Edu only**, +3 seeds each (6 cells, serialized ~6.1 h/cell, ~37 h wall). The matched-decay control is the +attribution device: it eats the "LR-decay + more tokens" gain, so only the residual is a data effect. + +## Headline (pre-registered: code_py BPB, treatment − control, 3-seed Welch-t) + +**The premium-mix cooldown beats the iso-token control on code_py by +0.2716 BPB +(95% CI [+0.2641, +0.2792], significant, n=3)** — a real data effect that survives the +matched-decay control. The control itself barely moved off the base (+0.0050 code BPB), +i.e. the cooldown + 150M extra FineWeb tokens alone did ~nothing; the mix data did the work. + +| corpus | control (fineweb) | treatment (mix) | Δ (control−treatment) | 95% CI | sig | +|---|---:|---:|---:|---|---| +| **code_py** (headline) | 2.1236 ± 0.0002 | 1.8520 ± 0.0017 | **+0.2716** | [+0.2641, +0.2792] | **yes** | +| wikitext2_val (secondary) | 1.2220 ± 0.0001 | 1.2062 ± 0.0004 | +0.0157 | [+0.0140, +0.0174] | yes | + +- **English outcome vs pre-registration:** the plan pre-registered an *expected honest null* on + English; the measured effect is **small but significant** (+0.0157 BPB ≈ 1.3% relative — the same + point size as the constant-LR sibling's +0.0161 n.s., made significant here by the anneal's much + tighter seeds). Reported as-is. +- **Anchor:** the constant-LR full continued-train sibling + (`2026-06-26_qwen3-0.6b_data-mix-composition`) gave **+0.5904** code BPB + (CI [+0.5532, +0.6277]) — the anneal keeps **~46%** of that effect at a different LR regime, + matching the plan's "the anneal version will be weaker" prediction. + +## Short-context non-regression (§C25 mid-training row) — PASS + +All 4 arm/corpus cells improved on the un-annealed base (base scored under identical windows, +`base` cell in `cohort_bpb.json`): mix code +0.2766 / wiki +0.0194; fineweb code +0.0050 / +wiki +0.0036 vs base. + +## §C25 verdict — upgraded to WIN (2026-07-01 23:0x) + +The initially-missing HARD item `effective_context_length_ruler` was supplied the same night: +`run_ecl_ladder.py` (reusing the tested `research/eval_longcontext.py` loader) ran a 6-rung +passkey ladder (512→8192, 40 paired samples/rung, Wilson CIs) on all 7 checkpoints +(`ecl_ladder.json`). **No arm regresses vs base at any rung ≤ trained_len — and the anneal +IMPROVES retrieval:** + +| rung | base | fineweb (pooled n=120) | mix (pooled n=120) | +|---|---:|---:|---:| +| 512 | 0.20 | 0.31 | **0.82** | +| 1024 | 0.20 | 0.20 | 0.40 | +| 4096 | **0.03** | 0.40 | 0.40 | +| 6144 | 0.70 | 0.70 | 0.71 | +| 8192 | 0.35 | 0.41 | 0.45 | + +Both arms **fix the base's anomalous trained-length dip** (0.03 → 0.40 at 4096, reproduced at +n=40 — systematic, cause unknown, worth an interpretability look); the premium-mix arm +additionally quadruples short-rung retrieval. Footnote: `mix_seed2`'s rel-0.85 ECL of 512 is a +definition artifact (its 512-rung accuracy 0.88 is so high the 0.85× relative bar exceeds every +other rung); the absolute-0.5 ECL is 6144 for every cell. With all three HARD items present +(`anneal_gain_vs_iso_token_control`, `short_ctx_non_regression`, `effective_context_length_ruler`) +and the headline significant → **gate = win (§C25.3.5)**. The ledger's §C18 guard demanded and +received `confound_check={n_vars:1, iso_flop:true}` (single variable = DATA; 150.7M consumed +tokens per cell by construction). + +The other mid-training arm — context extension — remains **gated off at step 0**: the base +scores passkey 0.083 at its own 4096 trained window (threshold 0.5) → +`midtrain-context-extension` stays propose-only +(`Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_midtraining/step0_diagnostic.json`). + +## Integrity notes (adversarial audit) + +- **Iso-FLOP is exact**: both arms consume 2300 × 65,536 tokens by construction. The *loaded pools* + differ slightly (control 152.7M vs treatment 151.7M available) — ~0.7% more unique data available + to the **control**, which lost anyway → conservative. +- **mix_seed0 was killed twice (rc=143, external SIGTERM — NOT sentinel; pool stayed 70–73%, + below the 0.83 kill line) and resumed** from step-500-multiple checkpoints. Resume restores + model/optims/RNG but rebuilds a fresh shuffle permutation with no fast-forward → some windows + re-seen at the expense of unseen ones for that one treatment cell (less unique data → + conservative). A log audit flagged post-resume per-step CE deltas as "corruption"; refuted — + same-step CE compares different batches after a reshuffle, the deltas (±0.37) are within the + per-batch spread of clean uninterrupted cells (~0.3), and grad-norm/LR are continuous across + the restarts. mix_seed0 (1.8521) is not a within-arm outlier (1.8489–1.8550). **Robustness: + excluding mix_seed0 entirely, the headline holds: code_py +0.2717, CI [+0.2333, +0.3100], + significant (n=2 treatment, wide-CI warning).** +- Mix assembly is **sequence-preserving** (concatenate coherent halves; DataLoader shuffles + seq-4096 windows, never tokens) — the sibling's token-shuffle bug is explicitly fixed in-code. +- Plan §2's optional **EMA/Polyak averaging of last anneal checkpoints was NOT implemented** + (single final step-2300 checkpoint per cell evaluated). + +## Files + +`cohort_bpb.json` (7 cells: 6 arms + base, text-lm-v2 logic), `verdict.json` (Welch-t + +non-regression + §C25 gate), `c5_evidence.json`, `run_arms.sh`/`run_arms.log`, `train_anneal.py`, +per-cell `train_.log` + `checkpoint_.pt`, `sentinel.log`. diff --git a/research/ledger/runs/2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md b/research/ledger/runs/2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md new file mode 100644 index 0000000..b17ea6d --- /dev/null +++ b/research/ledger/runs/2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0.md @@ -0,0 +1,107 @@ +# 2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0 + +**Objective:** pretrain-ablation (§C13) · **Stage:** architecture · **Framework:** jax · **Status:** running · **Verdict:** none yet (in flight) + +First pretrain arm of **HybridSSM-0.2B**, the repo's first novel from-scratch model and its first +JAX/Flax build (brief `research/briefs/hybrid-attention-rethink.md`, arXiv 2606.15378). The study +question is the attention-vs-efficient-mixer composition: d=768, 24 layers in a **1:1 interleave** +(12 GQA full-attention + 12 Mamba-2-style selective-SSM layers), SwiGLU, RMSNorm, RoPE, tied +embedding on the Qwen3 151,936 vocab, chunked CE. This arm — `ssm_base_s0` — is the **base cell** of +that matrix (mixer=`ssm`, attn_every=2, NoPE off, seed 0): not yet a contrast, just the first point. +Being a novel design there is **no bit-exact oracle**; the gate is a numerical cross-check at ~1e-2 +(§C14) plus the smoke/probe chain below. + +## Status — IN FLIGHT, no verdict (this file is the launch + progress record) + +As of **2026-07-20 04:15 UTC**: step **7,480 / 20,756 (36.0%)**, 61.3M of 170.0M tokens. +No claim of any kind is supported yet. Even at completion a single arm is a **baseline datum**, not a +result: there is no comparand, n=1 seed, and no iso-FLOP match, so the §C17/§C18/§C25 gates cap this +run at `directional` at absolute best until the ladder supplies sibling arms. + +| field | value | source | +|---|---|---| +| trainer / watchdog | PID 3164922 (`train_hybrid.py`) · sentinel PID 3167084 (`--kill-at` default 0.80) | `pgrep -af`, `sentinel.log` | +| data | FineWeb-Edu sample-10BT, Qwen3-0.6B-Base tokenizer, **170,034,304 train + 300,000 val** tokens, seed 0 | `tokcache_170034304_300000_seed0_Qwen3-0.6B-Base.pt` (1,362,677,029 B), built by `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:151` | +| params | **189.1M non-embed / 305.8M total** (embed 151,936 × 768 = 116.7M, tied, counted once) | `[build]` line, `run_ssm_base_s0.log:2` | +| config as launched | seq 2048, batch 4, **20,756 steps × 8,192 tok/step**, AdamW lr 3e-3, warmup 200, ckpt_every 200, eval_every 400 | live cmdline of PID 3164922 | +| throughput | ~1,247 steps/h ≈ **2,837 tok/s** — measured over 7,080 steps in 5.68 h since the resume | `run_ssm_base_s0.log` + process start 22:34:29 | +| projected total | ≈ **16.6 h** of clean wall-clock for the full 20,756 steps; ETA **2026-07-20 ≈14:54 UTC** | derived from the above | +| memory | pool **37–41%**, trainer rss 11.6 GiB, GPU 66–69 °C / SoC 72–74 °C | `sentinel.log` heartbeats | + +### Loss trace (real data, not smoke) + +| checkpoint | val_loss | train loss | +|---|---:|---:| +| step 0 | — | 12.4317 | +| step 400 | 6.6844 | 6.4319 | +| step 1200 | 6.2845 | — | +| step 7200 (best so far) | **4.9020** | 4.6539 | +| step 7480 (latest) | — | 4.8753 | + +Step-0 loss 12.4317 vs ln(151,936) = 11.931 — a sane uniform-over-vocab init plus noise, reproduced +across all three fit probes (12.4317 / 12.4312 / 12.4312). Grad norm settled to **0.28–0.31** after an +early transient (57.7 at step 0, one 41.5 spike at step 140) and has been stable since. + +## §C5 launch evidence + +Recorded in `HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/c5_evidence.json` +(written 2026-07-20, reconstructed from the on-disk artifacts of the 2026-07-19 launch — see its +`provenance` block for which items are contemporaneous logs and which are after-the-fact attestations). + +- **§C5.0 smoke — PASS.** `train.py --smoke` on all variants (SSM / SWA-128+NoPE / 1:3-attention): + fixed-batch overfit 8.8 → ~0.003, grad norms 33 → 0.03, checkpoint save→reload exact (max|Δ|=0.0). +- **Fit probes on real data — PASS.** 15 / 12 / 30 steps (`probe.log`, `probe2.log`, `probe3.log`); + 30 steps moves 12.4312 → 8.4195. `probe_fits=true`. +- **Verify gate — PASS, but STALE.** See the gap below. +- **Sentinel armed** beside the trainer, detached; kill threshold at the module default 0.80 < the 0.85 + `safe_cuda` guard < the kernel OOM cliff. +- **Resume round-trip proven in production**, not just in smoke — the run was killed and recovered + from a mid-run checkpoint (below). + +## Incident — sentinel memory kill at step 580, and the recovery + +`sentinel_kill_step580_2026-07-19.json`, **2026-07-19T16:58:48Z**: pool usage **81.3% ≥ 0.80** +(MemAvailable 22.4 GiB of 119.7 GiB, trainer rss 17.2 GiB), GPU 58 °C / SoC 71.5 °C, +`gpu_throttling: false` — a pure memory kill, no thermal component. Root cause: the SSM +`associative_scan` plus chunked CE **under autodiff** held ~61 GB of activations at batch 8. + +Fix: **`nn.remat` on the decoder block** (`model.py:129`, `BlockR = nn.remat(Block)`) + batch 8 → 4. +Allocation **61.5 GB → 16.6 GB**, pool 81% → ~40%. Relaunched 22:34:29 resuming from the step-400 +checkpoint (`[resume] from checkpoint_ssm_base_s0.pkl at step 400`, `run_ssm_base_s0.log:36`) and clean +since — 7,080 consecutive steps with no sentinel event. + +This was a **manual** recovery behind a *config change*, i.e. the §C5 / research-loop S1-4a +"sentinel kill → NOT safe to auto-resume at the same config" path. `loop_state.auto_resumes` therefore +correctly remains 0; no automatic resume budget was consumed. Halving the batch doubled the step count +(10,378 → 20,756) at constant tokens, so the **token budget is unchanged at 170,034,304** — `budget` in +the ledger entry records the original 8×2048×10,378 plan, `launch_config` the actual 4×2048×20,756. + +## ⚠️ Open gate gap — verify.py is stale relative to the training model + +`verify.py` last ran **2026-07-19 12:52** (recorded in `BUILD_STATUS.md`; **no verify log was captured +to disk** — the PASS numbers max|Δ|=2.4e-7 for scan-vs-reference and |Δ|=4.8e-5 for chunked-vs-naive CE +exist only as that prose attestation). `model.py` was then modified at **22:27** to add `nn.remat`. +**The verify gate has not been re-run against the model that is actually training.** + +`nn.remat` is semantically identity — rematerialization trades recompute for memory and must not change +values — and the loss curve is continuous across the resume, which is weak corroboration. But "must +not" is not "verified on this box". **Re-run `verify.py`, capture `verify.log`, before this arm is +scored.** It is GPU work and §C4.5 forbids co-running it beside the live trainer, so it waits for the +arm to finish. + +## Deviations from the design doc, recorded + +`ARCHITECTURE.md` specifies **seq_len 4096** and **Muon(2D) + AdamW(1D)**; this arm runs **seq 2048** +with **plain AdamW** (`muon_jax.py` is not written yet). Neither is a defect — but every arm in the +ladder must share an optimizer and context length for the comparison to hold, so unless the ladder is +re-based on Muon later, **AdamW @ seq 2048 is now the ladder's baseline recipe**, and this arm defines +it. The doc also estimated ~146M non-embed against the built 189.1M. + +## Files + +`HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/`: `model.py`, `ssm.py`, +`train_hybrid.py` (byte-identical to `train.py`), `verify.py`, `ARCHITECTURE.md`, `BUILD_STATUS.md`, +`c5_evidence.json`, `run_ssm_base_s0.log`, `probe{,2,3}.log`, `sentinel.log`, +`sentinel_kill_step580_2026-07-19.json`, `checkpoint_ssm_base_s0.pkl` (3.67 GB, rolling), +`checkpoint.pkl` (66 MB, smoke). Not yet written: `verdict.json`, `verify.log`, the +`arm_ssm_base_s0.done` marker, `muon_jax.py`. From 66f295378e41ed5b98a533d5bdf99653e300e327 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Wed, 22 Jul 2026 23:48:34 +0100 Subject: [PATCH 08/35] Rebuild the qwen3-0.6b-study arXiv package to match the 2026-07-20 sections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found the upload artifact stale: arxiv_package.tar.gz and the .bbl were 2026-07-07, while the sections + PDF were rebuilt 2026-07-20 — so the packaged tarball no longer matched the paper, and state.json still claimed phase 3 while Phase-6 artifacts (SUBMISSION.md, ledger papers[]=packaged) already existed. Per the 2026-07-22 decision (batch 7 Q3): rebuild the package; the human still does the actual arXiv upload (§C16, nothing auto-submits). - Rebuilt PDF + .bbl with tectonic from the current sources: 34 pages, 26 refs. - Verified arXiv-safe by a CLEAN-ROOM compile (extract the tarball, build with the shipped .bbl as arXiv AutoTeX does): rc=0, 34 pages, 0 unresolved [?] citations, References section renders. - Repacked arxiv_package.tar.gz (36 files) per SUBMISSION.md's recipe. - Fixed state.json to phase 6 (packaged — awaiting human submission), updated 2026-07-22, with the rebuild note. - Scrub check: the paper sections carry no leaked LLM meta-commentary (the meta-commentary the audit flagged is in the strategy docs, a separate item). Remaining is HUMAN-only: the arXiv upload using the metadata block in SUBMISSION.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwen3-0.6b-study/arxiv_package.tar.gz | Bin 167423 -> 168005 bytes .../qwen3-0.6b-study/qwen3-0.6b-study.pdf | Bin 356268 -> 356191 bytes research/papers/qwen3-0.6b-study/state.json | 11 ++++++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/research/papers/qwen3-0.6b-study/arxiv_package.tar.gz b/research/papers/qwen3-0.6b-study/arxiv_package.tar.gz index 5d53b8f7d41aad63c79f3506080f274aa61f7ecc..82c5b7cbd4169183c4e8d89bc517f6bafe9359f6 100644 GIT binary patch delta 167685 zcmV(>K-j{{D$+enh_XMIIQO!ST%BT}Mn_ISg)LESA; zvRY*O)^zj~fFe<50Yz5>lC}Pg z1u`iAY96iswtwPFF49miow9*=&w6qu+GN9sk!|BPU-INO^Fi zaqH_pE5qE1DGw(w^h?6cSf;N!GW|5Kj4sAHOhX;Xbj((N9vrn^l+j*egAUVqHqA3O z(g1-dFMIZII!i<4+33ZmxlC~zZn#@IH|TttlPPzi8vUFDO5d=+Cx@TnF!_wb-C zl3+3Bv47#yScaDU3(I&wfc^)D9D8Fo%VP#wPhi_F2Hoe=OzM;d4Yh)g|B*pMZ)|zY z8?tK_BksvSN$uuWuvNfb7#XfHo@cQ(twM0qf*o=In<`K^<=IkjmxAY6V?WJt6?nE7 z<)oMjqhvawy_XC+> z?DcZx_#?s;k+njj?f0YhFcsIKPLMdF`Ebgy5I64K{0eDQN4euvEK=6ZG9v>J2?FN+F;1sa`WZqP(=y>z%WKK~#vDJ*1+rfG;UCas)PO9p3OTV?R~Zeveh7l(=|$ zGJls8=RcUyQ#W><*Y4&YP%tsHjZ-b3y6yl@y_*j+9r0o~{AC-7ZJ274aAoDKK$wl1 z$5}SDFW1+{C?I*zhJp364zKU!Qbk8bC;brVJ&f;wfuX~L6j)Sa7k7=@w%QN{zT&sW?(W8gY}HK#ez9we&f-gHJ$?4bxUCcvgpnt!mVt1hh6-h-kr2Bb}!an|9Wxn~!HtYklY*jPSh8zzdFuCsS>uw;Y8k zfojXw*v`iK&IU{5&0~X!jGmhEi#9u#QDe$~X)55>zp%so9_msQLHBupiJQ!Y^l>Oh z*`t}l@?YDme>7k-3kj-QN4+>3GtsY-pzF9X&waAxOM}x=yK&Vx2guQ!U7VLT#^UTfAA((|cAIBMg zTP^RvZy-~TbMZvfqz!Xxn}~Bp!L@_>)T6r;aFKEMcQZ5{P_4H(#a6ElfBO0L)#1VK zV_gPP$*klnENr*yB3GXJBP)S{7!l@InM{enwH2a_KgG&Ze`E!L3|0Qv2fNe_Ri~a- zT;G&6sbMO$7<W(y{1;SuhI#G{-w6Djv? zn?m=MlHzMLUjkW3m}hNNrG|T)9-LQey<0vxr_rc5QgK=o%m3@%`k&$R<>CH+55oJs z$N1y@pRMOx&)xpd*2d=c_Qn>DcebAI?Ee4$&;Q1!lzyOGlTdf_&0TPexC*sX>-Lnk zT@(-QenD&gr_K{sKsmdUL#mirMi-H0yqVGl&j>YYC`UKL(55^QB(oi@iMIaGkmTA8 z6!jy+5t;}xjPI8WR)<1a@p~74`m6lX#zsR#4Xd%{Cb=8_j9Csxa>Zp4$p7S*u1u|x zD9hPBSMAl!jm@oR<*_x^6QVVMM4ys)xj)tEc#VA&i7I$%C$gXm0`uE{rPK;w4y@19 zam?kXIrkRG+V~5&N-_MpJK_jY5rV>u@q`C%!k|*%VQmhh;qDt##q&`e$X7n zvKqSW`mI3eaKeti@Ry6$&?j;QDF*GDt4y}OEOqhYy8hA6A1HjbebCXT=CIJD6%($I(;XM;XXcp@3a~L5wtj3rcKbq zC!2RGJ)H+Bzu%*_+ONEuqUJ8ve@M8jop8(pOiA5iwHX%Ea0-SzMo)~Xfm)@50Fi4H z;DwXAX=Hx;yKOq_q)5dCH$UmL?il$I_ZcH>(vgP6~n?(_gBXqJ9u`&@~j z;oJ8)Pg{MguX6H#zWVL&RIbr!G#xHdSZTWbBp>UA?@ze?47uV%uwNLd>Ok-e6w2#> z>}xbJuvG>I9zisWei$AQFyYoksR`=757SG?Vg!hS#>OO>0y&0Yb4^(*6x4u*tZpOp zh;dvChtrM&7@5=dtYej<&Cb`*W@lrwy|IZJxkQ_Nn$CiMHJxX4V2usX%u?7+^OSN2 zPhG}(qvT9T!x2P43-sX7Q7&5@M%Ua#)&V@?E`DN4R5O@b!X2KGb1rj3tTMS6lk+dp z{}D3JV#WCgd_we1LH5q2jVF@49@EXohVH&9B6k;`DZ1-YU(?SsF0G3+pK&AU>UWqO z9T`ZaL`LR+`VR8@d6K}bnB}G7Q}oA~x1ZfgnmIQqb=oJw1US`tt1W&V=k)&t(y4Z2 zyhLcQ@`1SJaL<5KZr$TSZbLucJ+Am?Ng0`b^*MSmVVqTK1)7mDOHV2Ob#lDSNQDZ6K-#7n4asHLt#V>yzVQ3#ARWsLhfaZHWLV~AdF&- z-|#Li1m(K~^Z}oV2C9*1CaIOh>hW|xHLqx{SuBIc~(?Lzh14LTawCrPbO=`Y|D0{$9vZOzUQiO3u)n=|g ztjUV9U{Dj5i-Ywg1Z+cX3US0$i_X_ni_XUL_Qn<(st46VUq>;?yhX{_d{!R?3?K(6 z>IY(`9ECK>m&V?V`mF)`01L7vobSqp$42Pw*rlEdqB)O(+}E8h zR0_@ewdI6|+|kn9l3z6uN@aW6veho`g?6YO0<6DbiDy0#Lhu z2TqirLsB$E@Ctl=5nzGH$rQdCh?l5_Xl6=0o*`3Xcn3=q4$S-kILqFDY=O~k5My>? z;=x*4t1$aP-aFV{Xi9k%cHbhWMw2|@W(me5v=XU`63!QJ5I|m|7ixRH~e2NaCK_pZ~NeR>w>ZA}FohEp*bmD!=Mz}aSs0Q}=JPc*C4C6dHTFpn!=>Y7E4j=1-MrWpI07k(#U^J|LS z=EgQ!qODG6xyb2%AP~zv5gmb7H8{dQ=gnTDRLLZWONF$+6t!DH}1mcL1BihvuAcyTY zD3;DfyR+TdeX$hGVE`|Q(vT>hq?GRf~&j)f)xRbAH{@pY27Ixk7CO_gW? zy%K^932NGWI*A_Lb?6Tzm|MW^*DPSyHRwB=n@bBo=8Q}cPV5pQ>!YsYPA#D?0tzz9 zYDQCz*yF%|Nmll#Bp}>AphH}S0)f&OWuNA8Wr)lsf?Kpw1iS+&W2}YLEQHR;3}tF z?E4FJ#XWBC_u(72CjkFS=U=o9fuc?j{B60cwWD zu!8)W=+G5!WNqsj5G0wtOmte%Er-eqw+G{P9gs*0P6%~ikRv02gK|~0#6|i!#jFVk z^^$9Uh16Jl*JN>SY9Aabfy%bzi2;b^xo~W1WUgSbkgu0A+K^^)vVP{ENNf*lIr`yn|pdpMoXu5bAw{Ty>p#Q8!kqE zCNa2hG{%fcrg?`0;FJ4zZRdEDQ>oRm^LrqF5@i$@geQ|FJ#o!5(Q@xSaP@S%_brug ztKHez+1XvjyOkntfPMR!JL04qZQDt|`GUxr!l{snp#Jrg+MJIuFKJEU8JZ1pa|~RO z0WxKBCCuFl)uwUll9pE4rngkaeh8%699py*sooi=hdjWEdfX#HD#nfe0fRTDK6hHE;Ju_ERqK%DS}@iJLE_Y}=|g_wE-Tjtc}aE0&;7kjjb6mr z-GARB@VGU5c@yWlx&SR@Ixb}81X%Q*APg2IV6@+>@@}EO%B=m7?!C@SCtc8gTD_-e zao%0Wx%O6sFKxJzoJVUbXspVGl)SmN+Jk(l#(9UnWIv}fx6rWz! z%T3?_t4G@3lfHewKwm?W0V8ySz*ShEVvb{N-6Jm(xo1gd@U^#ABBk$ufL1%pTq9f1 z%wH=cozOLuhFkw!Fr)f#tf8jeCUy-pggGaB)x)T_+)DKs9c)N7V!~;E3(o5v|6q0I zyf}`j%YG9R^|B1IM!UVlo}=@jYF+3&pnYjg>xZ18oiMC~8fuT}W1|JV_Cjr7U#o~f zKz)i!%9#~z=jaUDjT@gkCls%JZW8hrZ6CXRHhQbAWP9v7)K7CIDcYo`r~*L=K>iw0 zIDor-UylJ-MrnV!1_PjfZc0Q@lkwl|y$4`qW!W}tL-AmtiXg~A!tRXBOr}o?Alq3; zc9-nJf&{WBnUiET)0i`pY*?~_*bq=a1*wX3P!L6>*%3jcD!r(nsDLP9r}#g2dCEC6 zlU?|L_j|veysvCB<(%g{<*wI#-D+D1(O#(W=71<}txzvas9myus16`s+$+MyKy^<$ zX8(pa?r7T*4A0?l-arG50&XIh_re7DnZDgLz-Fb|R{OaS{N=p`aa)&n7zU&<=%crq zIGm785NFTaah0;(c865TVtUGI`}Ly_Rt87i+=(UOHSu;N<#$L!?#qSawZ^a`w)%!aD z2`K2@Jqh_o`VKNi_~er zym<|Gfjk#0!G9GZ4bF#fU|s(q*&ks2;-rZ6a2#;}#B8X469fCp$rpSFN5^{P5fXk! zRXm1}`E-~T7KHC?o~XVCMqTv5TH!$aQ0OW;VQyKNg?!Fr-d7$rzFn>2ZCDahM-P;p zBaVSQfne7Z`5g8&@>7K||8YD>xa;GN_z0g%Q<^X2vl)dD$^G%yRB3}E&fP($!x z$S+~_0mj3r5TQX|la~;yAdZ-ErB$T&E9T&r(G`2d=AJ-LQv4yFwC38{nTPL6HTp7= z>WO$Po=JBn0*~JaJbyzzG(6J*yp<>+fe|_~6Z)WkP^wq}V=F_+FQ{ijPYmY7Ez5Zj zU!Il^zuGMWf}xM&wc|o4!VOl5c@m#l;V7d8ZnoNWHwX!Vfm;sP2qOe;?0ahS-gYe! zJyJM3(PCjj8yZkTG6FIjopSN%Dj-)UU(OT?S0owFb*21MSb>Oop;(#3LGIhA*WH?! z%|tGLET%}k2`>O5IZ6f_wR`Q$~ee2p#wSaF*a5paM)q%OX@v%0o>J+@MdSMc|u`M zxCP2s`o0dB9kFV76?1HT{9S;|g9Qm&e*n>c+fip^M7Xzhp*$*;M>y+a^>LO4 z26pS^;l2(om@m|1iR?ZO7%UX33M`d{M;1UwEUH6F6|~EehFd#- zqESExtWb7Q`!Ag=;AK$w<3a3q#F?N>Sglq->{tQN@&x6m(Z~B6H3AJl3Tj2=$T9-( z#IHh=Vmf>P43268BgZKH>qm+wnpj&A&Z}EPCoZ3NDKjn%k`n%cm-Z>d#gOI7gWeQZ z7?%*jeVw&Jg%oN>77klqDV8d*1j!kHYn3bXGOYw;uH|xo#xvHWNh2zy3;M|AHG*W) zgt$lDk*g(qEp%pJ$^OT$pQ#>xvZ-k$iVBx z#q5l+#ETtQ`0wUI8&PUrbYUpvUAL+hU)T)ej@$zCip6QjYv`>&IV;?Na!=5IUG>h` zR~$@XxX29>FcS(6<*y0!68_i{)9CmCrZ|PfTN$gI!!!uSh+7h9ZZP2G0=IOM8 zdE(ZpjX99z)WbpWVi3Bx;h;@|F0K;hB#%`F*u5T5pH8_hLl*GfC>P;>!)Lv^jp_wH zFSHSdD%_W8U*<9yKWY*D9-oCpj!<8LqY6u09sC0K0qra)W%|iw+#j%C;GwPZCfzzB zWc7AqDUT+q-n)mb7dp6tsM{)Sy&22rh>bU<8^XYiDkt;Fv+s}#vd?@uGAax!6ICy^ODEXeuFP<aY4&ogK(Ws8?sU^S10Vv{I3C8?hd<%3v0`J! zGC>nQqM?q!amFA6?gpsPRH@~t$UGk7dd6adF zFUZZ8@39w^$Du?j@5AL|(yUggJGE(B6+pUTgXQxW<9zAavve%p-Iet_fd@I>6hs!( z6baj%Uo&A^!W9O8hxtJ7$O(WqKqk6^(3mBa5&>vr(i$EZAP!z*VjMiBNxB$ZSQ)&+ zZ1wzZEP=#@5m>U2@(E&CWxClR1&q|= zJ;j|4#=sTd86XKNl4VZ9L3#+$)Nl{fFc&K|sbWUO73%;Ps*Ey(03+Wu7jP)>A+#J{ zv0S_mN7+Pw+V9I?mdR8*hd7)uu@8}{7)s0(RI28X21Ma1(su#Rds4lLGfNZ0_Aw+> zmbe751Gija%<1~88tNX0@YUpvE9y#pwC;pVugI^$I2OtT4`6^IoUG7!BHR)U%x3eL zDQMGR)ygA5)r0y2>$2uR4Oz~o{ zq@=^g!gLC~G2f-w2w!e~p^WJlTm&I2d8+4ErZCTYhzy|66Jkgf@pCFNwXT6nv|zlk z7BfSf-kC0FD}FIT1Zm)lUFF;?b~z=suP{$w6~(_0@x;a4mvTZFQFS1*$n&`$;SCQ_ ze_-r?YgL2(AlR0A4^E!?q&S-LcT8qgaN^;Qnk&znIBS@s!W#?0#T#RP_XvPWlWd8xE0SI$?uAS{$Fg-=)-+IP|2LaR0zx!rP+AlNu%jp?H-Kz*F8pGYGxpGX5&Q zvxLm0!a9{G6SfnM6E+*fuE4_UlX0v9&y~=Bd&31B7w*D?VF0+6xcd%r=^ z6(S!DKZnN{w<`RG0vL1$_%${Xsz90aco{=#HMd%+)nASsl@Y2q)#XP)9paBi9?~#> z4qrnS_tN@KA)vhW|762^VhM?hS*R3Is39QEKo4gL@5AE^;!PFAMH@Q6tH z)u5|uNe5dAcFby}yH=y>n#NFR-lue^BycNNrHnO zd;>na7)TipBu)bCK7_k0rqpI-TrIzUn1nG3yf$#V(J%Wts049_#L^#8z+a&NLRj?p zR7+r=V^Djg)DU1o%g z@O{OtgcE)UE-T-jmH~70xL-+TITe)}hs()l7jH)4gx~`m;zSQ9l(oq3BXI(MCh!JY zwnHium*(X|!GSkj9k2ivik0jUlHRU}F+_0CB&VBEloaa9DrA+Z#453AfRZB0}Lus2=YoE2Aii70aLle8)5 zQs5oJ_)BfxEeYbhq(vxnX3intHb)d{zstf>#Z9HEqsB~o31(nH)Xly!E?WdKfrkI*gr zJe2A1d#zHb&sg&d4r4m-(*uQ(iu#J*>#5Z35+r%OBN|Q(U{b=FwTTOV$_(gtzZBEE z8&W*Ebe0AW#1O$6_t*I$#E<0Zcs1{_5&nlvDwUD^pUFfj^@eyNDV(O)`5#`3pM6RN zZ^UoBgPcN!F?%n2JsCKwlz}sV1`e{GF+HC6&^$6k@t7-np8$P^$xknsRzJO>4S&V} zvLQf!i2Dmcihn5kpZ_v{Tmbb7dI9{Gb_DPR(-)u=qeJF4{UQAU{tI0K4s3&0Ab(}Y zfYY1g8<2Z!xCinDV_pmwLH(_~1oeuv>E#RTEAXFekE*|!_${Cg(1o{u8^U;M+>R);h>IHNpD8%D?80Ai|hlm~p=>f5S&i!s#{!U&6qnMOq z!GBBNg5uoDy-S zhxp)5{Ht<5I5aIi5dM>#5JpyLFN9yvj)>n6!9xhNoNxr;^x_N5!3!^Grvi^_ZDhPK z>8=nGzpxJ8MXXr+evAeMWKN&}Bsdw_;AH4Bjh8+deM>fBGV~4fWcUy=zNuJN zsr8!}m;Q}lTmnz8u{L~eCEP{{LORQQG8E{>hq>AK22wD8nOAAf^bL$TYmE}|&7KB< ztR@i1LIi;f_&^|mDccyxBrqF^Twpc~h{&rm8^#UvY_uxy@J)p(Iat6Y2g|wSpbtw9 zWE(m#*M7+L%-jn`Zm~S@@R2By_SgV2= ztO3ekt&kb4MKXgG+?*=KCKYO*v$kkGf>ug(aLKDFW)j6?OX+rf4 zj0yE6NPQE!r0+ocAej?g$~jSEN&|CdX3c`n8V^3J&W^rH;$7cBKdUcD5ZF7FI@Aw} z6{k2rAI1TenQ?%?%kN>l^6D_fx|8m->Kt22tA*gQ%?^DcBA` zqBiq?IQ46yHvY`MK-0EB)3i0ody{~UC6bSWjtzTBLkCG!Y0vtZ=-5pnGCD~`N5|7- zq6==qRB#h&TDVu~!08+4RcJ!<_*JUV0gJ`!tKDIQYqvtqtTvD{iwQZi3Xn4k2|0ts z??Iz22pA$CvK?lKv;o3~Yy}%)7u!G%No1*$5G5KQUZvI2H_)rqL_p$OtrbpRza|^- z+dHb%cm=)u0?0jOXcWtBBsrFxKWP235J?13V+oMHF*QQBby+JzFi48 z!Br9`I6sIJTx{S357Icn0RtzvNt382C4VLap(Q@?rLW4orJEFlmNzh5&H(4MZ&85Y zxUXLl9K&BiGJ?@X1jpdcL>Dl@@hTs|ajV4$tcBsd%$vn~F&VB|JW%;_6Uj9nnig)9 zpXgc{BtEX|P4KrQ-?iPucTJ{Tkn-BJrJhd`GD&{wS1rjTZxG0WZfgZ@m)FAB#Un<(q*C3s#iYYn*oDHn-6m0|lt(Y75A%9Hr27JlS z-oX1ymNrTA*jHsl1Eh*lUfUp(1JAIJ%0V3#$6#}y`}qhRXhr=MtUQhpUF3?dKt+_z z6;ZrlHMN5&$cWv+96?PSQBCkbJ5n<|&?gni1K-}{d63$`NAs|0(KBRSc$EU~`Ua+~ zY$hh~NlZU&z(6AUVW17@1Ap3pRR(Rq0!bUNLemDc$V$6qBMt2+!UHzJnDc;^PthLG zT50A3Z7464wwntlP`U6-ey|D3Ns<)2S_y1*13lpy-RxIP^HwZJknC~*$*v@jEVS($ z@S#5YB2N6sHGcrWs?tm@V}IgPuZl!cv)of) zYC0DhPpyn&`MX9{OS77Q)nwujvYJggAGlh2dJ6=S-B3V3Y}Oi5KA_*z`gif1eksmkADK)z?azc)!q7C4_Tr*(4s_DhaPl^Ds(&y9D|_N#!6mWRq{4j zKf0cmJv&v>+e-COj$fDck^$wb>{VaW-lV_2m$!^r;wyJ(u*SEwQVZ(eO;~AJllv+; znfeB%&{ME^ztjU~vWw<(guHoj^+rJ)~b&tlr2BS6iW7~I0-UZ zc8Fib`^iE#F@GWkB%ZHoL=0|Vq%MQp;H|M+2btX3E0J3p8{Mx4#MBDr)(%X%<`^eq z)K4?cN;1GSF8!WY7#7@&4t8T`aj(&o0tJL9BzXbC-Nb)w)a1x$547iABwS4}R;8)&)M zntzSZCmR*b;JhZ9!Ffb8xKz>%D!Wh744SyZCKS~sF&=Ij5)g?9Uri(+-ayZrBG>ax zRT4?|?4-Sz`J(&}tZ4Yu&Lbv17GA+PWvghW89+5y`B;scw0z2e1W>2}fSxw%)q+Wd z7EUU$^hu6Be92GKhcEe=`lOUJ$ofbintyzKP4Yv-^8K8BG%`)w$Kt@x+^3aOQBn69 z!;tKKtU3(>zt-NfN#iH&rqZ-SJbpfL$W(r`h6bCTt=aqxJ4q7y(WC*VA2a&VY-mzH z{*KA(N0Kr~?MJg=vite;G}HSj!KE4gjM*WQzorFYoTcON0F|ENnS2>MEl)*3JS|oKv@p72(u~%L#8; z$mzVrjdLR6bAYbK(gsi57AV6S(8S6UIuqzGz{o>}3g7;h5*xorS{QyEIZ#0jaE}h zF(>P+j-v(_y2ZLP2#vC8rS-+iSfTFC93@0xJQnY;9h~kucf=VLPf;Fm!MmjwM#QjP zYR~3|EpdfPV~nB?qm81msDH)6s5>$}A|@sp^)*V15Em#c3)gE!tWqIwyze!{0}2fc z>nwD0wBDfCK}ED7G{H^dBZDL7SKJ#^l3eg5^Am9~34k(ydCv>Yg9}2#aY+AG%Ef7G z3frN>tcq2slwG2cgiGfOqoYWRtsZr8-|`h6?Kd2V(7c(!-D$jHVt*`Gd_k-ZxOrmn z3gE4ad2`*U=Zas&@_AA5z|rApVfBUPub3Yej=oqRf=)Vix40N=htWbFmcl~UbP_O^ zVojzh;;ouE6;6!U+To7E*BW!`cpeaOBIbySo%qbm^(>l@XbqYz70OX@XsUKcG?9yS z5ANO}9=2G><2PVrRe#|`Lfl-q)WWYRLmYxu&xIsPy<*|hh+p#sVKOu@1P?r*Vlkk= zg_Lq#OhJQ+k;QLf7Z(wkZi^KxPOHVYu)S5`Nbwx>K{!alb+g4J*-%QEZxqRUJ5r1p zoZfX}5O`KVT)1|*0`os+Q9unAtQSA-z#c3W8l@;P(U!{)r+;P%Ufyxf|LnNgnt-b)xqnd(1R(2k->JA1M9Vy46IlV7Y8X};V!F{Jesa(%^fR%*(z=i zBT}M4NJjGV=$PSVvqOBwU``@dUd#x}Au-EhPV+8oh%l{%q?{N3bITsnV!}1@Hh_l! z?I9k~tw!VGGR|1JBBbU>4C#mQ@o{ep0oVZN9{VKKDt~Lm-R25iFeP?WP53i9;sDsP{^mfg7{`(&9bj|4Ago$8y5{%O=v9*X_U8;%AUI!20>L z{!n8K>wiBpSagQn;(GCR4{wPX%R-9=HKS+PQY5`e$9wfgei~#37&hVzf{%-~k~o)D zur)_4O-9Z|meED|hCMhUluoq;&pN2Zg$w%nqflx(36~U?c1T!RUSn8XN`5)e{VIY` zn}I~MAG@6{S=ryUzx-%()82@M?lB}~_%m&qtbfMrrC_`Xo#3J5w#7IJDvD%m!-b*{ zRXwOTApAAzNNWOwb_?^9;M7%Rh?@%7;-UE2Rmqs4EJh-h=$>z*{_=d*V5L?VD~LG~ zayFG+WrY?O#>53T3<@i8{b*sV2Zs zWPc3y0~iuQFXYLA;72XF!u-xxz*ZGc8?zTpk}(t|wJ}n33MI7mN?5cEI>^!{lPygq z&$++XM>VJd-VqXTkv{ax94Zei9FX&F` z$weF!MtRjEjVUYyyU9WTZEl5Dfq6vBS9gYcywN%=9IP}N1*G!HGO&Xm26FeA-oVUA z6J44to^6@|4QZazk!EephA6F=Qw=M#S(*t^Z6*l8I_9>v6l`rNfZb%NK?q|U9e)!< z%fju2eO7cw>+l9ODFB=QS#8m5_hn0Amw^=7%~9?(?=S3e5#t- z0)TgkwgqCvVcPNF8UikYrW(W_h2TL~YZM#~J_`UWADb2nUa3>etH9nVeF_Y#iKr*G zIdsjQ5EF#^#C%vWeqn0Yfk8w0L4WK5!L#rob=QggTA=Yv!G<0|PDEjhyKqbC_gS#3 zh0`EsPweJuRm`$Dn4Vra>`m*{4L2K33%rA}g=}NkJGf!tJ=oYqm)9$q&8A`ryHqGQ z;Gl)TIijT>Zsf-RQ(q`sf&DlRmTt+L3C93i2RCOM8fL!W5iKZM)Oa@GWPg&pgp1ZK zcE6R#?B!j|wL~Qwnb>fi%?%9hmkd=v|0sx46v)6=K1l&xXXUq}pb~dS+2J53<^Nlj zq_Y3S48YD4zrW%Uz>V2|iFh#nmr10PuiJmG#gDT87O3?%O+URWr1eMVLfS9l7#U$J zaZprvOPp%Y3Uv&YoOLpU1%LXXDS)N*jEYvVk`Z?(RqHl{!i4mx)GH(40S7f33}hxu zo+<{}@_aGFz~Z=Ka(4bLfsMloE&@E@ey|j=tDU{dtY_&Tf zq9+xcA^T-qi!SQvtHQNrU<~>tsP}+Di(6TwGco8BGz5iSl$M(@QV-;JVhv;=GCZ6(XSF%8ESGx)l2uo99)ZbM@3v zx>qb#rXsMJX-X{PO$ZbB*vg56fIk~u5P!vZQIzZz9rJ|41b^FxOauxM@@It2J_UvW zTb0DTS0U!T2w1u`jMkQAbu-OQKqM3)TU$e+P$-W8r8j(>ttJ@=)h-g+y5g*lV}}88 zXJHTG@|1xb9DK;p1|B*j{T_jsC9T4Waq&VlA&lGNB)g~9Xt7eM!I3U_ViOmqWA-w) zfHR^d%?Q*&?0+kc68&WQh|bK>*?jQc*`Ln=g_(gh@ zAoSWr{~~sv)Y}fS8O)HyP@ouF(Iai-tb;IAMM!USAb&wX3h4)Wp~oJt6!TIp<47Xf z7Kx~|6&oU^(5I{dWw^X4r;4>hD1A{s$Y%j-tl`+CMqY3 zc``)hey6RwLkHLsJ>QZ4H1O_3RG2+Tp zH0diaa|t`2>CX0ZXcb1DTpyVc5(6y}&+Zt6c!$?vXEWmUOe!m*pdEX4*ojmEKAe+Z z5uS)xuv%qZhj^zyEI*1#Th0vH=gc&E+!Ith!q-imwEjXe%9MRk&p9a@Zs`EUC5it{q%LqogFqypR zQV~d2@*LiadSVsPW{;BXUNA{enG8(M`=35SR7M2jI85FXDLsWMJ)p-$0D^;vHm z^MCnM0IC4Dp_4E7lE#`4Rh226sa6?!AGQVazQ&r+CSc!-){^Wk&L@jG#mJ_3w#j6;$pa;i zVXQF86Nd3i+|uxHt_ST$3T0s+!l#9=(P%0D8Dfb#2tEO}BU!KJZqkblnp+|^V=t%h z-HPe@cm-B5@AAov=qXppaU(CDi>J~$FK%OAJey5*V_rO$PQ_d0#UmyffVPUb+^AD) z6)a@11P4g-6lerNhH0Rnvw=??roDnLN)eNhJ02lNouVW`)rVXh`>yV|;Rv-lWFfpB zkXN`iej}caM&Xz4lh8XNe?*aACoxy4h|>m$OMILtoIGm1D%H zw#8IoqJWz-X>J9dGbnC5C44EN&^n@+BhPi<9x=8Byq?Tta9apdEzx>gAV$FeikPP& z2Gu5p001%Jjsl(iaAO)m=(ula>Bg`S0dNVzknjo!s%n%WHBtm6f3pZ`t-W#aS+5@R z9TC_zq*Xlkhz2H_k2_*S!V{}HBNHsh%QCd!*=bNM-C4$t2uYs@pU`zi#xX;|;58M+ zryCeycgnYVPaylpzv+L9g4+sd(>F~Z<}^H&4g0I zbpY6lhn{k#1Lsj51k6A+KtT%IyfPZane*paV5NlpJmG>>1aVVjkqX5ECaX_L>+5!sZx&B=$FmVzG$|6bWIOL7#*PBO?j&x$-hS+7{$P zi^0aC>r8UA*%TnWrg-4c87j&%j_5d$4q_>4m1$Tve_P)ia?TT zsyI%He;X?z-W_ouNuXpH2J;;3b6zVzVH0K{+?FaLG>Yq60)BGLM%3%;olGt}>^L)q zRSdd=E-6#=1VbLFfkchj?Sqm*Gte19Rb!&tNR6xPHzIQ-o(B{8>)F63&r$dem^7(VgB!|F&d>OU? zBCi#!n@>{Rg)LR*85453QOu)c9mbxBKJi}yajtzWaqK1wj*MG~?*v3ChBG0YItj`5 ze-_@6#Bi4@>iI`9gG2H>@+6484Z(c~XOmp@WcUD{KNtd54xBpGP%#YiEN>xJb8rZb zT+9;(uwEIXxb>8*B4?+NEH6FS_9OI1Y<2ukoUDfTKal@T#8cw;VE#9gem(y8TKwqv zpIU!kC`J` zU0bBC2o*OIwm>X^EX})BK+cu%5gGCVX*h(u)Z8M;&$3X_wS8lKbtE7N;405-qa)>X-mN+aDf8dl#RlpAgbIUaVC-u^*{k!5Aa*);9Qz>2!+xqD(xKJgmiN(#fX5gntzV zz)b(M;ktU|!j_dK__jEwI%k=e)g`vBE@=xQh}p~Fp;2Oqt7uak?7+zanXrta<8vZ5 z6`F#Taz4@mf?(wl6mZ#A{SIP!x5|Wdbgza7stHXy|$cJ{DihPt{jg*Z$p;9I21iy>Y z%_1O-X_Sgt5mN;Y9)z!i*wfQJ(8mZ{;ifcdpm>}h^cgTBJp>mS7EZfe=6@7_osq@_ z0w{CD(Fc{az`KmFnW;LM$KR@ z0HZ|gCZsxIRAR!$a0n<0QwZJ*JZHJCZDB||0J)|(_+SfD{=>;LWjR5G%XtewFQaLo zuo?at>~LVw4oH}=qLEr609 zE3=DrB*Z?axy+b$hYWn0BNuwEMuJ;7dh`P94hq7;13)SQ-91i4pb^?^*A>M9CM35P z3`J4&f+AsP5s9T+z?BR%SWG-3#)t9)I6E=f%rmcJ)Vh`z9CRV%ED5ikUHq6?LI98_ zk}81+SAn>*B}9KDo_|Y5AW_N-h^9p-5y^;`yRvEHtuPgXmLZHvS4!$Dsi-Tjl(<47 z*B!~GyUZ)}7>SiwF4MH_B9@FsC1WX-<~2@=Y3ylrY)_3>JXgV##@%U4$U`mPVLySz zNVDxqkEB{F5RM;eH=Gd|#+o&TqMxdi=guXefv80)xPmT=jaz5(b2W2zZKyNOOmfhG4La z1BVr8hE~YJNKl_kG20}7u2O0W!{EIkq@AAUeb(i%(Cm!vP*v z)g1xzmnuJ%;eSQ*f?t>6Gz}KS#NsSrH?Y!FfY(ALNMc{?F%N2r#S~&Pnm<%>K{#-f z5`)r#s4(T?4fAfpt@bhOl+7$v{w+Q=m{4S`gTQ+Ey4E zIW87f!>(A-r^{f;JBW8T$H&DAt8>prnuP*PoSGP0RHt0WGR>se~k zlORSJe`ofd6-^v{+C2Lk;&-#-*|%fU$U>q|6Yq975`f; z%ifmYxm$pO5_ta?b5?1A6vk$JP>Ep6l2)fdp@MqYZ+t?RL1NTUOeB=7RuY+%Nl#0G zj0P!3-8fg^wGg9bVW7Y+<1dh7!E zk1THdXg;OJgu^duq1OT{iA_Eu}m^zJGv%P zKY&aHhLQ7Tqgl%co0k}F;^?T*OTwvk6r60*U1yJjy~bG}9&5~AD3^%lrq$k-q&+G% z#+hAAmKzCOS&+pkN_wi8J)qVaDo17df2Z6iRi|gv;yE0I<)cp1jArx<0ZnGjY_q*R zO}|8B9vh8-mWk|`aOKzaw9B~q@j3El0q_lOSFv9<3XoAj03WV~NoH!YEfWiw!ub;aUo$thKrf2AW) zNZ}JP5k4Ik0FU%F#DnSLu~a{mS+t5 zZPXanz|&O*-&~SZl?f~gs14%1(Nd%3&V>i@ROFHk<*HL3_e+$du5kwmtpk6=renwt z8ToheKNYN~^@C1?$;>4MIc)0<^)1{Op!=uWckH z8W8zuN+yJxPP@uLi&H#Y08F@2Kg{*R*W^@rZ)?Y()bNw`NH7VjR;rDn!)T4=lP^gh zf882qMOquYW-TM=+|p(xB}!PxY=_*nJOe`WjyjW-S`%EWR|zO_6vcjaU4;ji0vjU; zAdUk7%oZUd^uL2=6U+V7*_3<22aW~5M!ZmRAod_u5DhU93Vcrd=PgC&Z9wdZCil!d zZCslPy$AC(Ap<3D3u6%)l7XJTVn?XCe{1QfqDFD70eclDAc`eKZ%=Z6gD8tD$sj<>_X(-(BpcD7B zJ+Nrr!u~}ZLQxt=hzKO+imrb=v5B(6SPE0Wm^T z3@QSV4QX&8|LVBKJa-I|2mbSx=JQ6A)gn|v5ijH98B;9t5#+Wg;z3tf#j!+M8pc9o z!pV@_J~l@sbvQMbp@J6`H7x?MHx~Da2jBQfqh%zEkG2C0MaWBaWqDy`e{dQ2Vu}aP zBgEwtyE#>u%Z^km!{79uK-PJ@WGG)pN67+2q(h0)Uzk$|-Qq#c2Ea@V4J3IYXBh5e z&X;*WQt@_EV~0!@B_b0r2JE}aEHezXD=LQ?My~^~B5iq>ddaI1zcPEEgKY-DcW6RB z_$(KLsRiQntn;we#v7>>f2tTP0~HU61bJHZQT6ZI49z5|c3^b@uLZh-W!EB|4BPgCK0rD-8F^jQMaUruHWQtMka73aR zijG&0B8S1OQvqLv3~)m68RE`sWs@zf6Elu3m^YrdRdiXFC(iive~DKplIrC$B#s`B zBa6*6`{_PHAHwNkkkdc$S#UsrCfwBpc2F}Y1tnoSgKjk=KIulPSfoYjU0L&DDu$zU zWrYe%W(J2w#1>Tf(KTxfi$zF^VV4W`wQ7J1=#L8vwuM9?ZEcuauH+TtanogdthJ4@ zu)5@Jv_=T3X<#o9e`}({!k}NH)H9+{9RnadSto=!(10>Avr)-TRiuUnRC!OnrK4Da zoH4+ns%ViBMH+VMb(vFz4bvOYnpo+H8mUo*0p?|3hL%+?0B9buJL$;zak4%g94c49 zQbR%jY|AS)b4f@hfKfqTUA#Ov5hin1JD`Koas+xsZKnEte~vpx!u7>A_H9BT(Hg{L zroeV;Z6In@w#O>Gc?}?2aatNbYzadWP7JzK5^nF2u7CgMVE?NmaTJl#5J=-g4y6#d zooSn(zQ)N3o(QUxVy#PoI0K(t7@d~zWKV++<-y5xXF-~$yP@bS88>p=G@K^745FKf zpgn=R4VO#Re|42B`iW>FrD+3N4y;kPI7;vZQe0Ge!JJ)_h}w`m#>gT;8DW|dW!3od z?+7XwYj8tr)C8XA$~{4OOd*wl{|XED5S@rj%UtBDrW`hBWps2?405%l=y1!&^M!hW zPTo}(^eFk7a6ZMj(d|~VRscgv1F#^EN|``kUe6BZfAu&v>i>`v3?=up%kDAPv@e?6 zCMP$01Q}z5FEdOm@eRsh7-RV+zA3X{Et6m{RVyXCNwd<5M%8OX{)aR&9EZTG=iL&K z5Dvpqa1KsF*EtWz8FC2ol@YJTtRuF8>jha*0h3J~Doz&?M4u|u^0>zT)5(t4f0}+& z{;vRze>)Rqy}~)b+4FzU3P|PuayiWZWiqexf4nw7D*w0CSbrb+FT4)qZ#t$x76_bn zkZr0_1moj`MKS4eeIJtclZ44bXmbUNf=8$} z`7yi!CaOpbBRwhs@FZ?NK#AH0h&L}UCfg-GDaa0pix=t^D>b9klb4E|BDFr)Y!Dl8E1qw>C18R!Swne31nx%-i~O zl{1IXZ%Nf8;CNkdPdr!>NE=!sVi_bMe_wO65!C6L2igyQH@jLyR5=P4q9{lk;R%t7 z4o(s_Dv9}+8IyHaK&7s$Bl z7=t5RMtEDQ9vy9Rv_D-P-lNbM6eXcKiqj?CWegYi zA_!$mSo96y@gVENV)6pewvzK*R}}+OPH`fTwA8&_3T2C6?13gPCZP7E@M2nK=s&b_ z@v87f+rHQ>+P;J!(ST)9@MWN9f0Fhx0o-NePm&9YQz9L?o;D?0uqRX!FeFclZub&y zMY$;nfCAK;meJ*f3v1hzm3d8dW$;9*G|XyZET5%JOi@Y$fj+M;0C*Xaz3iZ+3zyEL z9yuH|Yn17*ELd@1G`eKPfx;6iVVtf~0Yrj^X(tlh8iGOgt^9umSNdwp1eH zuY*Dr+JvR95lc12n#(Goh~5v8W5S=HSS)5!ckyM^HB{vS>Hu`7be7FRA0?Ji1ehRq zv#A#76r{pZGsHTi%m`Zte^(3>aNv8zY=!1R^<~j|qN0Qn?sW`}XZMadA(3nb8(mSd zN>c8VI#G0%LLBj)G%5@@Lc~RHc$i2z3-w*&Ecuk{PIz*OK&Un9m2#z20U9H`m(bGD zF|ZEGl;U^6+#{^9yUrbH)MZ#tl{tG>qYRTu{= z!jPJ=2H`gfZw-&GpWnKd7#g`GXSovrOh=2SW`)LF0zsYZyJkr2zeqgY9qCTSExirH zTz^R$Y2U_}812|%TW7>_ zvK`2@3_Xp2Wlm^FM@A2!W@lJv76)zOSSih=r|NAn1dXMr*kr3`nk*2{ zW>a$!4-Q%I^Yti`IB8WSWYAO0j0tm@_<%ywj!q2SyXJsgT9)db{#< ze+e#YGK2^$GR-CHs#K$Owz5sRI+yfBe4zx?nU*h$*^EuAR*O;cDQ6JH3cw>>hfX1L z)y5J$rFEzkjsA#u06}a3c#9JP{_?(c)bh!|LRldIY9$i6?hdX>w^tV?3aglh&Ii~^ zNMbA@s?#k1TM^4wdCu%mUbToYK3z!Pf3|{Nd&xZ(V)9^>@q^6RmC9r~7(A9pWC=W$ z>Bh12LHGD&6+|MF`9$AHMvmcT>FzH020ADJ31QLJ!G-$j4oOJ^)|EZ1qg`ci zbBQaGWOA_%nFYZ@cWd?*ibjz;eP29Qf1_dqVJwrgH3OYY8juyk)#W2AF*MNLe^rzA zweI;yd<-Uazm3}hcmQZyh^Jmgk4!h;~TIE1@Vx@X`{ee6Ywz6h%D!9O9c=VL@R@8gj!PO{aJTbB}yWBS|kN} z3ZO32a&HDm`xLzh)jvYjzby4t2G6vkGDV? z`7K$vs4$>ZSZ5JkoX4zyl6^t*#SuW&_$q6gHU)K+_0s1av6 z3RSnzU909q`ADgP74QJDTPYqJ%OsY5DH?ZKu%$(-E~aK{hgO>%P^|vF&S=9Kd!Pks zTNbxl{_dxK*6fX}Q>%WT3=N0hP5e5ufurCRu{IL?*^7ke|oe`KyZ~`YFOfT#R4}y!7T>=kx z6f;uj5>VdTlFk~Hh_rG-W;0kN0izBA%-Xt*yjkrXMC=FU?Gf-{T(C9Z*wA6k9)3E-|mYAbed7`MyZ&o45 zRxoLZ5#*nDhaI4uXy#Ga2H>T`!9h0 zjv&xYh#Xh8BUSFO1g^B490mm+nL^DK?fz>=6ni7Gl3slLezu=U71loy^5Oq1kq51ys zUIw*E1QYQTgp)8T41rUKsP)1`)hFbk=Qh*aEVflBR(2Z>zm6Ar4Sp2<$5`rDngU=W z{U3ALR4&N>f0;_Zj{kX0eiZ(v&tNe!;6En0k!5hbX!K!|vapBIDPg(e!wF(`s+V8o zIw2GhB(?EotY}GXMr_S-2yYoVI6s(prJ~OS)2{>|;U+=?_Ef)jHG> zn*%8TXsri?YPCXn0>MlLpui*8kPSt7;vay`A6Y4@l&Ykf2qsbeDz=OX*`{3Ss&Rke zOt8jEe^w|V{j9ILsF{uQf2i1~ca|!nuAq+>azvPi@aUmwlP zbWl`-@`=b?(7s`q^K+DgDIZ}OfX`=(o17}JZSrVt9`U4RpkV6Ad8z(nbwoAs_ zjMfl0EEP{aesJdiffeMT$-@2+URg7`eF`EbTu&5Quej>;r>O;!*Uexv#g zwdDms7F&Ye?!LXRI8j6mIWI^T<{>FGijwv)N+B6*OhQz zIKjXKH;>DNuy5};I-raNnFNHdIO|X@C^L9Sf5=WhldF<=rl-jH!I zNZHF7SKFab@rZ+ki~i)X4!HSvT-%n8llw@KHLj+^mT`E)bi9Kgl%)5Cyc{#Qc|Ei9 z2fpdCN&&pCy}D2&&QiI?A$O!0DW!Yhh=Qt4za3;RJg^KYA%If6Yz5@bz?@*)(!3le3$~qin03KXM+m0w8cmTW&!! zunFuFMp3b`PGo>^X_bFv_vKI#NF9^@!2n1SG7`|v5_Ah9*~CDKkH8E`*d(DPr9B7$ zAOzev6DD}QKm+CuO}Lq%7_5Ksfi~6*5F8J0fuA7L0u9e28nXNn6^%sje@WvNR(lxS z0-zz~!b^A*lpr!RM)okX5lBp$kjlugi2j-9no%1~=dwA9|0G_I|GXAID*m%b zg7%i6X(&3tFcZ-)3~Og7f`}47KoJ{E z7m-XLRJ?U0TUt+@%w3zax^ANJAONGBbY*KgN8 zN2ogVaT$kbIod}vyxAEp{x+HhBKVDF83ST4i=3JjySY{viOtxHDLsx!vW`-RBnv~2 zBhZXkY`~5M*r^g{iJ2oh=7$!9g{36JemHz_CB}E_Xdq(Oe>&C;B_f0QitvQi4K3@4 zNIW+*BgH~XOCq1e8q9=Bkp*FUuy?P)UVCsN+y+rD-!X*LHx3X{Ndu&Ev z5e@|@i0`ol1qen~#7U9SR%Q@14>9;C@cY4@Z+#bOc5=clb4)ICVYjhf+<&A%t&cHHm8MVM#Uap*0aTl_!3!WS* zw`{|rM5!rN=_KxcOxi%0G=ym#1`Mjtl!OP1(?jqD4Bu&Jmt{^A$|YQ{s1^;xss@Zw z(0DhUYK|x>EN4?M6izpR?Jwtrr?*g+$qEcjK{BHW#WjvOA@1|Rdv}E{Aw-H5v0>+< z;}3s3mion^b&sX)04Kpu0!*f3aD6*KNRTqZJDZp;jNN7U--NUvPf zKQlCZcybTl3-!a3FmNn&6W1y?I;1DF+AV*Mj7Os@vGA=vKIIg!F&{W^D0cyMm#>sa zppiM{P{?Nt`YHK5M#zUiP$?yE1qK;(B`!iwBr&DfqJs;$fe^6l)M}L}M1!JHV-=(< zFmU5DN`};Rk)`SmSS+|w%(B43=qFf@LMH%=AY7f4OB5_M2Wp&r(RS`v=2ekFkQINb zilu8=Pra}xY%ZMKwbTLHtP|y{dkpX=(5&Kd!ty8xui_ILc1TtZ~BVZMcL&@x976|i2WM^f}z!94pECAcfWM-O4&>qX|1FkrE z#1nQJvx^sr_(HwOnkj(Q}LH3^ipvlTgJYW{Ku01IQ$CZ zG6Q}m`kLTzVihGk{Xy*4E)Yb+6i()+T5aAh70iUC@w>dvmM#Y^zESCMPZDshHN^ol{l6fCw^sR9Z|_)UL*Z4zMiN zl?5mu<=Qmeq!QW|^s<2yPFXyham!Z|tO(=LiBXwCA=FxDCaq1%OwN{#)FdEWj7&P& zf0uMLGmd6(3jj!%r2~&dCmxL>-G?h`%W%2cu6@XFI|DqH>`;oU07M9+BjgzBKxi(! z6?8onhQ6SCxb&cUs{=K90}eEDaOW+vMmWM}uCze{!Zr}%19v6Ym>ErsBDRj6T1FO= z_>I^>l)SHtR?SnI(IGu%u-y~k5yk3Bf1eTagxVJbpd(s1Wb1J`W-kQeH0Vt5_Z2%S zA@#~&g<3`*6)+Pbk_?ITpyx~qH6=u?*lu7}0e1|{xwso+3D*_4W)K}S42?6}up~GE z1~*{~A-bj3sDm{v4yz^)f(e0e9kga0AOVD7|6dEq*+!GUH6)mK8LO^_E( zeE3#J_K?*4C|rIpT7MbLy! z64s?$#I=>-9y?SBv+*mQW5gxI77SGEJS-lwkY>Voc1(iBV_g}0wB|?-H#Su(3e-_3 zv3xaD;V2-e0fd3PL&WKYa-vrZ1IAG{>7#{C5>g^xL2NOO3?>N4{S2ZX#@^J_u;Mff zH8!y$0USXj$_g5dk0xW9f0nwF(0`asfOclQl~GPq6m<(RldzFXWC=L>LfG>6gW^bz zYil4l32zMqH>0R9B_rdFGM4>SQ%Px>g3({Y2%ut|Xc+CaqQlcs;ihvez%j1zC{|SP>sQ zL|WmN)o=O?91K_Sby)#JYOYW=(?29>fyqeNOukDX%eYHDibtS*0g|KBMF}(QnZ*A6 z7;f*Ovhlo>r7|KZe}S7sMkgPz0^wBVkRUrK-=HqX0hsWJ&^V~6Wm2vIyc|%^Fj6f= z>`JkOoQFv+s3H9VE*GkN20b9{tT2$Av2q2=APjnWPD)`%Hs}M+jd_R-jcrI~#T;`L znIQSfTtG!oeaDE#>6E7t{K8&*O%bM8TMtY3b1*DTtcEWOf4#FF!%1GJ_-Dx24naH% zxCha=`RvR`i!S@a$@k5gaPz+bznIqIu*mwo(G-Hf~ z7qwIwuZv^vlz?<41vG8qxhH1GeKL3i3~vS7rGp7Hjq6nR0!U#-PWzo_qI{J+UM7>DXE#@AvniZH^Ut)(ggQUl^}uyCKdnv+s%BOI6Fsi8Xn(~lSj{F$LY zvxzi+JRVDC;_y;eqMMVEYaD+!u%Ag48!iA_cTu)x1$LBTgA5%afl&O-61|QI^K!IQ zDOc(hqo=qtT$$vWZ8slfUI?!WSQUOw_^J?Sb?i&$aBCrms#1|Ofe7>^AxcP+hZ~>*?5N_oI6%qT2a~gk@bw^T`S7JS0dZq(2 zZ2niFN-yLHK{vQN<**}mItvX?N{uR^K`6Zm)vhU7?P{yS%q&!DW>QiZ+Z}s#2tP0$ z%Qcn2`YL2~8J=kZ$!H&yED-v^&rwYW8KG9Rsd4%}y{;Q+zfFN3o#<%7@{=XehIOoU z&|S8Gq|h|3lOKQhq86%RZZTtM5Vlu{E-m#!a)eP8|mQCU}jtC&5{;6}y$C2klEItG%hjG$i-=_G-xGifj>NS2uCj;DP~ z6kyNHWKw@geDrKGmpcsjkjr+XipXu;tD0mPLsOnm0!8Q}t~+v?)Q5gjHF<8h;D%(C zlKOm5B}1R4D#&Vy#j|2r%geI*EFwi2o3}Mtd(Fu>)hZE_LnJT^s$sK|tjFA$j2g|$ zz4CG^+S>1xuql>fAeZH7&<4jWWotUqq{Vp~9$tTmo|Fluo>obJ*?9}8_dv5V?OvN* z$GdDUbDk!$37j1X9s>J`Y&CJ~8FLCqGGfA^+Q7CWb+f2Y$?hl)f`Nxs$E*8b1ai1? z-^}HDkUHcVKw%l^EJDN~LID~$(skuyj}{BnDq-!6t5+LHn8;B|RnLp9eH4tqWqvFo zjB0<{fCXS45ONK;#bSzBor0q?ht?0X2BCBti_O841?n^xYYht{-l0=MwvF#K2uW7_ z^)UXT*D3xT0tqrCjiEsh>|kpx3~JOMWoT5Y8A50G5N^$^-jwRbRNhi}h7__|xgTK( zW@sw-;4=c`9IcEAnX!^>6ZNT>iGXBMZ9{*Bgc=;GPE&GPAJ3TG{IFQ=D$S2xP5K#T zNUV=Eq=gxU_ESn2iB0wsGRQ+jyl;R7NQ?9|m0vfQa~&;3!%o};Gp@b;X0)$0?KhLr zS=w@8#is4Fv3Qi;w^w4mS8M~Lc_lb!+`W?TLfC?AG?^ybZqkdgc^uL2WfgcL>SKQt zBln#?l!Ddp8S~0$lE<8_CieTd2$Yd3YKb5_=q)!fHu;d zwWZ79j)1>0KUh~StvfVQrr7B!+Zq+;1dG-2Y)LPVYS1%@WHbsn~kJ(L=OQr2(JyKK z*^ASUfxKzDbf|6yP56r-<%G#orylp!`m_Ygs2Y7fvVmYtUg8#@&v#u-FI7K^I|%O{ zD;7vrMd0!c&12AO_=K%?q!G~)!+?yat0T45aj5tb|0}DL{&5$7-Aw*v5`{5$PB6p^ z_Jon_O=~s|?%ghPA?wF+O_5YHj^&r+bHT#dkq_&MXj2*Xws^NikL;I4pfF49uY(-b{UMb z@&H?ERQaA|L6ser;M7!$WDiPggy1Fn8X6xEW9njR14tNu2^oSS+o%LQ#AtNJ!{~R^ zrvO4gWM7W1^M)4l1ItA}1o;b)?VnQN%&S5hh|? znPeg?OAQ?^c8tQU#4HvIAQ|Bg>b)jDp6=?7M&WCJnRpm!cPr&Q%?AMcTyB4WvPabl zgTt?x$8%V}W=4_3Xhxk$h}O#IkVZ(cf_C@xlryao(4}FgC|yTil^IT<*dXAgOFKXo zDO4bge&8l&PW{ZiIF{wXw2QW27HmWBgAtkxV!7|0Z6?|GXwY3jYIzDH{K?m^xc8 z(CxSOW#tdTah~pnxBwFIKE`0ho!;#6H?`t_0tB|NQ8pyyB@rjU3)PKU-61Nkhh;RY zrQlL2M_&U&)k?T6!=^!9Ne~Ft4Z`T#C2`3@h{|5!!l=MjP>HVfk(mVKT>LR5t3w8r zI&^Z-283v$3m7TVIrt)2t7c>gqDkQv=~1hFUsb3?ldM>*LJ%GT9-FeSDcvE{9F{kK z?o?gb*NW=}jZ#QY(#(-Sl~X&z44>Sz01>siaWyuQ^j8M8?C(sr1zlNocM~gZ(X|1( zEb}ketIeLhM_UAgJR=jJPN*vj5Yy{^F+^%$-qVQR%LkG-tySs{g1#wqX1S$dH=jo$ zQdwF9fUiWYBGg*Rg^mCf!VNo!c<}*$rE#Nx@jTbF@(^G1xTvT_WwgSb*vs&-6J6Pv`OV|ad zl<0_3W4#U?q%n$T#UYd^9?fP4Hi27|ZFwkxYH|4C0v_EatwPrxf?3NF^QnZ@8C*<>~C^G&Ll&TQ(#FF;1YH zC{-p?`!g%Fdga1@aIp2C?2p(Ago{_=risE|f@%q~4Ppz<9xi^P4zLja^y@HIE>;MO z+@rG1B%F>iThUM{H7)j+(NdDHT9>TOM)%0D)kc-+@@*xq45>#91&~6qbBC2uEvU4ymfAn1#h{&uDocE+EkOwo!rt)R3l=jnnBVG%}pVCq*`DW zG|QlNedl$~SGQ`@MpF(zZR(0QF7{5bA{I4~0~WA`Sik&?Y6C6ovgAT*ydYfsvLu(r zDxMKnR;3$%Mu&9R6AO(F*T#IIZ*0*B#3S%I6^8T-240nn!6+<8R)_$HhSR7bmcE9~ zM$ra>!c&U+(Gca>n0F6jS~$I`=YrjlO>d&{o5uPmWLt};eRgj;*&mi4M7|pZ=<>HU zloac!U?!Wg-_6Dq)tB3$x{!-N4`huMq(El^UWzY&w9u-0o2hP_J_beEVUt>)`jlHB zju&Wm+M-(|REQ!nC=OsKiI56hC6gtPOD3I5v@Qu>w6mS~PzpUIj@2;qM(>CX#wYKQoJ2FW<$0qc9Fehb>iP6_(XQ^IOc(Gk?Vw;m+VKE!PB8F1Jh``2w(>V~Ekk!}o{~BTMJ&hY79ynB6tpQ6i#g)sRfj|)m_`Am zztAueR*XZUF44pG4(e{nt45K5d`-!Jbg*VV)Y2NLCd4u1Yq2##UxC27X&Sh}%ZokB zab-^^@{qJX!x46ejzp3pJ7id!yrxV=DP-defeB>b-4K>wa00W71-q~zV zSvta&kHW=F0kuMfYsMOhE+O$oZBBDJ2s3EwD#FB*LAJ?cgz7=Wz=01CE(ywcjGXsb z8oNWp1ls@1qts*@@rxaWz(-Sm!5{Xur7-bp*mZvNiPh&G!tA$mP%ye~8jxv*IX_c@jGJ4vxduHKTW1c=AJ{ORi)a95x zaq=Y#HB`=k?##o%1m2H!`YhbO;D3k0R3FxVqquq$mYT-| zt0BBn{YYiTZt!Cna)6R=>}^R-7Cgh4U`M_~s@8cWaj9zW#j5#H6Pl`}sw^w=MOgit zr4=mInnmOo&{UX3#-_&X6}0<%NRW%W;j#3uWmG7 z-Y9ACQn5hCi}AxEgQDMmrH@$3jgR`=c+gN$a^4kvxhcb#v5OzjYC-p@Sny2;YGVT0 z+Ngl;Z4K=0wcT0`3kn!C{x@y^KF^zAS8Zt}rM;K2FZ%oFcHB859i)Y9N!lADcP-v`%K#E zai@ly#1NEb#JD<}D1!KeV$Cl$(xNO%PjEfZ?CDBd=2>iiULE|DFv#4dC8D=F(izC= zpG%DmgO(P6L@>-MPg2)lg*w6u9#s8$hNJ-!0Xha zgwURJd)O@V8l{M3Mhm(rToBR7JkM~c@+4pw2*=&Zl+lXyywX*RKy+Y|D(NuIN+tDb zHAk^C7O>|rsTh*DyRvHtT7i}{2n-{T&W>pcMgq2fAX+hp0)1?;9I?m84h#6Gt7X_q z;v<3Aj5fFMd|Jw0V$hgYeO!!gyW`Yxv08LS=)9ZgZzCTX;>>_bGlng7e;hreYH5-p zik(ahNi|bJ3I(r+(A_M&tQPS-UYrnxZ)jn#+Ay6A78!En5W+$bHdM*?Fgs?jv)Ueh zpWfnsjTX$cw2)L0^JIyGQ4u2nLGRmyoE)L;m856z?Lb1!)or7vZkYRXaUrYew^*E3 z0s|Jby_iYhvjfUTvZzZgnJ^}OD%~$)cmjD#LR}1hfcBCn$xMX3xC}M{UCDC|NQ^p- zu!}86Z4N(!?nd1<(bFVGhlZL-mEe;si5zv)Ne z|1euFtm;=7`v*T8>Hn6FC$f_NE0;)Q-XLB|CtmOW_L}@C{NDhrzrDIrn=o}2?~55< z`_1K04urZT+ShLoYbp+(ybpW_4Ods9x-L4<33-g$u7W8z0>OENnhPVQP-V>vW#>hI zN%e?$duDH;zTylq6Jrx!1~pk#NmP#sG@%+SJKxNfw!xsu?%wha5T7A74B&MvjHrw^ zGsTY-swkxsvD|!qQXU03Mo{Qe*L|VXUB$yYMWkXDw=0&C;>OGS`YRif`s+OY2t)LU z4uI~*EV5|dl-zs)(<6o>!Z*>ZW4-u)`F=5X3s&@lFAhYg9xoZiS2b}j6iK^d0DJSM zy}FAexTv06!YX3+0!Ybt2$M$}gW8WqIEJK>!s%4JNjXT8RT*d+Re&7sSK1CN60r~1 zKMHz3g7|v_MG@h?<|)GbUN*{63b67F{WSL_UnYu%DQx&* z+HYVuwuOb#bU=aj0x=*suh`EfPhCmLmn?GYqG3WnP=W=YG&+p*_oIa(#UO3B1bJ6a z@)2Ma9G2d+ri$ssEz-m;WAuQ3WJ}jkTs0J&mlX^x>q`wlE{6iz(ZH`lqe-qZBk<%g z#;2gIh!&3l7e;d`X0LW_$zDs?7JLE&ozhb!>g!J%V60@h3W~3=2=Vw1wW7gdO(H0wvI>#XY;!Qe=W8Z zDu<-L43k9bjWU->iHXKs_KYPDHN_U(MonyG>@`8{oMf_Ih!yx4AO>so@yZxL;4s-v zAQjJdZiNk87%7OuN)dE!^@mXeh)RU#QbPH2{3djP>iM_{7kA7mXiOoN8H#tu zr9XviYR%0KX&^F-PTr^q6=2quR|3)Eqt#*f3N^cxy_*$c#eZHFQ2hWUf0}_WZA3Gx z>S>c50p&uA!?QrBCfI~U2ZKFWPv#(*9RXPQOA*uJL9aVNDJQ-RjB#`Xq_?aJzq!F; zUaC1Pi?y*)A%SywXq(WHuou>V3IuQwuoa=vJ`o{2%-T9zD@CT?;a#4I5ljL<16099tSQQ$A?EVB&DqqN^QSFm7&e z2>iS(%pK9XWmS-YW;V{4h{=@!jC0k35Rr7@fdl&%4M2%>KA~*Xql3C~^8t2UGMxzL zs2O;HtgQ{bsb6kjcO)@GLduN*`3a~$boogM&R7FjkVWRSWD@agK)aOaJQT{4qlP1Y z$$~@qkSO%}Qcb^bQuGB_$OF>UT{q$qC0e0|{huTeu#IZWtw9YGb}K3SD*kCQ#+@2c z5l_hkfxd_8?1J5)!8r75tg0h5RNk4sy*jDeJ7EEDY)x0jC(MerDZl`k?MZv(z-6qB%^qbC^^8yc^@N~c;8n&6Uk2P33(vu-a_6SN((icb$t3ng12VUJy z`9fV(K+h*#?5zUGD=m^uw+20dS$sleXwK0ZT=B+;{yBMFhH z9zD5uaA7WmuALSi8WYxh)rcc3fLAK^2L|czTnuBqoy1mwUWMM6)J;|btNeQi(I?_M zOIG%G?Ju8Yq%u4VlW&L>f7LK+$5{Ze#^5!5Rq*qKOM&`n(lTPqpA{9eJK8L0oC zu~z9q$O0TxkRgBvKuCplDdbUz*-Pu5-Y*Ple;IL#Q6utU95ifalLLtve`8c6D9@5M zqYVU#VhQH9kv+IKfi46%M4~0fn^3Y}ah$1^P-vkQ>IOZrB5@4t8gI1D0iob|!!v~q z^;-AIl*Mb!Z4cC@GMX#k1U2z+Gy2Py-gGFdrN1CySK2_dgH&4kMcUM)|<{Z%u@f@(;PR7Sf*iu%lb{SoT4H2PWTxB*R$b=On)KHm0Y9)ywIuCosQTsQ~ z2SVPrF*YV{u-L)Y;7kx6v1CtAD*NOzbZ1?m)F|oxDAY|(w^b){VHv{fz(xsBJXWs= z4-ivgtVM;&p^$+ve=wnNLs%6qK9(%jQskMkYKsGI765cQ+9gM5!Kgn@gFKc6D-I-1 zKSX;9lVr(?10}-HNGlbLqF$?>2z6k7mOrPi$+$uQwmf1WO9G3^F_cUJ?S>;peX z7_&tDw^T`@Gm~g{ajSOPQsU_1z%6AR_Hv^Qh-*Q$n927A-arMa7iFJS`J7y&LN(;D zFWdB04aq?Ru>u6xJLy9F4;-H`1LdtF8p20IYSBsVFR$3bBvS__5V2z~^lygY~zqsx^X;vQF{liwN~l3%0Q4I@;+v(!S;K$xcx1EmG* z_X^SlE4T>L9wC|t`WT*m_$)+~Dm9C;a0Mff;2PdDG5aASlEZmVx-!m4taXP|)!x=@ za~fI#^JUL3IwKQM$P~tqj91shBuza|g<0OZD{91Uf1pfRF}r8b^QIgN?cX2M^(V&H{qJFUuJ4=;6UY zI4gHge|LR45s7x7$W%ot?U2939HSYU143vTu~jJ$g#h51*sC5XUY!Jt)|@HfV%pf* zsZ=Ux&Y0nNS2mt-<5c1R3cL!msr4xI%qmAQ*3;r^>HKhaGM{w8-!Obw=$k{svAA)G z_;`(yC@^rY{FBv?zSt(^Lu7Q7-4~1A7E`-4f54%m=AghOV`!XE*2BY=1FDrNc|+0&6zfTTB?W-LxkAAM-h{EC_HT_`ggl5!d#AG6(#BnMCS! zfBXNn_)+%%{6am7&6n%}GtweWkj1jrZ$6_Fn%n~63@i%fR{gY|$aBCU8lk@YmM;e_ z;cy_AjHL+s#7Bo+G8e_Qs|8h;U8ot+B4z1%GGR;)X8A+8O-6mhUp-B6P0pOEq92X+ zvh+m|sxdM#tcql&xZRs$eA6R_Qa{0bf2CF!gC=g4A1wq?G>jypSt|i0*awl~xGLWP zVJ4S+BJs9`5Gte|ZsuHUdSEU(7~?K8kQya&W{Ng5?QxVH3%!_RNL3JTBZh8z84`Jk z@|c)>v62h|yrA@=9{FENkI>@^D=z}bb+SsV+l?0pt$X#f69GZw^4(mW3V&LHrdk(v4tF3D|C(>K0xmq2#v0xONO}MZk63KXX1{Ty_ z0UK%V+;EdArJTLh7;3G{EHTBl=Q1UYjYTlUY00(+wQ#guBD~_fcjYpp(FG(z*jLFR zNr7NtwEQTaWBDI|b4&GUBihbeT6y6R+?6mNXq$3G1M(~AULg7mo7z{gf4PFEV%yyK zRl#8~fe0Hr)15^cB%lKAPR2R996MY0%5! zL~Cy)26qq3MsxCeKrw-!&xq~kG!&9e>}KV ziBzV4Lp!Jh#CQ^^G*m$+e==D6+2t!Ch68Al^2vk{ga|M?!rshA_PESvq9VZNno>?M zuqPEqw;5}XBxM)wQ_t*N04G^!IU{N%{y&3D%vb^1t?bx4meJp=B{e5ZyG8KZtCOyf9$a~3TK8~sz_9~O4+yB?8uQ3;+zXytktBAg z$5iPim?$J?6yDYd(>x*&Yt0&!ZkjunZbP8JDHk=mg|cW$^@CMacqCMf9zDQi(L#;; z`lwQoVO=(jMZ62E_2!c(kth)hcH-C@eX*fU zp-T$_+Ga`Nt17lb0m;+Ust?3NC%hLiMT}}f6SG(P39bpyPVuBD#-za&Go_O;MW81m z$Ww<)L5T*mH^?(JnVS8PCT4wT0bnQcWolStHTi!aWAZge_PM5V zV95{5{Jx-Ie@NRaMS$;Ecsv=>P_1}%4S+{1V+cAekad=2OFMiU+sgWI;D&a7gZ(WW8D)9KxH&gwmaf1V0EgEQ3SAkNLS$j59h5 zSIQEn8g#f>Nh-IyIVBW@_{L1ctnX6MaG^i=TrecZe-qXzw!&jlSG}uK@%TiGW{)Pb zez734XX!IE!S^wx6S2i{pbqfHfdCBs9y|E zAVpj(k(SpnXKb!#$}}PtQ}YpLJ{I}KB8222WZ5iMDRVsLRHZcvWf%n{rtpLKThxbG zCfv%Xf6!MPbVfm&C%Hh0eNskp7@a1b)*6QF4-weI#_bzusbeYomsc(&ZzSwHD$WOV zC&N69mCiy_CLoLJNXdTkL~&`3I+@F~n=9RN$3pH-zafnVb}AMRzKmTj>$bRiS9cOW zmF|uub6M^SFN`KMZ7F+8-4P*?Y1Dvmu>uFle{C5gFOO0kI5%$tGg?QQluNC(n%A1= z9^GWSiQeAgvddy+M7sIDh#MnNP11U#xN{^^FZ%!}sHQDRSp^{~=vjtgvR6T=;_e+1 ziyF|WEksPu>8efmo7P)M1v(ogT5Aub`T^W6{Hp1PUhJx%S-}LE%AC=corl~ zf4~oEVeRvvLO#hNW-ziT@5SOx8exX;j`Pxhfv^18q_qi9LQ#i z0i{RNKDVT85K!>FcJRcsP>XWA3=U0tgS5Y5^>skrIIH4xM#Uz^5BfEt7O%4?#Feeg z5VJvQfZz%`?Q$g}?oobxTJ|DRt@1(je>Zt(y>BgMSEk@yo1C0Xnxch+mD-XFio7k; zsxp+N?0A9CW(0(cUJ_2-NQ{~~je1FtD@g6F&10KRiKoq(jpCutGMv1u-sDw9P#KPp zZ^aSfmK`BLekNk6xI*1nn*KZm9qz*ZQcjH5CcEOXOjZMR^jET7u_S%Rk`I6(e+l>_ zYC35~0EHGEoEc++u!hWmOsW-ZKGC{Qh$^p^TZ%lTh-}?x$sL2XT7+z6bF1mV6_|H@>`+ENO zHTlu_PbgRsLJ|yu9($e*{on>PfBt-#FnoO}UqGF6$dN+kfsT%jHvyGT1hsEd+=WuU zMR8XM1{1ch#*4Vyv0&J#It)(5Tu4=BLI8{Jm-Li$CJF~~_^|d&d>2CV5F6H2N=D6^ z8CT{4oKov@GOW~TRqF*}&rJKYtNZdARRoYkqr#>sV2BHgIucwaS;j8Le;f%^6z{q) zOjUr4K*~u>xVR89jc>eI%vWti2Dimkn4I*I-(u*lTjm&^Tp-jhVvg|9kvdJ56v`l( zynMm4fCvIA>ZBXABy58A&C)$=4Z&knv2F0*nHT^JJSl^CBCKJ zPBPRoY`D6Cf(jKL=7gmIe|7+{4}z(%A4-pUD8sH1R%4|D_MIwi9pS6Nko}$49mQl| z8SN2>@PVTYCO{wP2@D*R(~@uiaZQ-ve-P_|CS}DML7qAY zWB}(P6|jguu=_M@nu*7{APWM>MU3&nv5)6KCdg>00k{K?SmF;EqzFr^kSH3( zI1{4{#)r~M#hXITiLA`w8Z_NY#67z@Y@ksR2gmCrg5*Hif1nU()2^x>v;x(Ga)41j zC9*#tbRjI~DGdSsF0kJ7hSUfQzhJY*P^$@Vy(ryIgc=)2b>v(~2uA@xml7|o-h>wi zhsK1{#cx@Gw`g0m`Ce_b9(-rqb&B+V3xhMC>4cC&BQJL?u_E*^nJB$&1mawEB>iU%R}s5 zb`R?$d6UAg!?H>aEe|W^TDh@f@}z5u9siN<<58g#=D1s>HhY06R6YZ4mz&s( z=qgG&F%bz{R?!1>q(a6C0w}Z(B@3ND&@zL|ks5}4e?52Oq`n|dzQ=OxM2hgX;NY+G zm9wNW2#iUii09sf9_KM3Sg54D4@_htjd2>E9G7Ppn*7@NTBQn4q^fhY z5P*mi1JVFW-i%wq(n|o}!&8bgDcQoun9ySc>%-xZ2l}9_9?tS|wkB|yeY4lXn6)@c zlZYk6Ekg4yP$ka+!EbjQDy;X2os1;_4+q!J$!~S>Y{bMzJB1apJBVhr&HG@7_eztM znje2QQ$*mWKuPOz>Nf}5K*cr0Ka*_b3yT(E{-Bme5vOjJF2Ov3NQ5s?LJD#^K$T7V z!NByuOtX0m%sfCALMJehq_5{K{7%ePRL;17`9j`oQ>HLv(xe$?v|3&x)5J5;^}(8y z;T_|dC^jPAMBCAI6l*_BHO&$}7*nYkj39qf>|;(+Hjto_2<2jsX#^J#pHhp~}+)0S1-C1hy<&ympuhj_MiPXTG%N;6UM zt+k?bP^T~8%z=%f6>(+M+7uORhf*>OtzJUxT!fgb75c^qbSF1Z91m{PtI#K1UABK% zYath90wb>irm}^TuvU76|5KwP?yjMx(?d;dP60)%HQ+*L{uSmGIvbjw^iE+b!@ zQX3ZDk-e9l3gmf0P|+J@#>jvX4z*juct1cru>dJb!a5wW{V);&GG%8NwrP+u z7<3GyC`_X&W5W|?A#a1%g1tg|J>Y-W;W|V9pc^r3S)0G5AVF{($WLuwMMm+U zIpt6lz0eAIcy`b{wMp64R2z82)rR3@3s{Hpgj2>cH@I_?X@ygQMJHyDZiJvDyYzlh zLkIZ>#=8V)C1PuLuCxjy*E5E@Ow}W;)b3jSj}}?Bq4>-zU3x>J0~$C?D!TzA-)1VU*8A>_W_q zF&ROnSQwc$$(`~oQoK^sPgSJ-?_njHM$%I2WJFAHV5X&uA$zPgM=_WfKX1a>YBJi6 z#Y1AJ2q7(IVm|Q13*WU6ipPI4VMI=#5|LsFIVJG6#!_V>ve~r~hdk~ov!Kb=lr-JE zStX061rF_KqdcM!zos_?7C=4VHc5XUQO)_9JEbQ|3NcG=N*G+R$eul#M#<*=&BzTD z-J!rP6dPS#%Q9G_R3Re@YQ=OSggkF6DkuajqX)Jenqt^p9I`k`KtO-s5h_6o=@tfn zud6-93yHMLOyU5h3#JcXeQ4U7Szz?`0=DLIwu2%xY-K9NwW-u;SU^6jQu(}dJpfB2 zg)x%@F282KR}5t?U$rwU>Hfs3;)`em3DXP*<2R>hRO0gGN(rBMxKYAvv9z7Cuplsh zE6t0=$)a*g;PFstnG6}`a3qH~n=7?x3z!s(oKRD#G+mS$u;hAplSiH(U^Q!Cy64W- zgeXv3SX1rZi)tMx*az~x&UQGX4*~|G_H0$B7-{QMgH8+gL9@*iSa2mMo2AcOkE|<1 z>RPH0m|RyhuY?CXS|1$3OOe0W@c$5F07k@Rk*CW%}NF~kytOk88MQ-GaN`k{VGJ^00T24~pusO7th)QG{ zCVYt*04gwQrV~K68H*Sy;*~@XWh@TY=}nBNjbN7)M|DNUMoa()FATffH$_T7bj%Uw z$Ic1#|HX((6bK2C6&+Vstif<&8r6tUh@*`fJ}3=l3jGIkXBNgC2>Swl-9@crwFoQ< z8H82;hz)P$ldGQ~4hA&iEKtPpA*+XI1^Y~s*`Fm4$uyvO(LhOa?XclBn2t+@lMbL3 z3d`^s%on`GwnmYYE}$hDhIp0U_m+&OQ}`UA%osnX(L>yveOwwtQ;Dh` zsQkYWU5)Ci0|3}q{ddCPf65{MQ#z4;J^%lj{HXl@B1MHG&izW*3}(6?cZa_ROh4A} z&R`_o;L$Z)nkT;k`U3+PLb)H&0NE)E6osu- zEP(b(jUD0el@nQ;J_J2U@bXp{aJodJWFYg)M5MkBSf+%s!2Tt9m@$v1Ba!Tip#5M# z+=M00kS9NreyAX8e0@)Xjn&Ay!t0~r{VTCy z3P0_UKOiIIU9_ExsE$E#4~+1^tYsP4EidKS4e$)DGyr0|vvErJ zF~<@$GlX8Q1;>ky3d7dOz+#!P8VJ@7Lx**VNp95PeS8$EOg74_hy`u3GJ@4BP`feU zD$*UKs$pB$*m)6KdvC(zRYi+`!5U^=q}yYOlZV^PqjFYg8j8T*?sOWlk^r5DK^u=D z-27T$9Y%XISme_M6)5R!TC!a7nKil#QBVPvw|#Z_CX^yuZA+2qtj|)6EL(7Lg_DnW zLD95IuZ$U%v4v9EHJOxKq)>SLKIyv(;(O3qL`dM(t}QXT)E5gFgP5&an-lvw08 zB*3PvmK+b3rM84@O>KZi{Y+aXEt9hOaqPnFq^?_hr(Ond>RTHZ_4OH4r3WP^@5U^m zQesl0mhBA+!1T-kTBJq_&0jG;EXAB53H+2&T*rRks>RVR`kwMj&~u@jC-R|#2#1p- z#~w9AvVRqzs}i#1l1HY0TxcG@A2ZaxMUxIRQ1UpMh|gbzEgtH^Co5qJF|Wpxmob;o z;xu_Z5>RV#`JUGrksl~M2o}~kb*o)TZxe663fgmH*L9d25u3Ml4-0@i`#dt;9)q-- z5AsB%E3-qLpaNxt3rLV$M)ul4Pha*(E(SQge1+pO<%rEq>eC>9AI9BcJ_=cnrg96h zO;T~JS4gNw1UHk(WuR_-EhVflk;UV*fN|bIr(cXjNX&6+)=kZ6M}#adlfIVeZyB)) z(qo1&!kW$`%t8>&Y{NNhuHTceYjz$XQ%BWmM$s5t4W6c3t#wmDNmv2ja0&>3KT9Vs zB;_~zc+t$Y?h9srp)!C8O~iAsLA#nZXhgzwhNop{AE!{Cmi(}j?{f0F8A^7WlV(aN zj7`cJrdhXS;LME7Q8XT_4Qw&t&C?65JH*pY7CfP2M~yaD=#WEG#DN4d=|_{v_yoqE zIPr*GMG{G_Ie|mgo#e81I>H#aWUK*LfxggV6yx5U_5`GVQdsp3uyFEn2^kNBM(FTL zcO1jKTsMsX7B#&xe~QH<7CsyfFRk7|EheLdoNQ=fIOB#|G_MxksMpS^YnZ`C73p>B z1iDa(OtjLfb(#ctOe=UFNGk~O21)*q54DA`4x9{$0n&@F))(nJB!Yxi$Ok0pvGHVq zyi6s!o1j8}m|w?FK)hT9&KY8$AH=T~_7yg+uydj71!`0ZHc0|gQ1~x_kP+(eElGMx zOdUwdDQoadvPN-1Nq!imid^DAWH_0nD9H~RnAb1nTv0v^;#4QT`9KEe`zO!%Zlq#bN5%Ai}ox}kpaqtIg% zLY(YQ;S>@-r11M}PQ)@xi$Vv$(Mn`M% z-3p$7UBwz^h8CYem%%iE0n(#1IvwbAIvORhnAA7NupZ`@q4wJLmtIFy;t%|QvZZRa zZLZmWj@i`UO?Tj;IvR>bu5kMLjh0-Nf{iw}<1%Z)U<1$fXdIk629E=o-Ykik2=EJO z&NhPiMcDkM23w7KL2*PBJj?8AWmp>A1RiOCCyBfQ-D@BvB6OMmL@v@oMcU$Mw?Udf zGjb9Qr|Ov!s6=We_ap*dEcn8}DdXBCN)LH|Ni}UiO!)0#I#ESO#9$j>ozpO&DPfamE$L{jH(SAQ3V_ za|PGNSR5YgvZ zUVwC*bS*tI_C=H8!<Vg$0qEKtu@{(C3Eo^v+Yv^mhOb#vBP^t-L zU7;m|EjI(kvX6!eTE(~Hu7XFc;CwSOM&kqgTvpKC+cH{*2RKEz8D)f9RYoL#9MW+C z{vMtek$n-X_0!Q3lB|tJX+AU?y9v{j0v-bS2KhV2(;^DQsW`xtX+%(eng9=%MKZoz zF+NCl6#LGTKAA3kL`E`eV~k`4v*Q?q>YO?+8DlTjjfg{x@)5#Obv$PPz2?xF{^GpO z`Lj_%%7uJApZtH%kIw&&)?cOa?~Ubu)A3xw=>L;~`tM9G`Fj5MwfGs-~CTr!PNecS<6gLWnPD@=Xw@@!t2zSCLxLyl|5d1 zzyudAuT#TJun);6{G6^RI66=*3-NAW2RdOkB6V4#&fMv6AxYUQgtqkN>ckYKz_yAU zinX3+G@-#>5XGWX^{A>?7DzbuxEQNYVbq=7G4niB!6c1e#wQcUH)X?`Amm~*spp@G z1PuY~j^{TX5U3!3Oa>;aDtQDW)#W6LwJ6Al=3@+iM}!WZNGYvw&1x-GBL^eIYZ4QX zKZ!~FRuSGvvsFr@R}}Zq7Xsn{=Hl~sO|LpAgE$p_4N$b=6?p0t51V;RwCxdL5$yonioc%u>_jn)kHO!W+E zO7R8}w8;oS#IPOl*&{kJC+RAqNJ2^i3Xi3-uRvHyiKnAzDi~kW;>TZ@BF02JjI-|q z!N(yZW{*XGq1~N}WeMZY!5K+8rjdsLc8|vgcc(m&o?31tKI~FgPY>gk^HjqU)Ao{tsfd(_dZ46h}BEs(h$Hyj`7M-g#7r&0s?D@44gdQ5!B^tGD!x70?n z)JB8Z3S)+zix+^jED_6eXIe}WzL|_Ab17U*$LiRBL&)1)H?jJ0tqCG_b%Z3OOEB@n zS!`6E?ZQG#+*ydfv6x^9yDk97#H5QNyj3622vVFNbV3}3`3=dgpxKZ-OUg zyQ1bcKu9A4KIG)3cEZi*_;|BG7)ZdrB`YeS@=A6NYtpSY6Ishv?sCrjUlN;GWw%zC z9yQ8;RZT*u^^7g4{F*x`N#-y1xEWwBeJu2M7(h#H+f>GiyCKvoW@FumB{v_!$Rbdr zm#0Ggs8oznWyv&wj2JiV<7S=4TUd?~3T_Y@FE0=B3!0)MtBO`Zo*hD^gsmY6y^fpL zQk&OT+b;9e{uZ=Yh$eN3nA9FSo7E|8Q+Mfq)Fw=KjreDhNr>BIlHJYrlrmlbqHl3J zTjjnf$4J3whRIN_H4kYI)mG80RZt0XF-%<&4c(@YZoE=QX-iY04T?lMmPqCRUSCA~ zGaO-YO3A{6pclN*m9#^=E5oL7N6!mNx(kb`u^hT977tV~)0u}lv2j=1S#a{D{e!oE z2AL3#3z>l7X5ZVcK@X$c@uV5x8pW${dG1&p@dF zGz>FKQI(;i+y!nk%V(Ub^&c%3Bw?NyLyz4Ng$KwsJr}&3NJK1r8>N~`uPd<$I5X9( zI#_(7Q1BL*%nX1+X(9eX*rKscsDqGyhLJHIQt3f+Q1b~b6Wt-qCv2y}Ra~1?lG5X| zf)%0Yv@82Jqq3ey6Nw>suZlvPr&FCogl>9RvzdU&vt-*9qgEZ0$*@0cvfb4O)D!|b zk?W_N(y*|O5rhRbWOb@Mq0EUiGFWf3_>*M5glEE*1*1ldhw2cdW!U0MP1r1dbp{0t zL&AC)vnarLg;^(D7o!f$-_T?^Wb@Pnm`EqQnAH~V)O@m&iC9`FW3iBQyu{zxOwu^n z@Ip4OPc{|b-m!wrgwO|on!-$S z(T_g3M&1c(VLc0N9wd^4%ozTE)|KqUf6_|sIsUZQC}p>FQW$bmGMR=KB<{%+Trr)@ z2vOAC3IArg&1ESaN`1=^iW$(mKNj9qp=Y|bl51fa!n0<^+qMLSq( z`AP{aXrJOiBca#lg-3Csh5aKj!jkC|2}dX#z{d-W0FYRL2K7r7ix}>Iz8S7N1Dabp z1-{^?w7U7k3p0L3S8cPm+@oCq%k<&(Q$YgD#pb1s&rfrK{S;XNV2>0~hU0k@f7Dzn z(FLV?dUwNOaqGi%W_wTIo!;G^&4xxV0(tO^V!YWMZgejdeM1C4grcO<5UVJbpA)n~ zh*5u#ai`gikn|K@yHE#z#%aj(u1#XXLe(qSUaU znLuIOl(oBTYyd31QIvCvgEt%Q5%1e3R0cGHkZyGCPARM6T|=@U%Z=)GQbR|`0@2}4 z?tI;eFJX%y>=2|I1lb|9Wm8>=a0Ksa&w3Y`6>5JEv+Y9jZMw6YbQ>u5tYELv>GpRL zAFjH|Oq&PFb^RYd5Ry}W3=I$EfzF{u7aypsSQMoeX~L*kh5dig>J$r(K$lE96A$B! zjh^&q9`%?dA8Is-ta0lhw9wY;wT#H|MfhoFI^iRoc)=mr*Bo@1G$&CmHCdAw6*E2p zIPaPD`>dZS*5F6{BD7S!QU!RIWzVsf`uCoAWw0b}RT!xh8>RAp`sB>|k(u>|H*drN z!ZWQut(rvmKRY-m23jiQ;m)CAExbMzO^a9cOX2mIXm-Y$4~rxCLL%>epS@LfCdKY#Tpw>|`pINy5LmSXW&9n~248@OmQI9ZSO-31L|Z14vv` zb@mR)JsgGVlh3p& z60I7xD3Cn50ynh-PU^ z@x%7NT6$Ctb(Q4lX@csh=&tyS2!c1s3z*Gj9D<_#Mx`8`^rBAQDNQNU)r@Y~p*?K* zda*WyP2h%xtHbza#2#3*^1!}-Rk2b&-2Q%Dri}_>D2i317~Y6h6HT`p$qdQhEWXzM z(#eE*Y3NoG6{V-4H^uA~CIHzG>j%Z>gs$}5>dg97wLa6*Brev9BjW#? zjN*VbJZa|!i9a}dOFov=f)-C!83*eNfMCNoArlj#gy5L4&56vaKW zS-LZXFXbmw*@XUNHr53#wW$S?G`_C>?0BVHHY)|skC`Z8ETDp`EH4yjOX+x zh3KVEX3S5vy(R2!`ILzS;G%fi5<-^q*fVK0ZT!hpJgq;O%yf0(CsUckY!jReOz>pQ z))Sn{WYh)YiA0v4p}XnUiFhulKbh{Pr%hxtzFW6`*kakdV=8BV+)>VNy1OfQ$9OiL zH9px*51WW5QX9YH_?U6WWX8NslJA)6$~N6Goiv^?-j$$F=CZy!Zh}uWPYZu(kjlQ^ z<-Cypz-ZA4a;voLe6Br9A>qOjSt={!B6c>-oGn&U zGjupAk+ub5y+D4i8ZHB|6OXTNz@o0IA>omCrec}SbgT=18BLAUJcH9*S&W*DrQsvO zM;x!z!nRj+I2$I0_bX0P`)-(hA4Z z%&-&Rs|hB8ZH>IRAxvs}P}n9;Etzv5ygSFoFzp!6A$o@J))AryAnKh2XoxiA7^R{E z3Ra1qf@{p^Y?z7iBKOJ;(Z6Cc_v)|@h0h-rQF*$5HEX1S>E`Fo4fparlGGs9BmE*C zzd}I;VNiO7d;>tXlR!s~G=5+)MqypC!lYOxZUqZK4bd^`su3C=ligT>K6VnX5y097 z{B$*L3&%YPrGvt>93ML@tXufDPR>bLNv@<}B3gKA%2^O|{+ zhW`hDfvEqH{7)jD)%l-X0{K5PiP!zV*W!o#Ke_bC(-mZR!b4TBMB(2a&F)04vN|(y zyHr>QcDEUFl+YjT8}`^fCuk2Q*;dB@I7`60(vjTGmX221bhL1+I3nRXGP} z&i@QkAUc! z11i%nY{0A!>ptx*5z@V6P~lv|yl|zuM#xHZSUYov%SNI>%@&SXKJ#?14h6};zoQ+0 zxUTSfh*o0K#ZSSj!!Gb?i^0iui#Pb z~7J!a4{S=vd{EZ0Rs0{QADfsc?En*{k>7*Mb!8!slS{;r1yby$71Q)u584oN=40x;gn4EiHOChDhA9;8gYC(N&qNxY|}qaBcchSaT1 zi8{NLd<*+65CqFwgHVMb>}MA_LWA+8VsQk1jKI=7aWv-G3utEMU>`tvp61x|_%zKP z<{9)w0FdIgeu{)JXup^vPK@b4eWmDY|Aq&OJ=~UeGgo9T% zHVM6uh-DI)9Q~5SoFxTLx{bhpNivh6s|)*vVhO^D!H0122(A?_(THnlExEBLNeYjt zOFjvlL-iz?Oe#tD$|iF;^GSkNpY2J~-NJ9>fI-5HIzi-yo+Opb`JSYG#NdBpy5seq zz)v$h$1CpwY=r+Io=GI7{hvyt<6!^8$6vSqUxT0B0YdwK{$r6JVQe9Pa3$LF^JrQi zt;WPP=))YxK#m;ANv-h(K>I%;Tx8d1*(y}yT*yNO^D0c;5b*;fgb6ez#sc+B?x+I^ zC1l!xx}e?RJ+G5U1TY*UNp7HPkT;>LX^WLH@NQOs4OB*$1h%B@u?K|FWRg{5ioSsT zyA1B1Uh&{zD$r7Kf^`(dl9)?S!=;=$I> zG%dWjuCbxu8C!&6O?SsU&UPUwyx-(id(TnuJozS?mle?*gBNj}M zxMh$-3Q6I7w~xT+7Ko1q_lz~`o`IMb#W5S@A^>cN(yK=XapAMMLl~XSAPq z%P9)C7`l`>R+|vNO(Fh{!S#t)E|QF;BdJ&>l8$u^&IH@emiJ0nmC@0tf&|8hp;`v8 zp^w2{38Rl&NJe+L{vN3Rv~L&iFccA3z{zNn;ErW5#a(lSN289sQ>!`Cz>hB;bw)f; zJR=S}i7gv{cE<{{S`Chb@GJBSym#8mtrq$8>Uv?GkCbNM-z5K=<$qJg>rDxF8KO~0 z%jBvRD`t#0mYWM{zOhi#0of}Oohy>QH!)Tu{7=t(wO>A_c<8e06fs{9fK!AFhxxZe zR{hpxciMS-somj!?7zTM+anXzI$R3}@4JK%2Z3|B ziLXdMu2;t*1MfuSxA1jw>v<41^4%Kt%p5)@lb%faMUxA~t9z_oxiDhyaby${?cx{6 zP6$Cg5V8A~t&G_FF5z4Cq6-!psn?1eu7TNW86ZVnD+Z#MT@D0?0IH&Ou##9pPt7LFSFmXNK(tI*4-SkjHA`Rn@KUVB-;jTv>;8X@|4JeF7&=0OPg zL7yIIb*`5ru+CqkkYZ;Z}lfdwQ9cA(>e=)fDAX~8TDSrcA@d9+q_DRN0uv?U9 zjMywHq-5!Gqg{NQ>A34pCfsP2vH>x+ltbd`J@7hQVX1lWJMF2wd=3anh7t)TH{*+U z%G0?3Lu2AT_==p4`DkYPrX!Ptz;Uvx%d|;{g(DA=J@ zQwO!f=rVB-Vd?;UV#>&D}DU57F zS)&X?^|zRI1pwgbk0KO--lKeexp^gg$zsV$S<*Y|fz$wyb8PoPEWUs>CRNPZlagM{z(`vD#z|n$W{^!ZAfI#xg|ggj`_2 z0d{9EXexf&@uCF}H@{*A>uKLzLiWWV`4e0qe-{njY-us|?fP3p=)6JeDVMvlrf6$M z%uXT*NA6XN3Q3D{MNCRXED%IZqj*sWWY(OKX*R;dT3}W>%#TaktX>hXLIcQu;b{#* zrV3MG$KiQmSu4hi!UDGub}XL8!9$NM)jAALt?)&(fYarXDEbD2vTr7Ak2`r3t9jKe zgwP@4H)^;$1EUO`WmwZ)7{*mp6eJXdR|O^@qBNp3n+YnZQc8?Qq$Ngee-Q~u1?d3nmN_sXwdVOq1NRUY!^hE~hR*J`|l9ZKBG9V-lgQnOw|iww2q zd>u8*Wbz85l5TBY)%j6m1-UG7qVPZHsl(EpN7!?x6$HrEe4YUgZ4!3BQvU4A-HLhM z*>t;C#ZD+e^(H>|Pok?DaQ{h!efd=m*NeNIa#!KZ`=ZGwp`Q>>UueQ_XX{Dce0*b4 z#O2P2*XbA6-#HWaFOc&L%s#%qhQMj{xvL&Xo4l`S{GjlLb?L=4_uVG^9jMK6blr!r ztP9dVNjH?qX_oZE)7Q-^Zf$*Z)h&~-^f>cdzD%ZuD=$9=*8axqMPw-u_b}J>bmgr* zGf6#PB}1LDFv9hhw*m{*`v^Yb@5uVH<6#7!)pa2+&W9EYnroj+JL7wno$3Cf(*pv} zhyyP_3AQHUXQtFy>Ozk@SfK~s{}xo-Wr}4d&>^Z`etfSw0o+IhiRG_{_{~yG7M_D7 znZ>A4p4#}^usEn^6kY+9w=POI#Z2}6D7{tM+_ILi@CY`Wu<-9bR=`Ng?nYiA9VLx0 zK}G}LXN?EyZl%M+bQ}|9rflA|afc>ZRajI>8~qa?zGf2~?hl@qGF}wh@=ePUWi5Qt z`qu-8{7qpWqGb~szwh^3eAk)${Qaj9D_!?n^_t#DZ$tCb7ntinw$9~i-9`eJrf)Qj zY*>GB%TphA2!8R~ooui6MFH%rZ_%Rls+zw~(ii(-wU7Jtol5)RXHULx31(jui|%Ra zT9|Q-c!i!!TDRuUjZJwYi)8pGY)B`jCR%;H{i)KlJF|Fv>?naF?g!@m?m`FeA7f3= z^XE^9>p3QJy6a6>=>q38N3(X;>$%w~}V$<+u zoFP_PQMg>%I10|_eIuHrrQcln{l!PZIU6?LoX=73=cUF9H(+{-nkP5C-UJ)X;aHuI z+=M1<-lo_Nd*mP4!RG~o{-x5LJ{Ep({>)+Ny>CM4G2%{`=idUK3LU=`gvn6l*hU7_ zD(op;$6=mc%Td-70Yc5?uA9ui%wZQ%T5Qp0gxoxt+D1Qba{Ts{dqu{|T{a(^2|3$4 zzg0SOOa!MI|CQ?}sUK@f(+_k3BK*iL=F`*v%(r72O4eZM$S~BWun%MRKm5)@y~gTc zoOdI`+5BB{BJI|~&$Q_*BK{R58)mYX*S~+Su#XGibFB@jivW6EBkVM?s{%C|Qp3&| zcShR+it0ScCPjv_*5ssHSI(&2vs=~;{+c{x@ZRcH)7{Pz*pmi{x86xG+qy@nYiRb| zt4mIwJ-!TIs#0;aESlD{2>EHA5v3?*jJz*=6T%yJ`D$szao(f!yQy2A#bIid6R|VV z&i$u6@NhezI>i_g8e8Xlsoax+3zb3jL+g!}E99xa8ep z49*fKUD;25*_hnA5UQETE4L`@Ge|fBf@A0i#DKGdHDK!Mym@$4cRGpUn>W$ajICPaf)CI~^~%3KTsj zT|ns4DpX$4kPE_k>jT%iCgS)}0?{5M>Ki*$DuJc%uN_ui@rFh4;19a!ah7IvoHK>cXFvMc-8xwl5H7p`WU(>fmWjMh1KgLn14 zfS6y8WYl-xdy{d#H-4SJn!2DDiFzh&tRY}SK`!oFpM-!91u9r$@M{B+aacKMBKf6C&|jDASY{TuRe zOy1psJ z^(H>E)yucOb^q~2HBUfc8`Es?;J9Aq_u%V@9G!h5x&H)AcpmXYSKZegkofiM*i~7T zdv)!}t_7#RIX6sBmWEWIew7+!_4HZCMLNl;jWvZ6>aB}<7K%-lIC9TO&S_iLrS+C@ z)tus$X;GmW`WMV-ad(V{7KKUN=;TRxk55h&8|1?4C-9TLl5^Amz1&A8h%$@=7B+^Wf2Px{Qpr$w| z?DlBGxzIDnFd*wjD)$4`C!Rc45`s=YR=z8ANALu_Er-8bQP1PDZDUtPzMkiKsE>lR zKF6u*=PzO%+q9xjpKJ!EUZ>PK8|9dAoTrC>a7&AOMj3g&R8S+|pEeena>aA6JAk7P z*0hkNNBer&#Sd zY=|zFuYy;k6_T$1n0*j;jl=SCvwir8(N@ z>FvLLc0=z`Ow9$)+@p8TlQVHaZX!ov$?^(?-*xTn&&W#EijEygl$Uxuah@aIG)pRE zmk9DagYL^J-)+qaIv2>Cv9Vb`uZXms#(7kQeEpj9Fp5aE(M~VM~^nZh|c2 z=pM^Qjg1MlXl69ANLT2eSv@*Dcu>^py}o;=NZQsucV3sDv++s^A*FcU!SJPD9p&u# zQ6!RYj}O~5e5I8`cT*lX3VRgNvgC z9>EYoZ;LI?#QE`zx9}p2caj=E@!i$DQpVG6wj?tH`XsmLloQjyhN=mV{j!7_ z$y+6GM#?p@S%EP9=rw1>eoDDr$+R zr*cdWq4p(0ZXQ6drqIMHm(hs)`=OcqmaC;vF2m~z*UcYlZCjiFmb5Mmi{j9BH9p|| z@%Hm-xRB${!<4W5;V`y9NuM0KPZ9sJ|dfCOH=~W%zGY zs>pMoe@`6R=K9ROpsm)++*MDWkt?*jMXu6MqQzyj=-evR6Y+g4@t@EsdEHs=0;M|& zhpu^@e+Ql!1F!`{alelH(qRev+O5=W9GSv2d`q6OdkZ@wKtl>?i@&h^5^ zjGd03wN=I3pMmAu`sZtEv>tH_RVQh=yj;uHDt3@=i;$TL8PJb)>Jb=GsaBAh@^3Bl z1qE1~isjvS)K-HRvD($0cY(qVe^$^MT#54+w8ru#X*y+mZzeyF-#qhX_}QD$&2zzJ z77TrHC2NE;SX}vHwVmrgAEWdD1r|sThk=ij}`L4xA3RJA}&YR2c4HN&s zh9As4ef{i?5Fr_p-0Sf<<@*UP*>fRf8nJ_V(rE=s{!`EZZe?S~N^Ab~*SsP2vVEn| zmd64(%OB` z{kB>Df)LjkZuPNeo(D6_52T{uT>55DT|uK4^V7~nUiSPL0#ba4wl50GBcB4rPI;Yg zj=W|9y)iP)m4XQp2tbHwb#d++8CC`-#^gDo*iec##jHu?bJT7dZ6)Tp} zRvAc)skDXQrK{K(MhVY!ilvMAc7x?*E0>+dLQLb^rp=iwPx`fA^NlBpOcIvZ*pB9d z%x%Rw*ikWZ()@+9`itypJXYXl{{3UBRPW&?YC=*Q$f=O5njGtK{kg~y~W$qf? zs%pA>8F2gOi!;9-c2#f+MCZPbpK7eR9xjh$S+=ocYk3sI6W<_?{ijT?KP}#Hhdy|G z($4f}G#*P>;~b>_QS<6)AfxRirQ!l4hfkgVB&+^K=dAJPzDRT555*muweC%0%|8t} zgiliFe4}pXzQsR#$`?%P3KPBeSWQ5n>!Wn1{eQv0`KkWQgj#$lL3oa!nTpwm6#^_h zoosXB{1$hdpyvKxV_ZzOSKv`(BXXO(qM4{}c_*!Tf4WmH9C26|Qn5{3B#sscNvOp^ zTf5Som{od{;rUDkilxDtq_OjFm-Qb_6#vfx29nXN|XUzgZ_K7fpBq%aZZz*WZs3xEVa004GpVPpmQ#An%Y*hp zxRfovzJ7;Q7}WEiVAAFIf!6g;01n+^O6qii?1v0t0$FKDRZ2z=Yxx6yulvja&xFUkmPXg1cUSOX}v?zKpi`j?ZTSunT1bx>-l3g|&@Y>;m_F*f#`{XV+ zY`?cVf?jP}41X+fj>SZ>3)0BMjvnm(bkD?G<`m^Lt!5NLZA?R<{STKQm%?H5j5rCv zGRyl-Im{Q&j24FOZnxmTip{f^Sn!3Z9<&0ZB;uWbAM4-S#6Vw`S1l4goGt*`t~4_^oq-{3{7RsTtr_mK&5l=!k@q=PFuvzZ8<@ zH$AfA4>Wd{hU+yC8A9m0xITixe5(OANJ)AQsBNMgC}c2TYrH{QeRzVV`w9c3vB}qi z33%y9>&bMAHqWM`;DjJr+a?bX&`x@x4hbC54B7#@g-n^dB%r~+7qux7EE?FOuxgnI zXfBHeu{tM~*bFXuF!#?JY|~rw!NfE%05xRLLkJojM1egIQ0jJEh9UqAS)_r< zd(ku{mLYtB2coh{oPDG{w9)LF0tfsT(e)_Xb;u;hcl)_EoE`<>X$u5IVBdTZYfTt! zzs)F#Gbi_Id(6`M48pKfLWl>#A@JQ!Y+)Rl5`ym)0t5TEP{3Nm><(_)Zo%=Y{s)`4ZTVhKM?PGi*$B?QN%D4Uz~H#<-m$}vb7&1>?T+*!uTXdyt1 ziDp&ew^z~hMfGxZ>H!Tw{@v5JT4FG>oQYq&YCvAP3j{UquHY)rE8me>l#Ly%7vo@! zU2{yRg*BfA9`mQC84Ri~f9)2l-zFgk08)Datfe+M>=dT>3T(fsO150CV6q{BCdc4FbL0hDWGo>j+nY!teLtnZud@bEx`D zH|{E@0g}0FPuBL(#8E4lMF5M?vO2#UKfAwXDHli{=44jo(MT@%EaqU-Zvq(UKia&- zgV}@W#B*6{?C+i+^k+<9nTcd?2*hhP9z1Kz8lOtbKvs>KJMw1&O_N-+S*+Ro3hE3O z^amk9b=w6Myhhmq6T=@O5u^$fa@GOmf|S!!c5hk!P#cuZh^H}qN5WFv7@_g_!=%IL z9%U%%FdN%)FbHAp#vggK1}-Z1+)l)jx5DiZ=w*eil-8{xPG-O|np5#*6&6Hky9<)d zQu{}L+Yx{QjP7-Qmg=T1VHX;PxN2GN(?StyZpx{ z-Sg&_up{dmXIq#>efs2O<|;bIU`W*sh= zR;ub5v>AZ4PelgfjymEr_qthg|4>@ZeZGZY)~ZV!GhkzS5yg#SqyQAMrRk23?S7g7 zm{L(LB#{kZTk4U)y^a!0@;8i}5SlRA_+`r@h=nE1xf4Pwn(-SEIy$(WI&%+uDj7Og zHSI>zjnz2980fP||FAD}0>U%ZHM#p~LPIJzPM+WuE_IKF1DBitvxp~dsrKSufBO`U zMSCIk3~BO*udC~xO`Ww#FG$+o3~wF3YHSGuH6nKZG1JK@cr1kUcNszpp!dY)1AMeU z!L(4CFY;YCOuwN-uBodI;Ev!1{eTc-;}OiV{Ek8_5{mKz?f?{RgInif$~AY|aL=%E zh@IlXH3)O3uma;-sLj;SY+5SeG0e)BK_hN=-*22B%SO%k|UwlFw^YhS(Q$lm(h z6b5PCZhvp|-!(d_jiw#=0`S0yvR0InU7rWEamX!V=jdPuY97dnHgDeQYH1}g#0p!1 z{VFIxnMm@+BKBn=`gG#r7<-2eW*##A}6Cb>r*;3w2t!7O#OA=FLrZ)0qa646_;w(5}5ukkq=q<3h!!-$z zCT)f4jsekVvX)(OQY5Blu$Qf8_D59p)~EqBvlnAyG@v*zN2LdWi1I{ zgNR|KS{|to7DW;5d!z}pE`5cz`2S%G`ih_?mY{ch1z)Rrexv2(;v^+}FsR&j>)gu% z^lo#{bpS%d0e>dvkI1j$E|Kas>gnI@Ww^u1{BeML-`#TUhB% z&Cp5)a2S@wq9CR4l-HDg0VxO^k+G+YRSv*j!ouzM>#;1df{+}Y>9HIE_LA`7czw73hE&w5)62|UXCn<@3oa640VN=+5d8i zwks?B4}Cr6$7%PxIyy*lnRA3;koxDNTdCMVbXn8-FwtZX!_2L)s5?|N zKzI(9&B|d#p409vSHS@s_^_MkqK*lkOqmBJ2UA>8*D3v@#3R4~R0Fd*~z^M3ZCDu`pnp9f{bl!!apSdh9IHf>SwU zVF-#=p#FM7Ft$Te-)@jYJdr{uA^MN*%_|F{H{I$o;h4@ z-zr4x_fHcVw$svf&`skX%=6=uDs(!n8um6z=1Eh@v-8bZ2N3osd6Y#Rpy`u)A`rwc z8EV2<91nbA*!w~}3$dQ~Fg}yj9gp>->w{Vww$g`^Ok*%rT`+wWN2FLC#m13YKPkw1 zv;i~>#AETD=+@NE@-a!^nF?st_&oP!~KdMQXM#ohc4o>c-Lw=ox@^oNH6 zn#AtWlvrWb(8!S#o?*Drn8YL7NUlPHcc0{v%6m!4+4T8(w0YB#=>!sTKmgG4o>M6p z6pJ#sLuhbB$|QoKtdyI_>q@pX0drtGTM9w?m4Vp5Tu*hN;bl_b2b(fLfOSu?9F$R% z3rNqsB}l|g<8~mFHdCNVa!HV0wPOpJp^_gf3&BP{U@a)w^>|XdIS%lmR9poG6)*p@ zb@S7JgC!A?rqWO54XH*3b!e`3z_{=ZV!vg?qYTdnn_d21j}@2)ig7Y&64?DTR{0gi z4m-?>8m|p&W~Jz$0~Q=%lNGIR4{Q767#6M@k7=@Kdz&FVdKMrZC{3IHo8@jSc+eej zLm-w}j|3tBSB(Q)3-ej79@Wbfx$OP0YH;KJ7E*n)b4v-b>`H6P$QB|`86*p%3EV*q zEFX6RdeAIDi??bSCxL5j!fyE5qH|NQnA=Zz7r+_fWqz zV6c3MRYR>!vIGTq`)AMAvOa-mYsH29^uq{$$Cn3g z|BG|9d_^_?amRCL0_B4yVjwZ}t&e>Mfd?M|2*!bjj?Yu3A9;fWEr4x-p)J}Uesg`H zy)XkTob`NBP7||NQouskVivsBLDl14RT&@2R&%-OIi#3D-`brN3*q8d6xIx=SxAg}-&!wn#5vkR0l z*GQ21{Wwe6AK5Rkxt>er5YlQ;wp9y8#Dm%iWVi0)J?3^h;(@I%hdL9G{Z=<&`h<*W z<{ncK*-Rkg=q)DnnV`wGIR0%}=E`^kJ_h4mX52+D!h|H!;iXq{i zgOK^^qp6^}DH%o|y-_ZBi&7OTyVJCshi2`REzTNHY4>{wq)=~z*WUA(#MQkKsCyHU zpggA>aM%LXAej^6G=eZxz6#1x7G>6hHvfqPC&!>O~GN~=)H;rI@1oe!~9J^&aLuHa%K4re`1)jHZ^%Hs|Xcp9>G`&5@9orfD<@C z&9mbVALMCLCaMg4xMWeg-GB)(w(4Ptvhr>0EIxVkh}@s)-7HY=ztU)6wbwI0Y70Z) zA^{W*6fh`IPRA7LXHTy(Gt`(SlsaWnWc5o^IohT$7VJD+uRIs{)}bVbnb5>E@IGze z?@uWz5=vqGSk8qw88qUE@!t?t7=XZ9mK&Ve>Er?P&XU^x!qzMf>zssN5fibqK@ilk zfm|$T_Ug=DZl6aKa(oI;BK2^~!GYyU2LYiy;?ES&PM~Da<||_MANQ8Nxk98GzH=6G zFdaX~%#F`%Sw^=WuC%ZQOEov9fiPuPxDGjH*~B1gucyZk%`6=Url~WEw$S;dco!^r z#d+BPLr${fhdVHACh1gfs0D&tU=BFX@3Cvt4DEI)8w^s~k3jTEO8`ypNQmp1a95Fqz?OREy_et;nU%{OI7idqg8wY# zy{6oe%{V>@L1RFOl}?(qQwC^jVZXA`KUiB^JjiC^a6FV+&Nqp`i4iXFpJ(|0P~Bz^%z9g02Y2!Um%1yY!SdW;D-N3 zdk8a`CM@uFK1mP0AQAu7W2*eWUd%4#5R~=L>Oq_2QCn)%ln_QK`%GKoPJvL=DWD$4 zATJgWAVj#6_BaYrwUSfOy`K4IvtaBwhf6bva` zHUOaYBrz<#!w&~0?PVYYxXOaH^8=saM?uVXy7Hs)03B>WpB$YkM{4xl%V#g}U~cAm zOs%oy!`zc*mcnNd8G}tEH>I(F!GSR3j-nab3ysVU9xzaz1<<5^fBTd$WdxpAYPlBK(i`d8N!9lDYz`iUC_BU?%3!ROfzER}F^n%(yo(u>QW)-wd zKv}Rt+8=DoMro}w#AjBQiq4%OE7eBV&mC5S{7KXv*(o}{qpt^`%B4utMkevp;pY^c zIXMde6*MGAGoT-4b~(9QrmMhk01u>bO`?ku5moK8EJKW^kj-J{XneEWA@e4Hb*vYL z^^N@WrdN2-8OiY)d-nS?jF$|wFl}-lMtJA94+1E&n^5=)8dd?CxkWFC@L(o5Xa>%Z zSg0;wh+afmuV*O{Tjxe;1a)9%6R>0@H~Li+90g$EG@ddk2(adr#ffZmY_A8YZ0;c~ zvjV7X%fbY;I@nM)Z9c$ZX=S;4vXpGJ{Yf>R2fLrB2!fX?1dlhbnwFR1C#$zoOk63r z7Q8gRhM3S(rzzvxfg-)RGZ?fM3Dc+?_hQY>%^vlaAr5vbgby>~0bp?q-hH{~5~TwRa_wHCWNB!#6# z|J%!dXS={^0HGO*soat20Rg?g9uv?21L7B4>h){A*iMN6K41v-LG~6y`eeBudjt~V z`m|e#%h);ZuAfQqndg}!gD@!i?X=$z-7y3HFBgl6ysCYa*?2R>zen5&aR`Mr;vwpK z!48LbEQ?OlPGKw-^l-kd+r>a3FhDa+B4q`I+HmRP;XQq*ojhh(f>eBkz+ z03PMm!{#@=rVbHmtVEc|l7OOmD`%%8%n}<;oXMBvJzcnFS-JcNGzCKEr!x+mvzpB2ajt8E_f>&WXjV&@iYVQ`cH1<7<4N=4|gM* zJVB`!oNuAmjmpw?jsjqnfs0mh%&kTU_O{>JpJnh*Z=0Sz7;WyCls;?5okn1GN(jGF z*{vF`_wXLVAqO*o=>tA<5Iv-zgql3Z$Q2+V@H&OEMXkg|O<{}|8fJm-FVcm=wryY) zS6Y5}WWa!mfUN)zjn1bmzaBPh)ACc&_xeVc!up>1dJJq6Iky>$r5r8}gWReqLv{Y1 z5n>QheKrWOwLbN7(Zjq02_J3VQ!XrLEd7Bw;{J(9Qh>ML)K+)lWOiUDJ5P%Z8l{Z# zA}wu9GU*HEI9MrTJzWUP>^C;|(amJUMR5Lme-_KHV%42?p=(G&bi&%@y)JM zge+|H?icN)$@w~RwvhhLEm;H(mkfjWOt$n12W}1YF#1~F)<*+P#6_upZP;!AMPSeh zv->ms9vrQauwElv?5wqLqN%3O(JC4xZK^?@ID|2Y`$>?X-F+t(1yzC%Af?E1!R>cK zL1;LEWi_)dG*wA&A81mh;b2L36IBvyKm@mtit_9UGrww2LdPp|gu9h-!yk-lfdongJ9uMZ5NDBpQL zz9>ZQZpf4gxoZX^{|=%2g8dp{l;inO=6;micXR-eECdY6G1f;gSmMG4a#miGS)IW= zq-mGL>&Z>w8>R+9Gx>y-(fTOFfj+J%t`RC6ydGhI^uf>&UQ6+Wpawrm*1y_-{7Z^O zo}~XKmCqebX$y?&qnZCyV(`!ad$9a@7X7%60kb!Wpz2BD>ygu3YZ1a>rb**v2>73c z2%w@BI3UohD6!&07Ifyim_*rzjsbAU;3EX?XGNT|@Sj)(;&DUtq04CUOoqpNA(UsePLP-lg^Mc*Pv? zV7+2LqhSwI8P8sf#LWnCj3m!#PHF9EJp9#`nUcZzg-!Zdn>vOp@=VMyUhTwcLGy{BeTIxrP-Ce-vxNi|-cFDxL>kPF+{|PxcRR@gnTq4fKTqfL}Yq ztp+o&C;G%{Sw+;h_eGJ8(0@6@#Jl|aWUS*x8e-y4ZDG48P69|~PH04n6 zIp-8}@!wfz=^47EPX$faZMmX)&*WA-=?i^R6QAQ$lWjCq}RIB0~G#a#KF*VY9 zyDPEy-pn8NMs99CfcvzAb6Uvxv$a=d*|C>Sb@|1Vz@RZtpQdzl7py~s(JxlpkD}cz zFC~02h4?)mD~Ua>t&N zoHsW*Q{H$h@7UNev_T@&4_&5aRW6VGzSsaAe4TW0uK3bx$U2aicJn_F`On%*+zOnZ zZ)yKl_L|>@*pGmR?vJLOclDA*rSudIn}7NlzVeK#e;?ltMm*+Z=;6yDSFU)s$B5Yv zsL*|FbIu zpwu$+!4)7Vy9tQ{cS=uzcw@k3eCEz4t^}#P`(8AapgE%vSzD2H{aIVa6YKS&6R$-y zl}^zQxjk?#+E13Qd!O&S_NVc{N7|Wfpmp8N!Nu9-+g*4b;wBRiXrs1w`p^JP9UVy`?Kk(Xuz8Cy~m5Uy3 z(U!8NFEYBv3&Wa8FA1%O*(epW1?;)cZ#6w{g=&o!O{O_+Wl)HQ2kJoP*x1g!g2$6W zA(M}>_jO$)R>{}4+5Wh^SSnpx<IyG9tAARg1eVg1T3x7>!)g6hH;d?0zukRYo1# z4(ATE=Jl$?Spn&Aj7Aa3()m^IA5t@y>5b_7$UK$slECE>?o$g1E-%FX^gc#z#VYfc zAC`t9pV-)N*tEF6E5A~2MQnwmAq9r~ZTm9!Gpwf=U&i=Js)_YXj=`iYT)#x9#b19srXF6$Ft zqc|eoDZDNlnp?JST)Zuv#JI37@|*3{2I?OD$Kx%E1hOJrm;SCq76vY|D$7}h-(+{{ zu^HrVNN*HvRopzi$&YDZ8s79Wxr|iae%rmECTR-xOn*{$`r+rWam+hX)Jnne;IQDG z4_|bW=Uu)5*dtlKOrsY`Z6=YUsXvWz6T4>P=@PfB9;Z~Fd5vHzDjXIxfcy|44W;p& zj*`6|NB?Ye;zJVSDfqvbsRQ53^JX7U-Fk_x=D9s!&l7?%K5(9vSt{?S%6BFOL@wq^ zKYZy0d`Yjto7bHEAY{C5q!u1YoTLXlw~B7ANPIue1^kW2T(Xcd;eAx~^G?{-cw*K4 z`NdHOF__k^Qw0~p-V$VtIuxCMkPcDk-!`nR z$PolTRJa0|`geP)D3)kV2fB~KpVHyWqErjrQAa+n-``Gjx;&(*{UA&JhHiy6UGPxlnHLw0@9)iy@;TQxM92yxn>x(2-+F59 z;_~+JvX%Hh8DZcA9spkjv$v^i@;|zj5S~|axhW_KKyp9vO1&Rg_uSUjirP~@CC~ts z7D&r7-AOL{<5D_44fBz-H#4|X-+11aaPV@ zZ+Sq>DZl^fPi_4h&6UTEWv|?uc`WDGR}~m>uF%(k*{?k3MMjo1vx29xQtww5y1!yk z@%LN>0lU~n)f?d!Ln_a$kQaGVZr;ul(Dul?o@92r6tUZ7%GT^IQ{*@>W3qQpYGhMq zU(mhs-^Rkr;OQ&^8@6hdu`)S8HmOexV2^Q0S!;`UFLXL4p?N*}<@6+?0etivx%2zE z9)DLs&9h_YPMx@BDFd>up}U&;r)&7VdoyIwIt=WKCdGg&-}ZHT@5Y>MRJ(`WRlxKKT+qoaG>^NjeLv)!qlXU-JeI7f#+ z7<1C8dT!iD7>*=t?|HM*lQZ=2@q;Jp{^C4G932q?sY5#Ak=gPPlJ49m`IHw1 zW}UK;j^>X}9{QRj1;`0v^lT*QGp5=jf3FDva;_qN+B+rDQBzX`2SYlCxaf_RDfhZW zy=xEmm&!HkLyvKpzRv&Y%pMVZ_QZXszhH+Wm?W3Um2y~GSC>zUyjY&0Xq4~wmt$Bx z@27v+N9((u);~YIc=)4j8(WzXn;5~1`+JzDKtT(D`~Z-OicCQi^Z4z zQgNscK!_hCR&c!-9*l_j+mVpk{pr1G%A0R|f-Ydm$HK`v#?DP-<(JLQnHGzepUY9E z4tARFd)~e3bm6yA*EhC=)817tzim%gY4idYzVYw>e!v@jgL>z4;h6Z9-{3H-PaTQU zxq6B>VbQ{qqSYr(mBkqha$uULpB)oeYYk%c)5vPXMac};m!Z3(2Rro`SDvGybvcImR072b=XX--8e6N7qJ$0=8|F1d(qJ@A#V|Df3vo~LvPJK zrUkc`bLw4rmE%S&&)EB`y(tS>UTf#xQwK!ZLc~YsUcD12dAuPXFB<$(zNXpg&QIBo z^}ORWkF$dowO(t5x9|KXH^zQ*f&<(7w}}5D%JGQ+>7Dp!n{2dRjE*7uE;?BS&mDX< zo#S@FHgx#GrRGYtt2e!Y^U6vt_@PO!W|inUxeRW!qdekq(F^^y3ycq~0VNCHvd;DZ zC-fZr_72>EY-mvNdQQ|K=W3z22JLevs!@wDt@`}H8IpQ~3#$9#!BEOa-`1Whno(PaV z4i?av2o}+a`wuFR`^CfNUMQz$Ew|nCQ!Twuiu@8E=8U+Sg}k45Flf|lk^oIkTmA#U zSJMi;ZhNX;4P+g46<<*oc3Ehty}5W>=>S#mtLVR7+6n$WbweAKSXt!y>{>LF=i|H2 z3lpMGEA~zvTtsyVWm#v2_4xx%9UL9 zE&a`L>tl_de(O=6qW<0JZ{PnEQ~&A}_rHwQcKc?QvTdh9U%Q}cd{;36FQra@4*Uw; zncZ+vKPR3kZF?Hjo_)>W*@^j!DPh;Ca{mUda;cWGN14s5ajN>FN7S%+E#VMLPeekS zdzaBWJ1fz3ZQ22Uio*U>R^ZM_trLy2Upq4~4Jx(C16F4B*{V)aQf8nFbKR-Ev_JQ| z@36;DSq@+EnW~#j=I-zAY7Khcn*xmgczkEE0?wWeJa#$w|RdXbm~4(lxsQl z{H#QjfJC{Ok^2m6b97r3GK4Z7;*Br5*TReqNt$`MkY_Ekk4ShDo(W$lI3JuxNKH8R zz?Mg%>N)@2AXcBso_y{KApzG`+MulJ!1ncrmEF7BZBXwgvT@6QY4Hw`!w)`1%Q>pA zh7vOraR0e~5YS8!Pruha<_o{Ca`d>9mE_se_i71)MV`tlkE};LH(y@j(EjsE_?hlcc;kPGcOtx=^(a+LoO40z zXq?F$t1Y-|%l_*6QO?GI zDHrwH} ztHjA$3+5t544bj+Cn=~`3NB8ep%G~q6 zh{!oZzgqjIrlOSS~HwjwKTI&Qspx~ymR$b2KhI$7?j^&xa# zqR}z0Hp+jrvdOO$)!J%)%mC@)6!z4Ku@W+p#M31Esdu_6TYtmUVORBS%nZ-M-zt{A zwLJ1Ljyu~M$C(L{E^(nX3Horb3mQ|C#lh6gRNRfV;&c;wx}|_RFz?S&D*}Y1)|wn1 zq;TAbNwT{m_Zh8xqd(YDD#B-@+z9h9{4H|gVafPL3#?Jw`tYplf^*5Z&`gqb)! zK4Mz?S3q5)e7;)v4Kqx9Yi7?fr1<&8$8S~XUn$Rz7sdO=WdeK%uYk$3QU&3N;O5x8 zRzTJqy;M!RK$iKu}$KogSM=53w3MYo*c-EyQ!`St&!~ zr)g)KJ(-_Ur*d0qsR5iyvsx$9SD5zguMc!PP0ljX&Re$0XOu=dAz;)vqG5)#AmjB{ zrq6V4;aTbrM*y6oku{rhkRHjbH$l}ay;l8v;zh3@*%0ZbX71loXK7Y{0b8jo* z>#KN)(?zIK&?)Oz6C^deNPPuX+k5MvH-f`8pZE40@?}%@WI-+iSL=e4%ZA$`xePvq z+}jE!MP8I&yk~22Ti$;!(y#fB)wnW-?0iWc${WymEchfZU#rJ~)$P8R_yyy*C8^{ z-oa^X<2U9c9b%TPpj}YF7&@)lXyYwE|H80&5fdN+B-tD3brtgPz!jfd1?iu9wX@E!<@kcbtWubAYpcZI_ltj`yf21nO-aN) zs$Lf7n2UTL)?^(zBqcw?tD#m1RTx^!U)}s?oZ&9^aYgB9p~uO~JMe z7I*(>U&P!F2H|c`Z1!E#`+YgqDs;Dk{Ao%-cjwa!r>saUOmf7IGK)3$oLpPsRnyq)Y5 zEc8=WStlD?Ju_i*eGA;D5@mkC4Olzjl5#&*r$>fMBl&~~_hdd!D*VH>)YggYsWMAT z(?1SgqJ5H*A2jgk{c(i5g*7f6;~&=83b^!t*ls^nC1)Wh%qOm8iTh)~{N0|Lj`=?| z>M4=-KeUG5fBaEspD3mIlTs=nT$(4U9;?wZw|#oh23G}_?hiwFwZ~E|-kTf!kn|tJ z;@?POe3qoYweiQb#1p>Kuf+TqMg}&x0Qx^z!}EvvI~5)6V?|U9OaNRuW>yAVI(h)$ zi4N{35^Zr==~@4z*A|!R>97?VUM)+ZKk)ZxA!ro;e~do1&{oUTNQe6go$~2Y_OTD; zpCV6{J&xqZo~Qe1Locr7s9>b~WMs&~`nUGc|I1PlK>tV@;IUKGa2aTyqO7z(!s8|5 z(eL!W?UAgr*^YE)y72o zBl?t|fBBJ;$0+k-^odwCTvq0%3h4lkg)%-qNBiTO>9K2`V3B^r0SvhG%ulq@(LO!K z$jtiVnw6R1$2}&xrxNMt7#_=EprgZOd}Kd>mHx-QCvW7(d-PAOXQY23?=O>0z~3D& z=y2(n{~;UCiDh~MN%`aNzy7E}{I$*j zeq-q{i(rNEq=f=@X~V~v@Be@)B09iq_y+#L;-!w#hn#_mi?HxdFxqvKZn#%b%fd%G ze_5Hi zec6~~oCYwz_l#&z6-vPb109^@-_f44f02r7*;<-f*qRz?1e50DRB*LZ7prsXlTs-x<_qc zZ9oR`KPoV?Fz0&`a8f?DpLlRg^mL5Oe*h+B zvcD1LXsxFY@#GQ&LHpO|aqeJzlq!APztWy2>>qc`f2Pp^p5~{2q|q}2o=Ib(|0Rv( zaYFkgjpf&TtdBF(zo$Jp+cEx<24MIneYEtnfM4^`)3W@U?@0*%S?@39(E;dx7FoSetDLG?w@?2128cDt-L3{U{gKoAJQgk zHe?(=l_n+|MmZT-v5(A8A zVR9^`DGh2+9+_+mUt!jk=y!HjH-nul8XR0aR-d9yauI+5EvQ@LOIw}SpT4!FeLvhd zJMz!wJ&(tK$=^L+*zX%CV!nE-AkflrVxTtz=RQW_ZD2vN+cN5C+u))bJcm6te7aX> zbYtgy|H|Sz!2Yq5lJL=XV;$H)`UnxL2X1%cZ+DsavG8rCFXc>kzGuY6r;Hd$Y9ZX$ zRpdle66pon)=H?Q7kU&>oZtEC=-@)NrRS$3?&lPLhaI4xUMZt-u*sN^5N4<5%HbD; zUMvU%(KZvNMg!p9vQ5Th5kOAm;p8hEK#CJEjTD|3d<)6j2VlsBP)+0TDC6lMNu{;et9^A zAlMZaARs`~dqa9!(9V`UL2SZLaI7LNfPRO(t8eT3GIazEvGS4BW2v9YvM6@?;PU_JthEmz9crvj<`dgPfmR^Tg3n7Z11qXLLkNC${>S?C1RT3TI6XoPoCli7gu?CGQtO>di}Fsk&A~i>kVB+-LJWfOC?>K2!uOdHPEfc=X=e;v|6dWOUdi-*U4`!8r+^K~aqfH~)i$#5sbA??B6x6+qL+U-_$$Z|cmn!Br zGd)%*p>nufR9^Ih0MrkAAWrjT2lqjC5T?=4@CnrbI)8jK<#P28;+1r-qF=t`zb*{? zNe>|wonW)AbI%h!ufe-Hs|lljn}kmA6$qJn-s&PZEx7$b}cH zHCtXK>*@e2pW%o^;<$qs%W5W@hH&lLeVz9Y8maq})oj33MQ?$Blq!I$B+i+OpQ>K* z2d67wiGt+WTC|_ak@-{-z0i+h74pSef zA(EX0w^^wqbt!e}=uEzULmLih5-dx>$ihq?(0MT1{e9t>CbfdS_3Uh%^N7LaU8EqQ z#T1_nP~$g3%Kgf@0O}SdSFg61tz%w;p1NIsggL#|^5D+drQ~-zC|4ZOmwcJy_Zdg& zj$Zb15yf3ht@RbFq*b5?cLeg35Bq&7VN!_-9NE4+0X3F@ zHmqiCXm)fv-QQS=b@s6Tet&x832q>q?4jX$6RzsZI7ij8zhyOdF34x^W_Egf zY!YqwEnq3`qx>y@^V-No53eix9kI73A%gYmYmVM3JV$myJGa)h@IuG(<+;aM9aNFn z={=zHfm-X^fI(YHt=~?S0n%$l9^Kmy*C8p8!}}4^rdGUAFTK|zX7*#is#m{(fu@k6 z>;a)_5Bhdw0flh6qiD8Jupv;=2Bqqnfr*r7X-robl0&tBcVPy&e!8To-4J)Z977`F zBT}aXI+8gpJ$RKSK^yU=I)(^}j8iFEzW59Oh3m4j*$a;a?SWCG+g15i9CVQ_<=mjC z3`sJ|HOI#0rQJ9WdqAQNKOQd~6;D;MWDz7cCqsSo3 zob(mdyi&}6m5p(J!Ijf!8xwi$x}N}tIV=t~K>p4;RFDOZ*UJ+U`NKV$gl6*Pg-Tj! zhbc*B6Kmv&No{ifWtG_cnWpKNg|EdtyIKF(u(iV=Xl@Cv=}pz&cCW$`Ww z<}HUTx8@~{&+Y2hzEWT?>`lkf92M1-KHFA}PF8WMmKew(?z zRu^C*{%SyjL@m+hsdQpcu4~XP9pPC;;W|a6d{3T4Bb6%Q$*sbXPqf8m5fTh>SQxxC zWfpn$g0gZO`b}}5O)lTta9~gsLNY-j$9a5z==LN8sS4}T=w5+wZ904+JtA>L0NWdE z#d6sT$k{?ZiOvSYA=oU!9%6JXkXA2Z=zj3(*F(h&$hdEYJoYmNqcqRk(N|bSqo~be zUkAJq(lM#`DMp`X?`@}ENWO-RH@#&2xF6{2dWmeiKU~yLbBO-1Z9#%FmWD-|vn+Oh zi&Nv2dGB=bHTuTM#_9NO479TOA-$A-vI-lLiC%qR34`huPhnjY95mcawbn1#og05$ zkPRAqOlRA1-e8N)_Trtf4VVL~-QDRnv{Xkz3oQp4=tGfD)+rJF zkpqwx*7Q)d-%v;>5P7GgE1ILSV|c885RDG`q9YCLlH3g|G*uao{^5}f9Q_Q{Hr3#O81|YNXC#R=yqB#K9{_IdbleV z&&$9p#EW~g04kK_xBG$LOf_{={Ryf!kt_2oWyNG93LKz}bD6Q&_lQQmGYmLWqo2WA zog>vI14T&_M*8!`55*d6Yq)YkSzv1a-VaLjObq{-#b?~|{8P{gOY5++)?zA4B z5SdDkc-ElE`_u~B7^(FO0svtOPq-W2S|%L7IK0ez?i_PjgL(fB9Sz6URLWU2>SKmn zSF3e3JbT=Mn)i`+Uh?DtV5;=UGxw`s@pO)a2iAET0cB^-st`XY9&+1%)DTuBWe(=_ zQiD6ty4taqqGSoj(Ho;xd1q0i;hkY%F*GywV$T{SolZIMFUTW!)5-zEC~PGG*IUaI z9SrZC(DA!ue7%p~W;3epp~H)IT?_XD@3einDegD(cz9uHXwdi!L!a9`F#pEOy0;Oc z@Ljl2yq76Ff-fFCzg16v?RzbIs0=0AdL!x#(eNqQ9>x0i^~Qu=8tO>;P^oU)qD#o4 zg$Fn3t7Qyn(N9@4CpEOoX+_Q)W!<1QXMQWz_b+=7pz~o)gh$sTcd&*%KhI}5UW$q^ zRnQtwMbD|C9ge(v`$pDg+$JFMjzGiOL~A^82L*-dBU*``*u;^4R*@+c)Ilh>!{GU} zbLlr)xRY-{Yuuc=&%7?AeMW{Df8%8>qqL|wA061zE~2Vt4%ZdoZldutzsrM}*<>9! zGLBC4B9Scqc5ZEMoJhk!e3+doy|{ZHy@@Vb6h*hGuxa$=cnLh}RLaIDP1XJ47fOlc zykq-l=X}1DeOpm~gDYOu(HtH6#jl0TT}Dm4s*QPio4f)<7r5b48Y+c&GUKPVDo(N2 z2$1=x>b0}pK7@QZ=jxjmjqPuaG+BFXW$!b+ABwEo($4QJ;K;t%PmI}BZ?OP+4Gk;Z zT*>Nt&;H^Io0swQ@o0~I*qqOrjfXcRYQ>FicEJJ6#ad#2QkBw`H=$m$Nd49x#=VSSqG$A;dV`kn((*N*$7==j7cwS@?axp`|0^ZT^uIw# z6|FR_RQU^kZGa5`#D+lB7m=5m01}YFMI8^Ki0eNzKK^Eqo{3I|mVp7JQfwqNQ=2xF zOGz|>n^o#Nt16nR(F|$S@D6}F}veP+Mlyr7lsAY zlt_9GeDvAs_M}-+K$*mg(mzt>u24I|%n_%lT)@3+9?!s@W~2sZwpGa1BV*7hy7icJ zxeKBHWGtp;D)Z3=heMt3{oWp^U?9FuJj(7SH~79tr=fB{tOGv~_vh2B+eAs zQRD^$J9~v#6h?kwsX*eSgkxd+5=VK?7Uvm%-ZMNix*2|Bh2W1Aw+6uj^SMmRqWMy^ zrSNtvM@C8Pa~t$XGa-et?+(z^D+IwW@VrNXu?RTyt~7h}u66Y6U)UBBEhX#CRlD@M z09q=a*vW_3)p;FiW!|M`7M-B8u2Rb|+Un`1J7*?A4o3u(!YSEO#M^OA#e9}oM>kb} z@j^ToDJDtHVAz8j{NfN};%bK=Syr(nH-e`;Qi6aURBr0XJv~4OR|6eP5@&-aeO-j{ zk&Rk2nrP$9b2%ec-_==07>?H4wRpgA_!ZN5bdv!}Z;~iBf0zP8LOHuI;}_4%joQ{r zCadsoC|AA?YY!AKZga0VJzj|3tG#}IhIx$7Fc0uMDu5(w6fhptkdrSb?R(TEuD1OA zTBiAHkH)A(iHI+F15*fCiH?Z`1m*gDOUk;q`k!RQdfG0kVG)WhKVjp?1Vd{1MOw}l zZKFLptf>_ z&EK>~go^==JS(mJIc3{Nqs879yM`3rh9`-e8Ub%)pFHnPl2mb3c7}|!Wg%z&UU9m| zwuUOzGyGzDPS{v~W1vg1hw(zW}UEhe!ABw`?h&+ zL5W#!J0W~*F!YY(g-8O6@WT!OVPt9KF6oZQl*j!{DkM>Fw{X{GWC90&a?@i5-vEhb1@n37ydyu!4s(^%A% zi0!k*9==Mc9`$BLYP{SGWC9frUEDjmL@*+!k!iwGPIb3~k6l6CMTL(y@IYJBeYgg_ z_gBV$hJ(-OoB-C}Sd8C)%7~ixaQ#?}GbX!o!YjA+;v zRfD(k9jH?;RpH2UrB+c+bP8w+4khBOU5@{ZSF2`AUBQAScnV3MO3qg>QZ$iy#S`ZX z8X@r|6SC8d^?sAavtzCtx_N-aN+QAx2oc(lRt&T;HFqZ}UzVcD@x{3u^#v4; z7KAkOb#qFJ-;lx-hVE6Z@Z*CV%Ka9 zmc_L#&i$ncrc|J!yPKXfHI&Rbi?SR&saXKfq4frC_Oh!V2>rWt^BabI6Z~pL{Nx&K zzR)03o3|~%vtlPbtg~GRou((Lvze6u06Cu5Vxyn&A(wrBRqHp)+VppTy)3h5`t&&^ z{dY!%xU_H(5G_>GF47Gbo_ch1K5ruL?!(+5j4vxNcv)0*T!JDbES(Py<=d-BJR9ck z817jMO-jyD0}BrcyZExwS26|E+B|9j7-N}mU*2Y#z$;a>d}XD~PPAzS#qq5N24ud} zaTBI*DAwHoq0n*CY0U4%g*bM)eH9J$zM(N*Tk% zgg~z}3UmxT0IjS!SBN1U>_rhX3vQbw-(58PscL7|o|%{G^izS`*Be9mcun64(Q`#d z4PVZL!bJPeQ9RD;;_Udt*bq7N>{JvrF5L&_A8y`%4at1@K5%)3RR%k9r<(E%2cOZ( zS$}5@3+;ew#{~m^kmCl2fLN3yj`qKJ2z^8Es<{jE*#`u&)=2;dB$ikmNfA^;XIUzo zt7YAqYG4@z){?|yaa=AXZpy(ZQusaDat^%H1B?N2M1j+UPU@XP$6NVmyaeh^xZkN8$y^rYpeB#X_Ad;-@rbsUn-(;X~tAR-`~i;;B0=Pk~= zW3VQ~&$|)^2eFS0?=ulC4n2x2Kull!x><21khg%OeExYNcO*-CWU!yL|4n8ZQe&md zG|a*mXH}ublb8`a%%KOV50rW<)(}M8D&r1+M{sMfSfySRURvcQ)ZWK>^wSSF{>Z8) zZkO79)Sk%2yR98CFADMzszLhE7C$qQ8NPVm7%beGed=s(pJkf;e*YTt1ptQDx;=nM z1*>&p^+lYU+8%#s(02+hX(g_^{u%p@Rk{HoOXQu`VJ(&%J&Bg0EWNL|f?76Eoxc!& z&F|1Clm;Yy0k><^rD^U=Tin^@i3W2hc>$xY+g0*0yCCj(pTq0IfZtjmWIC9I4KGT0 zlGSTtG6P$AQct_7=#fwNGrjYSdQSH{TV?);J}Zz%CG@1| z6%nl`#CdVwH`v4bGQYDaZf8^;C~e7q&T>@+D=piu@H#30w1*(t=iVE7o#q)uT#`K? zsK~&N6FE2bJ!nN|TQl0F*;%SIZ20iQQJWvR2wd4J`=V=W0pD%aagOj#5FW_Mcg?k4 zQWt&Zx|YHDIvu_seUOwHPEKKhZM>-9qXs9k#7Uh`FR{`Bq`ck|`|<7K$vbR+d$!YO z8vD!+fblol3Sa^}?nP)TgA7+%8rAwTYf^k4tj3AGjmHt=GM*-TN8rCGJRx3j}&Ss>I|qll*((j%q1vE zOS2tBZQFq>uhB4ZM#LuM!9uJk?)-StI`UZ18AKC<;`xV$Pakf9Il)~V2207gW`3O$RxObA^_qX)2An6y0p-lHBb&HFk#!!)?2Cm$Q_VQ< z)#-CpsqF^Hm9$QCdF_JoT+{UP?qJR%HWRQH@ET=j<6KaY;{u&Tuqu)$FD)UNT3;l= z+dzK31p7kaDc#P0mtp?(61GtZo(moE4skE(W1OP)E9hX~k_Ds<`4uw7`j1!kgbkqJ z)^tjHvM~HXWOePXXp{?)$|;A|N)Qj`Zm-a_d|8gFg84o{EwLjiv#QY0H!|cRntEG(Zj(&3iZ#y1u}lK<1YrXH1MDqp^!2Qj%R|bB*hUaq`5V+@f_u zJ*5x^2o5tUshL1cv4R>N&WMp)e5_rW%Co6X{T`#5pb?7eBCy*kNhU0_&BQWyjAole zcK%luF+8K`{LX~RAM&s1!=3noQ#z(|wGH|Wk$FAI%3D<|@I*3mKo{@pG$=|xHLDM2 zakP+ss)ABssYlsbGpz@}q33xr-%em-(vFHZ{0?8f;z8GchO}pvd3wO_DBDs4W{Uny}q}LKh#UY{xH~^HXFW_TSQX=vTF{(tdn9 z3KJnw-&=Mt#U4ut6xbN?jVBl45PO;^E1Y5}K_*p#Y8y2^eb2=;?8V!j)zIaMz9Ezl zff}lU3FBB6SXuw#gvk751WYjWp!^=5^7XI`cDpdq!iRWBi>L-;%5_U9?b3-yz&_J?#{0 zSEKRwBJMp(_A;=5;W8ZiE3a*N7rL57!xp;ddPdC#aMbd10ACMSFd`muSuCrMtA zz~f_;9Fjp9{TeC*T(8gtTPd?Ybm<2*n0x3lhKT>zMteUt7$Ad(mrFM%DH6rk zfgFV;m9G=PPAViOq!T4ni|tL2q*$evkSIr9C9_^_)YI9vpghyh7it-QdKb9=TJBEr zKsDxzxOZ6;-3_W-GrM<)@Rd6MvNGNAJs)ia+6A};KQV|;U`63V>zJnB@Fm0?OW!wW zF)!xBXBzs9(gXnfhE`1_BWm4)3$J0`T$&lWHg5nLZd0Q;V?giX!8_=Bffg1QF?l_#7EI1H;4+8{sL~*5p812T7&2))?=t^3#SHlY{F3 z!up4;LKTUfPHHSw_fWF>HBKaB)>+eqU?&v@qT}PNrD-j3N`|({2+cV@W4#iR60Hh= zLp)X_@Iq~6g$^Ts#6)fIC3p>4Y1t%awENrB&tKWy?_W~dxY|6^Ezi9&qyG*0V|?6p zgFJ4#$H?lra=%E9ek&>pZoRcIL1;k|qt)Cbqfwd^ZJj1s%I(@x!MP-=;8_hnP8yc! zxkRo)9HS}|9D}Q)f&i9Ac&nI5gM>dlcDBb>U@4ORK>@6P4iTf-Tlj^y@f9d`E&x|* zm#eo)n_dRUi*Vk?$O|CPZZbY{7zE{oqNi*;FG&`~V$wCw$ zs|v79PgM|qVso%cHqjPnSBGe2wKLbtL8f=K^}IfBZ!(s31{_OAt94m^jvH|Rn=rh` zKpy<0gB0Z{m|p=8ZN$C z8?4;EtN$|{@tk+#Z|F@w<{-~5MBY+{7g#$$GS8HklqdlZe^au;+WT;Jt?2iCZ61z2 zQ_T@{-ulXYXzOKk#_iiCrQR{OQCFfNQ=iBry5Mjm$HlYepV5Z-bIyQp{G-Q+aOn~hlgxwB2HS;4Iy zIWOtJe{Bupf3xq2eoJ6k%^y{i!|xDoxSr-qCkv0d{#_e%;Q}I%m90z+yRw}9kiV~7 z&5`YOb?hQM^Ol32Z#VeBNBCnIKcRwVdAymEMEbHiMg!n}ncU8;oGq&XuXkO!`H+3r zwH0OC{aE4Hibw>ReC_l--FxV9qo2$5Rovr@+QS`ve^MxBsrnQ_WH44(j)N{Uov12b z2ec%{1wmlR2O_(3f6WjKb-O^7Q^cJ(#uT;;36zTTwEwUu1jXt?X)ld)1|X$F*jvQl zU*`Tm6qUf*(`b9Pb~Rp+aX%lz+{Q+}ucEA^Gs%f@9MellzwfMgX8Ucn&l{5AB z1`3`*e=jtu7qn!N)h{biTHdjaHyV0`t`F}UOj;KjxX29$Vp}*70#;1_8Vq!%hX!_b zND9;HtDUTOW=Nng8&zHE4Ihw1P)wpOUlxk(eb|2}`&!dFH;03D?AiNF=RBuB{?1BD zB`s_X#0}OoMsen&pTDLPPvO7yFb6gW`WiZzf6G>RBO@domr|j5JMyDp*BXG%YYoKT zonp>Raf?e1S6D}iG1LC5^BMNMRRZ{M^lUklelhG_2-)&p&pI8NLEW6*^qV8KvbTxc z1I4P{L@9obNsX^e@pn2aAO7N2g#V5?bClc3NPW1gKe0ij466`yhmwSPB< zJ-4ZhUUL-P9YvnnbQu{q?Ud8(NuD$3Vw%}lYYy1F4Jq5*y}kp|&{PR{hKJASk}SWW zf9rn8gLjohgXU8M2}^z}WIN9XZGc3Gf6!G% z`K2fY$;^fbVL?eOyNY}0tmN3j8@ut{N*E@@GXy@P&e8qOtK_%g9go^L81Kn7!~<8( zJXxlzWWKTg=Z7)^(I!aeS2`iS9X)Fu6ICSb~wx z+AOH5*wq+e8x?yua7|JAZsWZ?e~mY>|Q#$@Ln@;^2YU4yhZP~Sh(s^@-ArTZ;Q6IK`?JgB2fqvg#8a8*0=#WmPUl4Gl*Z+`VPl5fxN0gJ&-&VCnv$sYe?a{J?+&Ac z`j<%@FL~!C3Z!{au*au!Wl77IfjNWf%wbn_YCgo*;_U|FYLlW-a*q`9BOsfNk>oJ8 zJ?^JhS4C_N^rGge?;N^hd@aRB!}-e2(j|ycvW;Wrj(5 z_~~A_Vq>6Y_V2V3FJ2&6e+|9TO&5u&+UmBbIIDWCaC84MT~IFW8RMhpJ&FH^&dg8D zv$nD2HPo_x@*)+}`WdHZWc=0F`lr{&zxhf(uA={iYw~Y7BtK;l*Rr)X`r)+kbk5)p z*NuP0)F5b{2zv4*vH9sx{Wk~fzj?a;OoE`{Gt$@Bvwk}I@X1>de?X5*V`HhMqxZvS z^65lib5lKi+n>=N=l(u6NKY4k1Z9%W7c>6lR(n@74CnQ&?TaF?bre>zQ=>9I8KM}9nc9n(nR(n#a7JSzEe zxGazPe($wjZT~9v1Km4$5{`AoOn`6zdj;Q}% z@HPL-vz!jVK>wVV@so2m?SDsr`2Y3UC%<)uN0DWC+_|yqogylF#qt5|MC5F|A+HC-48eUr!t>> zT6h+} zg!@lue}%_s`6npIRCtSv#_5E0b85#evj{Orh9$ ze+3!?Yacz1X_mh|`wMEw zV}pK04Pj(@1~ueyiemmh4>jcP_#e7@Iu?)Q+nMYB?>ztgar4K(@HhMqrXT0OGyGrt zkN+*7|BL_eEdB=rGcz?KfQg=ke}#d8iQ&l)j~YPt1l7dyAK^eC{#)Qcj+KYZF_;mt zGT7Zgm_d$w$y#xSSj?vMXBb{p3N>2l@mm$K1Z?*jwPoWhcTE`X$Rwc+g+Um?y{s%5 z!gCti-gd(@IqkapHe-6fRAqC2MwD!uu1}G7<n34_9SCPO6NBZ@B(k@v=oR8wkSA?pQ`q zL%YSnl#x^{>}d%oSv#pOB)I^jD+D#hI_oUlW9e_Z;hg(k;l|d!e^7}R+B63KR;KVrP|?xeIgalpGkdv|?#S_p9vae*;~otcX^#Sp|FKI!OGEEbZ#d%nc_$ z{)q8ip)S#vxcoZ@PPb9_luDH1SsyZxY(*@)J-LiZEv%SsS%|nZYQ?5D_)PTrU^<(dK{p zST8!#UW8rV*_Y6qE8G1^yKCug!W7|6dGxTK;cM!om;Oy!f3B@mmlU*X!*tvOH}}jy zL;4;hxuL$SWv>NGnUjQrZ8rjpy_cZ6Ty11v(b}aKg$~(S$_^p}NRgxMwP;rEO&P{T zi1Z2<{C35dKSz4mOys^DzR>FjHD@-A@YJiARRUxkUr;;iNFM8zCt>a-PdNvuu24@U zrA&Iwg4fM`e;PFxb)hLA3?fU#VZi4zNQo8Bi^OnMC@C6Rc{90>XOlD`fMQ(T$w`%= zjWrTMT8tHp{UA4VKyaZEdW2)$3now2=s|L{^bXvImSY{Jz-D49dVn)eq<6fqaX3#Y|+2&5;Lf7UAb-%m{j5q&X_vjr!BfK=9+ zd8jp$OvA*@k>ysjk@KtOXNzA29cR3$i1l5=ri$vkseO=x@zv{Qv}}397c_)|nDx=F zKr(2_VOV`4D}pLWV#f%WRT*mfpz@Lu##?6|#d_PX6MVllz!w5?z(-~ze&S2O_Yb(- zhMyzee|Y8v9QFn00Ng+LqzrtBFuZR*vZ-y>sou<7ao$ALT`099k6+e}2YnR~GiqTiz14K0ci2>SHq6_0py=d6&80pv6WSjE{lri>9h7XTnF~ zgEzm*9z70Q?F_Hf;buGKFdu%Sxb)RE1ol*he`!|yUB+=9xr`TZA_{auSSz7WKmt9U zZLGWBR(*sFL=T~=I$7w-hEzbxB#~~SJMwlxgba1hRX}?VbV`gh8K0OELS=u~KuoI( zuU;32m*339_;pQgtDgkjXaHtdHWAyKg>snJv3{c1J^8vb=u658Cf{*+So}`eQYaovYYL*9{%{= z?!DT^hu|Yw_VxGzab@YEb5aB!^jAj0f3JN8iqAMx#^IChpj$GEA^PizoV4~DZ{|tv zUl@KgmIrF-CIvY%WoM15Ls+b%{GT$Y;B%oZ?E^X8Y=>EU#lIA&6&&=4PB3$ zK=n=bSu_Y1;Oir|M3#n@5)ChNzTb8K_<~Onjr>GMFAQ1A3cmKi2L{5hTZz&be{s64 zuz3?1?zN^iREz7>gmSIt6djYVV|*p6GD{C#|NRszZc0UhR}qF=6d7+pEGy%zhTvXy zX4q^gFpQoh=^*y&UBmSVoeaypeN1>5tz1PKUxrmS&Zv0j@vPfQPM307+AIb2?F-f; zamiGanQ6nSw!C6=ea#dkE*vtce^ASy?ARzW zj1MxZG^-jX(+M_I4P22n?^}A<6?ErFU09NQ$={dWV0&2CXq(A^GZ{Q5#OM=!1hI)<@`w1KmuNO{N|D3i4xQKdENNGd8w+S`=eTfCQx|^j}@Yg<$xL$OVe*<}RlTo-r*5Wl4 z1S4}o5(k-L63LN8=!&&9nU<(xL?P9Jk}$rNbDCAIt>$*Gff`C7=S3%kNE43D^%^@W z=6nkzkdS!#QN<-Gq|D$mu+5r~=mUIaX0{#nUfR}Jt8z{m1+`C!tNjVz5E?5o8*(3v zj0xyVT>F@q-ZQVFe=mH0xw(n?tx0#LmltS66IiBp7@asYx3KxE5@ejrI;ZU{KxMjZ}jtOX2ivq8A@_5k`7+)m?Ra=e@VHR<)oPzf22^VB$mO+ zyr_xv%iw2yxM&m-rK*`Lt007CjAS#mP*xa=qpG+;&#Rfe@u=+^OOAj=UrC`T#IK3g z($wrdLCC#EI9qkLH}-!W?X79Hy+L1j`9{ovPs9 z!}{J7Q8uwST4CV3b=C41@}aJnd7eaGPp@!;sCf*@f0h7w6+HrEvDP*>mUbGd0G5D1 zDO_S!_g*Fn%WgsfF144qwwbD=^^<+gg`0BsM&1_7N!OyKt;yR^|@Re|Ub;SJue zFAeVAr1B-w7kYn&BMWIP^#A6b&Y7i8trPNDx~11}M$~^Fs%6wVYo^ng+=f=*-1?oM zfx7=%B8g5f$N@qfhD!*fb^>jEet>G}&iP&>(kW7dZ;dLJRt8u@LdO}~ z5YKrK%;-i40Wni~>I}1W3g>3ImFMLZX3Nm+#$K;ouRyl9y{0m?dQBEEQGyg+i_OPe>X^{!XLE}WoW(}M(?KV3S~aB;G6v_ z_`1lVkozq5Hp;Q@hf#TIL73wRg0G~z=BOp{2gFQL;nE=@i+Y#G8!ycAfu_Zpfd%fm z_od+dGQ2YCrocu*?fp>VLBc7=Fl4`=(C*p2W-qJ;Sn+=0^6)T99F`eke{WC1Q!0Um ze{R2HK{ggbmY|`G%z;~)phRDPG#mtvOiiXU%R{MuJq)j6z<(@Vg7R*Oxl73%z`W~VwJ@KxDce;d3(4Rjff553~wX9`129hzR}9n)tZ17v;sDZj-MwyU#4Lg+mQoMF8(OzuoOcG5 zjB8E8nxKZZ>CYJjspJO|imc=ul?}7&`AcOn)TlmFol4l{n*_V$FK1%g+SPj;f3fXx z3eW_{hw6-?sJ;EnykE^?*1UgY*RqYRAhC@P>(u!w1r*5*_J1Exr(_rMq{49v<7XTmxziOj1sT?c_UTl}}5 zgeAYRO#vczTjmuUi%#UyErcp)e|m0|d=G9DT)tkFC!e-lZ=Q;N%OyqejV2Mtq*K0< zp`pXaoxHP-`LYe7m$ddFCU#Dvt@5%Z&W`GE6|l4H9Pciees5Yu$E!-_*InLXyRLwI z7|&@^S{%f8_L{)vDtKv?CD*x0a{9Phl27M)6gI4k-drf{26WJFL2!0pe<^CUmBC}u z)=I-VylJuZBWGK3zf!(UK(kXigPU@~^0yv@)MRhT{sgx>x9~n`WHH>SNTc_M)-FCL z`yXT%KKL>uneTjC#HH~q5jrT2?5naT%OELK7+`HNP)rLf)e;;~5B$E_ENfvrhF7ho*X_hnx%B16G{^9qP{1GQ1?~5&V zCirF$Gug;B_f?AwZYwU1YvNoYN?V%ct3r#MO7+vv#j;I}?rE|lS~2a&1eM1sX+ZeRv&)U18_Z zFbeMyt3O)s;W*oa7o)jYL*HO#*o1LD5`wi3Kj{plO;lZn+GQe_ChLCbitNf{g4U9Qa^%9C*SrZje9x$rP4PD&%k zyl9OP&ou*a5%i;iW6HJkptvdXo2|d5c)8|lK*w>rE7sz?Uiu=(5Bn# z6O%iU0CYf$zs-2`6Qh)gE6Gp|!#hFDlh)|W)zFPpsOImETrF5VDi(gzDpt*Cofss% z2cazL?~WD9@;7bE6n|1Lvgd+j-$_d4dtEuba_>cc7raSW=0Wc>hyA5s&TqZ@qr06p zu==sEblJ^BwAW@aXPUQ;``wvg{7qr+1i|)!Axc>70Kxjb0?&aEYnabA1>Rj#?~ru! zW7#b^RxB7a#?2X4d)H(KQkY}!Vsq2dy>`>Eiqa^$t5!JAkAI|W9f&@f<20syEhyFH z*zDkjv6tHP?3JASvWa+?+#Al|UQpoy<{l9^YRfTl$br?znRe0h6>;fZtZxg)v02Y0 zV=MEkhoY}Jy$*~QhIhe%Bn+)p?80t>Iu7vm{@Zm8QfpzGR`yODo_&talwpES-@^4b z;X>759SYUZVSgQBKKeI(u}nUyC&^(OoPdjCXT;vf-^2^G2(bdu66~S4LVPnB-ND3kWa>$%2y8f<>Nb!)^#^EXMaW0@Hj9lpo2(-*0p01`6>f1s!qu zeklxMSk~Nd4z(W9Na3_g?Hxo?GS2OV5TK|mSIwpjD>I_h!ThX|_w{{f?n}zAn~Ob( z2a7DoRe#!6#3CqqMUyPZRA+_!iHdt&3a=Dt*UFG{wC*|5lTP}Y8-$J1U~QslNx~MY z?u_lx^{?pwVNoVga>Deb!kr=ec#H&L`^`INZ#t)<-Ye=X&IlVT)0awgzL!OqdB2~K z6hI?L$DchP?lDth%7Bz5Dh<_$Yu~j2AL-G!{V?7(T65^z6eGh@CNyXS4ehY@LNYI5ar`vM~I!`7m5>C64Sv*;=wzb zfbR<#M(g7bm)n}uGo+y3kriR05BfleqJd`JWgg4=?MzWfgX56GZFmN(R)~k(KTPc5 zq_(eBY>%hEc=Zyu_&=O%@|QEuf8J*3Xn&$wws*Eeym%4+;u)jM=di!%>3_q3T}`UV zyc_q;(RC0NO;2D|>?BVRb!adLBFX!hS6$k8sV_^c*4xH9)i8-DI*bz%XUb2P-(7SV zM3fAuQ|6WT_r5jN?4|8X=%I!h2+oK0GVKSrIN|Lj7qKdMtIoz` z^b()bmzU?c2|*uIEVwvrtAFcAgf`ziU4~y?n3r4fZ)Ry!JQk+9|0|D}`S~(j;V;XP ze>-A^=^yXX|4S>(|N4lT|Efv9ea;NSm!3xf7=LD3Tq=e~R?x9BKmK|8Rrr6^VKYB} ziulVUkeQj`?~P>n>n+2NgJGWTKAp_)w#<@7pc;e+)v(2j0u3diP$B!o07rEyCQak|| zH1vTjZsVWubb%B3!H&a9BI|aSS=qGXPSjnogRuP8IGs)H^#)7Z;5wN$Za+FKv*{aF zn{4#?kbbNfTX6`J5KnJ#yl3K>wz+a&j(=IWwmFaejL%c8esw=V21^}61jwocn=tvB zh%%bW`DKCa_(P8`j2&)R-6&D_<@8zMHOOL4od~$_ z)EBpt{dvu$ywB|h?>I@s(p$c)$ITIhuUDZy^Z@b8p0<7e9!>*qjnFBHJUZ&(Sby3q zo9I7weuc?S;&Cy(bz;$bBMwIM_HCVEN2lP}TjzIpXpC3)8_U^q%B-wmDzC#q`xVHL=3|U*-I)8A?&?WP& zvC$1n`lR_H^zeIf0GgnCxs{6vcGR8SXEdevp#ZGdXk?x~1qmtJqu-|C6A{RErSh;>7yP3%9 zKT9Vc0>23sR-I_su!IXqjDKfH(+Or&MWiyoxj|<(5OsHb9X)V!cU`p`aUQ+9JBE9h zzcr@hIkzvnyJ`3J4z&NqNH(~f)9ij)N%#BN9qYWo&Gf)LJ%xt^r_uwLyL54VmbkO6 z{e9=Y?)Ma>tXxCod(%qy`z+RZ8?E&eBga(god*>Ap`1M+K;HJ6W`C0|e7Rc88Pf?= z#ThRDW@M*ZBn%v3+!@`;#5`|jzF4zB_I6Bp=my$^-#3l^IYNJ^a`eZ3(A(z`nadb-})RR$W+FAO{f#u^|w?HV^A>liMGJw%+3Al>JT@7f}a+z<-I`+qqMZ*u@jhA*yNK(>`H zojM0pdj;}V*WqlSPe`|QZ^}9I-1X3)#KZi1^|@q8di6aV)pW%yL5W56edf95;fEdZ z)K52ag6$R_(-&I|bI^z%1hrKn#bMxl5)K(8oUmh?xADPSlU z6wOVNJrq@}Ab-7TV{s{U7K1>Cu4?g{lgk?c<-j^{>R_AL)p9;yPRd?HFS%DxNpnC) z2y3>0(*y^wD}6tA=ku7R`%}$KzN`$Q?2OYn1Ez|1)?AXX_D$(b)7t_*iUh|BtF(D_ z>oG`Z9UI1dZXY;!sM@% zotGs0^3D*M?0gDQfGx6!+)Q1Hx89DaxE>@FS|M^?ECxcpU=K|!bHbQyiIiLfZhq)K zqATUlnSXsX#L07{dmW<*yGSx#6ybPFvC2K!HU@McT1k>eC63=s5_eXau$;JL|dg3IQS3E#%+ z31m6X|CoV^hh&l6FKkzz60~Xc+OBY54!@reP=6sWb`cGf11c9~#}XY_HtGKzOVB*> zonXq`M9t*dfH0~Lyb`H~fHGtS_p!V9`vfR>q7|J~MwyLrFZ2gB1nsq<(8G!E3gHzJ zjJ1AS)APuEv$eH5Vi5$2=Fq{#ANWcHB+lIK%hcUhvu^`Zz$+N3yg_S5*jOyy)P|;c zzJLGfGcwQV7;!B@soVws)k0BS{3vF>BAgZH+qugyYfZWoUNtc4LRKx1@l5k9q%nkb zpV$4UV3aic(DxPu%s}Bl;jk^AEz{E+#+!jO6W(+&skG`+aqF`Es4skFTlE2aSf=Vt z<~H36jSB>(Vab#&DI)~NXLJO{lscc$qTB%F$`pA*os>dulqL(?o?Lgaa0k{wm<%7Ds4Wi*Bt5P@_ zJq|>hMw#Bnx^=vZA=~VeNTjzv%9MC2@*Is;o7qDYqVCtp+Uzb@=kL->-VsK@$#i+d z1R*i^g1OznY?I08)fECSLx9sCjoJ_xd-dYl`?Z_%g?1Gqc&sf4?}wIBe$ckjL4QwJ z)`ztr8b$BYa4Xc4OCeucR9ZZ*n15T}wo#|r*nR79jpepWvunglP+EsIM?pUq{FPm< zB26T)hYHjPcn-tz{HB>3t;4)bP{yrgC9x&vm_?6Y@4P6M0?B~@$MT-0dCfs%vW`YA z%VND4yX{$0CAU8Sg!%*$3?sVdq2QW$byeFtZ|X_ zigG?pV3R0*DmC)6J!|fk?S?hHMsvMLxP{XW+MUZzqKVw@JB_AZk36=ab&_jPMTto$ z4ws1#<5ti3Q~FlV(%j&E_8m%L9|fFfK&@KKPZIDRxdT;T9(&KMQqc{fT~L1Nu_h%MTC z{vOnPRklSA_<@8-kAFVM+{<)l$ANut4cW0zE}d61Z(S}p_|o^7n~djqV;Ab&EM;%z zELNIy%n=`8u0tD|o3V_jGEJ#31*I|O!91ywJ^4%Y5#tW0jU$Bkqf7Gi6?>~G+TP>3 zI&<;rG>U$5`FiHrYwIBYLhG6Z-%T}RamGuyT`DSg^b+qBRe##sh&hU{iQ9rLXw|4? ziYW|vlHV4*IE#r-I^QZ{oyW@dTGc8s!!qzx%zbD0Yuv4(_BRlgg_Me5g{fwp{<;#< zLJ%+~#D=kx=&tt)g#hwq<#RdqAu}09C3BT*^Kh8JoY#EtU4?w?$%UGV>}@Hj)k|bK zU{1TH^_0LWLVp(u@u|%nPH0X>MNy&=Nsey`sX5750ZFezVl_Jr-ptCZxba*PC$&p{ z!}-k4XneI0 z?@1MN;D3XBSB+Ug&RfhQUhHZthDt@GHPZ{zkQ4*jC&!)+rXF1&AkYiKAANNtlpKf{ z)wGi4+T&r|M^=w_RPu!YP7ptwuLWu4P4;K2xu!NNAnlpHoHq9Uu&an+bwUD9VOMxF z0|@-@*ZoukZ{poSv9BQ81X4AmGN|wAAFQ{Koqt6!TImA7YosshT?#jw$_&emf@;6I zWaSpwoATcUIW;W@Sm$Xmp^2J*yX_S6%~Yt5pMu!zL5=5+_UhC|tq^=&Ro`9P@ucB)8Qiz>A&?;|8HoI_+;C5s0HQU~`L7G$A+>|*+ zx__fKL7mWPZk}Ga+|eh6(i&GPEQ{;nqpyq~b9mfEC+?hV9I%}l?pP&{0`pgI0S7%^ zvyhZ%PHzO9Z@bLK7B4Z7sBb72XcT-n)|C-T{T76h`p~GkObJ}tJ@T<^`MUXpz|taO znD+Np`bu0EfGMjl4(+;0iurMDHw&;P>4vO7k?pNJOgfJC!ey!d$j$m z1HO3ZE|H`4Pq8h^2{T6Fv73s;@E?1PIp1~&Rd+Ay90TRJN01qmGnCY&>7f~lSHa@( z?pY4dzzSFm$XCvoj+9-WbAR_@$+xXoq~nH8S(mV0@ED7$$ zgKZ|WCbOorC$*9N2@_(;|HO7fRNkxf9Nk(C2b~Ybgo^P-k%8Kuy3|2!%SzHKS7Q3x? zP3$}h>gOORwLyiaY_W1*JIxZetEmAxR5)RuunrZWk=t%z(?7sKc(CUmB&AGi{xs!H zMxYXRLN@&FX8BNXnJE17sDXsFLCNqOFK5QS$9t7F>g?Q~JAZX1bI#X!j>VXus54;6 z&86V|M5jL=c#TAxv2swb0tT?wr20s(I&`1l2syx_!kst2C zWJDB(E@YOBvwug4=6t6ibPj4GGt)W@#b|0VWJ%hb7p|xDv@G#c;D+#L1?2?GnX(sQ zQjO#`mqN%@(a*(3DNdTdiHriWKhM(@`g-uVNa-EfSnsr&uTZCOIAyBTy#)skh)8WT zdWRzenHfcb?>hRTrt>ghMYgY+@j_V{1do0qXGzU`=zkL_Kt@vXBtyOndfHmAnhpWT z(F-c)IP#Sj_}I+sh0Ap$LP^efHvvsubFl4F4WtohnaJh$$@-7D=`dO7Y2K#|b)l|i zma(`s9E4uO=Ok}vW<+adhMSTr*(ly(D({xolm>=;5lK=YARw&MvCIM2=2h7U=mD$VHEN4b{Pl7r}g9BR{er588bfG0ROjhcl2eU4L{eZ-_>0Dbj)mQph)xsBfbv;Oqkc z_yDTF42UOnrBS$dJpFcFLPML&X3}ISSgZH4uu_D!9$fA?er!RVbcE|CMYnv_It-Pm zI{D5xZ<$;6y3dZ0@m52DBiTAnQfV#)0;dea&ao_m5r;&QPgz9><&3YD)+FXLH8e!2 z4S$zfR2fr)dYMf#Hg+tc0K?&N&=M9qOV7gcI5iA33ec;LF)$H>OgUS%*_y_~GF#Ga zGPQ7fi#)R|uLgBXA^q1Jb=lgRVOSUx_S4p@F>NQMO|`4v{k@eyv9EP(zTx1oM!ct3 zxm#_pIcahEyzFoXc`q(^MzPY=dcX5lmVY>V0hoNsJv#pg?yTpHC^hZ=66Co$iDte? z;>vrI68eIiNa;>`7`l~)!V4gDIP-W6x7iOIWoRPTsxKodqO=6|JFK~n2_DzUj~|EJ z^(H19a*nAkMO5m{+;m(-6E}<&*lIq3YFUMCYy?}WaY?R1zrDsd9m-MHmWv*EGk>xW z5+-f=Q3)8PQ*w9DY44+SI6n(Md`xzYW53f}5z-j*v7>3nHB|LESyC{Wm>n4e(Ai+6 z?oJdsn@$1A`^L_K;*5(3Jme2^`>z$+ku*#%yH78VU5K;7O6QD_N-y1&=&Lo!lti>v zs;%Iinan<%y-~r7ty(j|$PtmxeSb^SV3$V`1p-wANjRjY$ef#eP$%KfVZvI>Ggd3F zlyB}wZU@`kuKs>a{++akAH|m{SALju21ypScuQN<#2c6y;lQ<;qj-HaBd0ovuMA@u zD9WX^N71fS>3FSk`zR0Sg;BsvShwKr$yHQ7$wQ^IL|hDn2^-hB6`}IRh<}6T9Ep39 zGRw{d-lWn+bIQm_kTr1US`xAKm`QnC^$da%x}@mt>1bn&I(!B@vp}y^n~jOnc;4J` z8-cTe+aS!6$ZcNhcV7F<0%H+%#ei5CT-Q^Ru6kzVKyg-X8{6-HdN_KUuM;{c`C@@PIvHgsdzMdyJ6*1tkck20SGLq9Yy;GI&(u++s|M z`We~^5}HSDi!%3@RLL)@=yV&h9%*3B&;!IOs|#f<-kZ zmo=h<7L)=7&%+&cX@8s;^)={xanuSu8?o{f(F`gU0z#^pVwR3B&CRj(g6shmERs7r z+;{tU!!CG)%%~;;hNdE?4NwO-sZ{f3(qb)n`6hw~VHRtLCi%s-K3g8fGJJFiWO7#a z$$)A;WtS}W1M_|JbO$w!ZbOHxqna86KdKr+Oiw)QKF$T zqJ=fn#O@6lQ8}nVAjO(M>@4lzT5n+B-jE6I4-}FC$$pk36IP0= z4~|YwH&{FQ?8c9ed{W90HiR4-r^ZCv>3S;^2*dvFE>$vk!#=%!h<}G-O3Pv1DOK5xtA(!q&MG55$wZ`7h4x{qA>OvzxYT`rZB~pni9;7hpU`DE0wTOqhKMprlzi?d2i31l5Px}7#y~foiWR=$buPsPW-h$6ISNQ7$u(hRI5#P^RVra_Nb%7 ztXsniXOTtV_ZcS=MqhZlTPG3IXKK-kVR!nGyPIL63EJQ(j7~rZme!e4`pJ0=Pr~HCO3+$MoWuI*MELLNPXcYlZ`boYErUDb({!$zTz6P zDtN-)dtw&#Evc`))nrB}(@MxwEGHt?-M?Xfp39sK*=DK&pBd}cVmr^`K6+Vh+>$R; z^r)n9xTeO|pa@jKp4ABUB8pg~PX#Z%vCM zb$^B0Z4B(V~ zK36!F1QJHxygz zOL;S5gGv^qwCJ>dO^WFNc%r1IIvZUI)>V%Me z-`O1V@(3eKngqZ`2MT{JsFCXqWRZK2fq@6Y=rrJ2aJ@<`{3GAo^k2ZcPQvwBDgke!n~z--8h@SC{gdqM}~vOYyc z98(_|)!TXpCHHVePN8;JA=GVx3@YxSY)xa{gU*jPW&4p(e717-SQBp)K1@xJeMg-JP00lX23 zZ~Rxbabc2&j0raM>oPZ#)e34Cn1VkOBFn!nTcSFbXc{rzD=10gwO33zh8h>ERAGx; z8FoN_C5Q43n5p(y`HsR&cYn=42Qo=R8f*#?%d|rvV8rO#0=-#R2z#h>ZQu=`PM);B zivE(O4F>QBF7c{)s<1Z)u_=|&zNpy2edy+}4usLbXAKqBl%f-jdw~L8L>)<{hsGxd z0e0uyDv4yd14hMV6HAjzi*s>$W7ath=0xrX@pqr%znh^al)y5X*MC7*!P;D@>GZ3e zQ5#Wi^|SV;SM;ZUy>XoGte91GDJAH8F9eyp-wSmMC7AY6(PjR=JUfLWudK4TGHz+` zcsoqo**CYdaiMjgVx@AWajtbvdh=%7Zgu7$DX3Z3elb)(%K}|!q=m?+s~|Mo)KPOQ zA%tc1ou+wyTb=H=@qbCW(}JDQFP@*afI~`0@k#7XL*A*{2;+VzFI!kNmw|0cd>zv- zN4MxvQ1SW<|BI;qb#h)I0rWm}DU-#@)kh7(?+cyy#KLu=U2^&pm~m18d@LGw(&?grDExr|Xq&qGs>@`~cBj zFvN2)J0>=auUV`ZHg@QJ zur)(DL@aH`&c=ytjm)sazK7)UB4kFlVYe{44iID>yMIE??ylLXxBdze+K)Mj+Xvip zM_dS?L{m>ebk&wII<7c4$9bPVFNN>Px?mmGbggy6V3yOgo>rR*T?-1R2d1CKiP1IJ zRovt;n@Ac$-#rTvL$p$B3^+;w>b?h|?>?C{D_t=sYYy>|C!_o9986%>uCTxF+P-bO zyM1<6jeo&JUuvPB(2-8A9xxwsY_zuUsi>i$qD^1K8vgsXpIX>OYM8iMqgXop{8u5Q zwkh%rl~!8ox-}#^A0NXGnDAnwH>RWnU1eQEpqj6@986G4*@liXEr8oh%O-{2X1`h- zTE1JM#6q~84LArmf7~CY?7UvF;1&po%;I!%e19>CqEUzt@gSo>a(Zu%s^KF1LLAn0 zN??i!_4W!2kGFsrLtflw(EPSv^QFcMj+dz;S_>sZdO3r+b4!FTQJFKdi1*!FuzY4y zroYaWo`40)B_TZ+OM)pVBw3v!P67cA$EWe|R;o)!!ODY<#l0!%BMCinN|l1% z-+x#%jiMmL@2z9+;2#nSXKrjgB7EOr%;Tmgmm*X8ogM_zz|FTd%Aa6H&|@Z9kRQ0W{bg30gCwt(mDFj26N#*F9lcX$#JA|ZZFSU^IP8rop+S7X?cG-n=fJFli(52S$Lo$K0zvJ9H=0ZC4Qb2891^|j zEo}uuXu7Xd#8M0gjm;@1m8}dYtbd>hq^6y>oDZO{%-?8~N>ha9M(-1o$te@2`h6(J z@a~E{Rc#QsPiiSy&Th`QD_G6?EK_ETP#vn=vMSo9;=%v$vg_8LQaFw*4r_L`Hq*ZA z-dm@5ldeyqk0J~fYmL?2$~m%=TKr`OO@>#==O)>Z>qU%x~!>EMO)8t7Vo- zFc-0o*6STp*%7#FVu1>lA2g7 zv~5ADIvbM+`|(a)1s8N?8i{rHqj$Ymb%Jq)(HkDJ4Bt$?9cr|KwtrHG69yWlH*Vg! z3B6O*qDA8|C*h66=^6l|j|#zeh{Ci+kwx&rWw8kTs6+kMI2|d^X zV-`Qb9J&}xUGbz*6s_=UhhOHJmNucjf6E)DVvtF&>rd# z>DYentge8YJSq`KQ(99~l_2sh{AC}IIwY*F%D}D?x&$#fw|~9}!jMp`dm~I70l|=D zt$(*=8C_3(PkkXl*o*}QjTvU;7~CZdBgoP`unrAVW>c_hnb3uYG961?$p%wXrgcBK zUZX%#F$dFSGPI@e5Yexmx@F<3@ck4AejV3efqeaie?057rDI^%{VT@_nA;2DN;i;l zhw$WMai6#5^M9dSC+1#inChR(qq|_3O?SWC+1OvWvT+#r8$f2DH_0hfJ}ihhgrIytp`z%!pk?){=7lD`Os+^38nk@@vCh1dyIm@vcU4OFv#Zl4`YQptMo>aq3Ya3h9 z>;t_iDTSv@j+_yGv2{l_rhKVoMJ%R#y(X)_9;OJ(m5N85E!i!Wf3E0y;P{iO#Q~nh2aTW@2<04T4)>y zRA~xJPlcYOXTF{&Macapbtsbz;bzTxG>1Xc?;UD*sVuwoHGR zm8ndXIlkGF7ma!KN^^WdR=n|Be6!o~LJ~eXe2}8nTv;ISELSYu?6@#rLu`0WUv@Ka zGk;{gQF|fH6u<`|U>#uS^~Dsd%byk|6XcCo48;1I5O1%p_Mwh0yZ~fFXly+chSCSzNN4ggY5{!kpb$EniPd`8?Vlf@Fpnn&WsD)z&yRIVW0pt zE{{)KSx`!R-?-HF*^E8tZ0h`mE89@n1XJxcd5Un<*2t`K)W_(&pVZY(0alnT5ccv= zXl0ejt#T#k%tS<8&;cbFWh#B241bO%2WK?Gq zg@(jKfJV3_9|uSyuo}G@{dXwmt~01eM$hgAG(S_Ad=6fwPCH#`xfdeT+JA}y$ov>w zY4`@W_>i`d-tJtwc;t$5Be(O#e^pPx@bx#Qu|A@-waZU6W*bx?fF_>Hlde{clG{|9=OL^k3GJ6adjk2R|0ih8 z|0F9P}}_)W8{x32pXP$LDDeOKmC7{M#V>I zvixzXozl-3#NY62o~rwSc&zthF#Rrk4$WUk%70)dKmGj&>dl{iDYU#bkA?|IpkclK2dg_}{u!i0)6! z&i~!;jQ@79>;Hl9jP$JklkkizEOfs>EV9ti{{rR4LiY>gARPKFiJWEgb~uHq%<;ObWX_;?tSj#cs{?!`|aen$; z@opf8g+j@$l}eftoT)ORBXl}c3eBFPQ&y-7NDDhwp&s}qn-f{VZ8hg^oj{AHrJ9FWrD*X;Xe_MFQZE=KO^1VJzGNQ)@=)pO&}f5Py$O z2q|iuV8WBg>0!drbp5qWf0ALm0aLuA+)f<#9;JvU_pGt6u>|aLd!!^iEqWxP zV4P1B69;4l>ug!;IGV>I?syj~>q)l#lTxD`!*28qfUUaS-+*PID;pxVN{AT8u+jlu zN0GMVztYfYm~uUEMsRd3WPu#SIDeSO0F*doCak&~uD1KVs}E6(y_RNMw+eBU7meQ)o2G84g6~+OCf%VR;{sR&w||2`7!8bFmgN&6 z_SQE!VtY#V4Kug$3l|}mTBa^lo|22ZU~(ktnQ6zTPeHqO=G2LQRDaCwqKyOZ%iinA z&8IPw0C*<5wNK>?hZPAex4mL-(7Lxso0zj;FsXeAA!a}U3?xZJh1t-%%4OPN_k?Yp zV|K-7+DZD9`FhalWnJ2uVNdL*ezm*cBtniM#(Pk<^ORQ{DNn|Y-h(3qOg&rF-@RN_8h_nq0y^SUnPO}bHRn0E zQJE5x1>v~KMW-KzkBG=KHD-D6Z~Z9L0?faaPb2BpHjs#uI(T)wG zQj^PRGpjKXI%oX`anNdTjb%PwR#BjR-uzbN#U;r3t!Sni&HkS8W8jFZ*o_;-bunkvEtMdk6eoTdAde1B{C#TA1KtUp(`BBay@=-{k9 zIJm0}`j?N-UN2h-T;s8v^+f?-GIz=)ZGdqYWg-Y0c4oU2}cj^N2||I4{@T0RV@ zO<91L!~5|RAN-L48sv1Ef_7X}Mk{3n18ZQ#8X6pyf%TUA=> z&~IVBN9Tiyx|C_eZUbo~kgBJ|)_x+@NuDj*fq${PJnaePJam4IW-9NyKO!sUI^`nt ze3#zty3KhkDqS|~E(uEv7%ci@_`?Bk&wtEv5WV>6G!-G}DJJ^4O;jB`D*X8AtE1ay zQS!JD$bW8UoypiKzi|tjIaV$foHb#fnXCI^;Vu8ZLtl2|XZNJ!>B_X2Yr#UVa#=`e zrhjLC^#g&3U#grjG4T^ZKIXpP3eYrv*@Sq9QqZuAW}g?cyHU*h31ArMcV8hr3`{f`et9|f0(60b{^Q*@jakh2-ETM21VM< z`m|24TPUvSvQdl8jUj1-qgzsi-SCN%;c`CjUEPMAOI{3H%VQrgLz1qBrlj26I+L39 zzgj+1O%Ez-#<#|i__(wdTG~8@WYJWGLN!kO(Fd>|@1&%-5&ycfV&~L(dQs1-RewC6 zGi$|ux(ubeJfdWMem-D)(g0zw0LWaI^Z?D92rdFAEDQg!VuMtd!0~jAD}Q$NGZ`_OqBJ>d8D@ zZ0UJMRVxN)aItc>)a!A^uD70TgnuoLfet-zt#CE2tH7nj)gyo_!`qlMnyW-?1>(T^ zSlcMu#V=eZRJWlp#O3UwlSuR=ui<|I9RD-trEUPn8^G}faJ&H=Zve*|!0`rfya60< z0LL4^@dj|b0UU1t#~Z-$25`Iq9RHsHj#5(p{`dZGYULy({||LA6&MiAqJMGfX_jA& zISBFfJY9DZaEqfRCJ~&WO3{ZiCNCFIN7jB&R_+^&)L<;)=B6iO0P@z;QmE_wv#P4u zJ6J|4i!D>KQnuoiq)P{Nb)oaU=q-BXVVfLv40qLe-NBryDi|y>|Rd6edoHPcjs5DqK`Ld;}=5C7*%IQ z_M`~Y66M=ccSeF>^y?#i+B6+LNT|EWXbaanVQl&o-YSQD9@o=eFOJJ(B z&T*QP_KVmAGRixyp>bDjFO3I&+Hc^op$nUjL-z&Fcb(ezcL4hWy$bu{ukhl|Z8|EK zJEz%%7d0b);!~_!&>7gJWkPc(m_U8KPyCnl^X=^x#rMJaEq|)R1FIK0OGNzx^E3os zUir;y&k%me8bWHnYY>iLD#QFp`+*0edbhT09G^t5)~Yt|j=tn}wBaHj*{E?QgRP5@ z)B>zl-9(Fe{Asgj*(LK~-?iP3ngLkK-NZWjSK<`LuLd$AWaGZ0`qj|)PwS)hcsPIB z9Ej&y$i`KQ<$pd7qI(X4*sQ#%5A<2dl~gc9NZuZ$*&ODrC^ZoJdO8_k;*>h|6R>HU z4>93i$;*!N1q|&xk9wf!t=Sgj^>*b7;&76iy=pKt**)iQf+qK^-Mm_rGx@Ec|ATO^ z3Bvo>^DS>6*NPR)JqKjXAE79X2`ZkxBd!r2PW)9>V7gGE!dk~~PIDrc;xt0@Z*Wn%E` zW+3kpP+TkL8q&;O(uWPbG1l4m5LG)e`Gjj?u73jioJSD>-be}RZ8#MlBV1Kox1R#{ z#hbWZFD-S{v{zr_r%$Y>2`A$82$wc1ygVSw_-ohfdkb-q>sl(B(E~w`uWz)dyM6q$ z7|H-O7Vy4!$3U&PE?$Lq5K0PFferh(9^N;eJQ?a%fz}1d%sJp4uoP#_2WtnZ#Y79c zk$*4#rRMX786U{~H>uo9ri{hW&3RVb0=3R8kZ5bkkHE2xoH{~`9xp3|Q!dQc;py#R zV;1Sr@jt})w_j`&1-JyR*Cg>RR@|B6w`RZf*X=*6UrrJ{wBI7#-G&%z(tVvtCQkT6 zKWAXj<2|b}vMNl1aH@y70oS$TV=|Hf2ygyNjZaX=Oo zus(6*WYs3G&eduXI)P~=449GGEo?K)7glNvA;XrwTAjy=Wn$Ow|M{n0IE;un*-Z9~ zUqXLVsDviP&pR!Ct~h9&q%}TRl(p=G*MSls(QBA=Vt??d1fp!VRrb4z`FHE?7)L;91pP0|xSMew&h?14bK zg_v~{y2;c_-gl1+Y#a@3l>3KKfR7#cBNjpHg1pfn$ccjkkHg}1&j0UV6 z8aqpWI`Pz-SvZ2HZGpbeAK#tq=&yQCJI2Clq#U}FEZRCy(vl}YnFl7Y>1tDrZcDRL=%i0MlW8V~uAzM>Ne3pwj? zvL}C3Y;|vdwJ=$~^jQCVaza(cA~S?hjcp>oknj7oJ1r zWL82YxC(;))eU*S^ZlzEs;|nY&Zka`_uM0l$1&y7D9s^7_Xo-VsLUm`?sD~FFYaVf zkVeKJTQ2wKA+q0gy8px8KR_4B(zS`Y=vrmlwr$(CZQHhO*DBk#waS;GhXE#k7!+Y= zM%&>DxT1_^orRYc@5?kUXUD4ou;pt#ajN(>C>yjx?}14gtYuBr_BCLOgxIrir_o?T zkF*59Q;evZS;zBcSqBd7jB}ebB}rK_S~3W@q*HT*nYmsF-#K1)42!R_skHaGrBe>C z_Fph5`>l2$mx7EI?&viK%AHYv)rF9!pH4+PZ;VbGh;f)oR6VSi7RF^M)biq0)k9?Q zdv3BB$!aohPsTk=8f?FLp01lZBtvT|Y*J;|U z)IMt^OI&t=%&P9d4%iZ`ZvTTEajj`e4lh?bFD<)d;I`!K0Sx70k>|}o6 zz>eu^^z{J8nBPr(@ME;7kip9QWiG5Jo^vz?Y*~nb9zK2aLh@~skMW&oXoTG22%3pd z`^f`I86Qnd#}+%7&IDrU%%lk}?N`{1Rf)bu1<|6{LOyW^_J5@ceF?qe(CB6F*a z=Cgv;=h?~}=^LeOJL--`@TwzT!Pj;Vnt|dmO+r_$YFVYCMw!6U+j4)IjIwjO5J(gR z{w7G68F654(WIM7%zYy2&Msr=6HIfsYaG=j%x;gXnumor-!_gzEoZf znF8lgO5411zq!o6B|Iqh^k~ z*Tk;7@yK0I;gTpmeX;o?DK8NTP8&^^$ztGn%eJP5N=}?p@3E)vH(*Dw4r*RfOP-AQ ziD}A9K`FEqP;ZKVbZ`C=1B9fL1+!RT{+P=AxOyCY=2)6Cor?n;$B)#wbM#&4I8?>j zJ9?xmdi=R7lnpy5!q-;q)>JO31--REqFJ1o)R^KH4cn&Zf+44$UAQTVm=Ovkb#c1_ zfFhy@ncxtqWYY&FGT4?Kk;KhDKSwkfU2Zfq?0|?Hfr*uWf|5kzJ3ek5TF=UzM1q&g zb2Lnwq|1pVttc0)XYu0~{KO2{lFdvhFDXz>QDrZYmI4SISh7(;sHo6}1~LKW;HO52 z=}2{8+eD}YFmn0SG_9v^xoKyCd=h+pm*dS5buwQ=TeiOjtYweu6gyrFVL*AmC%0c? zd35ic%|TgzZF$n-g3M|J&Mc*T6ZnST|0Z>;XiQksD`H`IQ8%q>;;@lEh!C>O`e&(Q z9X)*biJn~eR2t2Z)OmWv+5YoROaxA9%z%AcIy2?y^Ou=9LAKaFQVl5^ALZjy@vS|A z69d9E;uRSSx7ym%jG`=eQ1Dqn4%*`q_(<<3BX2=}lX2JX2D*0Li;xx+DO&7Q-84$lVVR zr0n*54$WnHN<%$urh@DP-e=vt{1xYaCj6;C^)!%;91I>@r2A1r#Kx z=#n~r&Yh;_X<50gls=j0j>e!zI8W_~#Y2=rp#iaDLe|&)fl&`Z-D@3zWlY~Ndvzvq zGO7@GPa`_W$cw5(u@SBIQPf`P9vbfJ|zg@XAyOV8<$_y z+EZZMkH$!IccU9e+L?4VCI~rg7AE)nsAXC@76JnaY6&^<;4i)Ho@{G1N<}Pb4 zE4I#VnR&mKw&ievx+tYR0!M%Nl4)NPoj(%Wl;r7a^!Hl@Y7;4>a{olo#{c;+Q}Ip> zV;kjK`)qT4uR{_x(JFxX$I%(od1$`L>#$gw(%tuM$ymbu-S~?|(%SR6uKf2%v+wPH zENO1bC0=7!u;>_0H)yAwR)3;Ta?4XhE8h@lbz{y zSUmnPLjL;L6kJi?)t&M%R)kepjcXNHT>>0f(`g1V2-EZo5z$9>BK7I%&py`dwv~vL zZgM2H@nm8rHyI5j4JQq$3|eXyY8$(Mdfwl-m$A3+A9Y`?8>Jg38);g)HdZ!THV)=@ zcctYg0Ld>@w{M{U&Fz!t@*&7+xn7!dF}oQ4Xt2t-{v@}R=6Gs;ZFFCiW40y%o&Fvan$Vl z^H?rK->8;1Kxr=2$g}7Yzo3tpHR#t(zj8}7)IAimYxCY$lcDSYZI0`gn+};)5Tk+p z;91xjSe{+6=IO8MTvfl7UDch96H-TAxBE#{hg>Z0@dm{87tlsjWjC_&*4A8-?KK;M8i0Zpb@4od-lz28*u1-Mn{Qt%9OcU4%@E20f$X+5u^Bp~ceI9ePP8@oiq zFi%)9x^e?@`)TS;zf+|MBAP6+DAI-Mj2{6_5_qa-Uh8THbYAv1%bsjOP|^%S*`Ovu zo@^H7j~M2G=#h)&AYW}CL$Uvp6fi6lL&iMj`(<`UgaXG%)bzW7W0MBD$Y*;6t@CP9}dzz5Dd|7Xpf z`=HG*9RD0az`-a-$0|ynqnzUTuVRuG^BL0_O7>~PkCDj<{vOt4&2wNP?C_|30xtu; z=?S=D;?D}3J8GtX>Tm#lW(=|#u~1?)#Uup$~3n40B%S(;X@3b$?)FC2lMcJyP$ z8Py^)nqs=eBC-i6$%MAl#WbK9F_sgEg8G0tQg#&mSklaY3Gc~YDUA5p9U0dgg2`sb zBbTlk^fWKmDa3-1DG5bLOM;lB$dZap45vEHO64Z87M4XA&HSDs4(C#;M386}tYRr0 zbvP2Ri?-Pyz)KctG^!2BzbTv%t>j$$94~&oi9P~8lD7Wf9Ya0133m+Fi#0hxyVSZ3 z#V8!KU4N2)V5DutwwB|nnmruUp^bEI(17cNWAnchO`=Cliu2pdH zC;vGEM>lluC<&@!fpq4MmA>=uymQEhfp1~MbR`m6Fw%+++8Knb3`p3jo#?R|^aSai z@BD#NT1>1#1o#i2MZ^arNhFmrr9=CRLG$33;2dXv4X4UoWL18e=WC-k8L)>*c{SAD zUv>kXz!lA`6CJLHPaz?F&XCvJ(2KAe_xs1zJ&Da+U9UII%@pS2OQbs9#Z?kCE1b@A zfZ2020x$t$x)049KRK1lFRDE(3;2nnqfluknbgjOT~1s!Ky0pO zbd5s4Ui$P83$@DRC>tTV`b%q*G%2l+pCCko8~T;6s-ZjJj@n(mU8E7zn{B4A9xb0Q z)II&%Utb}8kGB<3vv^!DR@!Q|INlyjzEEs`cenF6M+GXLXTAs!Ztd?_3gupwFhAh4 zyPLMU1G)jC=oxnwU3R*Q7*pG3I z8YEoC)6&eOQs;jAe^jIFy8jgrXug^PwfBf+TEWXD-T*!ZFvR z<(%sep%M&j&;a%CMBzy`vv~%-ux*xqlHo!gSI}^6erk?g3d;nd0ir?8 zvsVDFz*>-5QX2@E1qZSV z_cPuhDJLs~f$i`rDI#fzyh71Jk)|+cPsIIdkqq0=Bvry|nTZ6M9;qAM@QJX0z*VZW zI;YXbZoYx{)44xheaIGtMw{Na#rJtX_p9v1kIT)r#m3$JYq##{3bTS0H!7AlD?QzJ zeRM-ewEwM0^C^v;k2i{nx0c#05N+U6%CT2wFF#r%S)V^jek9ALqn4l1s}PiB?PraD zpW*lYR<8=CVF;<)uP{9EbOM)u)|0NrELvRkr{9z%%__=u)F08x)=i5AtlW>LWqQb6 z{FPqZ8ot3=eybzKz7C`rV@sg5gjLp9Na`(```<82!BlYbk${2 z<}FHx#pKHt;xO9z*MMTjV;i7Px{A}7 zO$KtOTEgeqgYl6N7!_@QzlAo`LCiyI<5-^1E!CpUsZpp?h3qLXC*yl$6hrJ(u3-8% zm|&ujAp=e4t5b(m1M3{*SFHh{#Z)?mxA555Lm-e!{t)q}Bym~ljSP%;o~k(MMnnj! zJw3($M%kfs{zZOcW2#V#G_e;YW!TD;b%^G(FLNrxvNXzkB<84pR#yjSToqE5IMt*Qo|fiH0MT+Zg41sQZiT; zu2G+(H>j~!r9Pv2Rbi|?f5ARSHyZz$Hur|ybG96EA{d2_(z9ucs(?j$_`9WUJ$Zdm z%Q`B@G^$C8MA#{Rf(fvAlyL>hID|u#wo$4nWd$`3*eS0O>?#p8&xtbWeG%f3$2w0Z zFBS7y4Mn}8Fm#3D9i-mBI&rDr^{%yD{=Q^MEw|p5bp4dY9P=W@wITbrjB`u# zD}bycn3*%<=$6t)W}A+XYjsD!(P8qwf^&KpyE0o9eQ!l5$gXm2>GWqQwA>LRj>` zL22*Lc2khV48lEJBd1yXO!#@1L@z5$)#t1R@I#>&Mb$oEZj4~w!oEqA6P-})!P3C(SF~aFYTGJV2q-$B zp_k7D8YY|7ok;6yDcpu^%$?S(RRl#X7qRLG_AZe8O!{q30Xm3pJm!Qp-AS}RRQo1I ze8(|=R$cK=0q!9o^fU4UsaL@2*sX7yrPl5CRSX-g19G?Lq_)MLe>Yc$B2IJXUjg-AysAOm=UQVRE{-YJzc@%R^Yz7YH-sFql8y)!E}C7-{8qc~eC=<4 zE_bJMjx^$}A8bRaf=BPl!CKcXCW()C2<HS-lY3xh@F5C4{w^;7s9+TaA(7EItGF`y}h|fIXl1w1livnkI8U*xm5FgJd%?P?T~{Pqqx4 zrjfLWE{IgdsA26g=`Md{7!{(yN2iJck$nz8LI>RCVqyrPY>MwoPDAE@pMKSF(p~o?oTv)|Q$!^yYW@&v8$s_{XCsAGqx=N)5odHY$`I=jN z1|*<2KJ0N5^sAt(YW09P4R*IMvyc&kt?FRxuLyQs7LUp&7CN^VFYwR>E+^N{(v)r$ zATZc-C22K7K6P$==OJBxg{vfC3K1TPSYYPXe&IEhiA+v_4^aDQMxdZeE7`Dq$@@pD$F4z!Sqg@H`nZx!5z)KY=8nZ#AxsFAl-GvUAd&m-$IAM=@Vat>e2N(W;Kp=)d%mYny_LXfh_2o{-#%uz z9A+Z_FO8fmu2%bG&UXc^*4yaF+Ln7wb<8MMEtR+CfueZdp!kA9+FOjv==m(y+L?%TK$b?<`m&h!N)KS=fdOMYgr#cqn^1Fb$cn*>5 ziZAsBOIJ(JriM3;PjABni!hVOk+(mP1|)-&&8*B?t$)&t%FfJE)#+baQ8yc zbdgFEtu-le9S`1)pzX9O46nfSQT5p|M7Wxvzy#<6I2ENC~Tj zq+8fBM|$5|EA8%O)=CC}%&#pa%Sm?pQ{C6^ht zjDg}{;LJVf(u1!J_Fs8IUg$hfMf4F-BrOSA4w!p%AyGrj*Lbh3!`TO}1iZf(jFs{z z7E5J+bO>t{0UHK^9g9nhjUM5hJw#MoM7fhyw8e; zzMjUhXxw*>gjGl1?4I8k7f5d z2e1j9}%Ki~PaEswq!k*DJ zkjIX(B@=3hwWW1sU@Ov(8<-b901snp=axNSiKTOBmo`9&g~8rAu^R`8L4l|a82Xlf zN>{RfGuIb>QIxt{G<4U8in!Vr{^i1UlBz2e1c;?m>i>!LWM8^}!xh>kd;`jPv6?;l zgw%JXZPUyN@6x7EiqLhBaItfP6?%!4<2=9uGi}w>U^=H?ft^DWpmCv~2`Fp7a#xsa zy?axL%)$#l%!1m7n3;7EPPcLp6b-_EHKi{Y9c{fMXBha%X66lDscC6?-rlFd$>jpR zROypmwOi}4DAfFBtk$Kb^Jqq($qcXZdMJvhPaKLI`(Q%J#x;kDBf^5)4PTC1Uuyx& zW#muMDX$Ea9HwMUK;_yOh?PR^TxX1x(yj*{I)p{R_OkBVV+Q}u9V#5T<4-_;Hho`z zC)AodAOQ2cx~oIEVb}`*b5F~^N}T2jexwti07Hy*WS52v$;duoXWW2psK*g%9;rv6 zgXEajpBFY@1e3^y1mhPVX_tyj;?SBn00i^851rx*K8$H6YKPWAh&^nN7D*X98Nx*y zl_$eOf@JgDYfTuiVvXfMZATe@I@`jwY6jE7rjHaF|6)WG3O|r-B?Jc*TVJ;;gJu$# z$$qfVJA;ANze`ev+Hs0t)C0vD1H>EJhh@{cHEDCx`xuCI;lT=pVP+WkaS4NF7WfmZ zp6%iKSpSojzo%PfX&TU*k-D-5k%4-K6obJk&L=y#$gy<_z`P74#UxOF3G(5ps9{Uy zq)2eu>138W{*f%;iri+28(v{!hMGOzxdT1{5ubdvKLIPw%pVBm7Hba+F4Akby-yX0 zJ{1o`J&Y!R3{!|8ARdc=hTn}PXjd35bcO(ozfgb;X#dv^-HpSI9oXK^FpLch&B0yN_3lM zNNDRDIDH~M4Lp3=M|}Qpruc^Rd}dv%_3U@w(7>?>pP0}@_Q7#~%t9?Mqt3K47a3WZ zo5HzzL4MJ`@N{o!J1b(tZ%AB~ZqqFU0WkbVYf$GbK+S(p4#)Bjv{+kV+$N9?uJaz5 zr#LhWC_tPjNT~=F#=F#9mJGC4GDs=Pc+|hA=CV$)OpGpLbrtk!`1q_SVW58xmfIOs zLycD~&phX?Ip6`$*6QB7o3DK%`7JO0px;Gaoq@lL{CTak#e@i@A9@;_a-MMc_19=1=PoJfWqc6 za_%jJr;Q&0yf!M?@u{*Fn)kLY(KU@E5;{gnRxLSXP8RqcA*@$nP;%LgnQttAYPdG4y1YBeV;9e^G%~9k-Y=HMXy))v>+UQRSjM}1hT~?E{>K6LKMuJ6 zZ4S7<3uydn2i)I@-ap|6NBI+h01u*#0j%!dTh<>;q$EX8m&+%U^8E%JOFy7VDq9Tz~9??$4gDzjTEC z*}=uY!ty7l*IzoqSeX9g>SAGG{Bs$@pV~~v!o>Qg`k2{R|Dr8_c8;;IG5^^w#{Q== zdRDrBKR$l{p^W8kWcGh7WB;>vjfMWt{bgZe|I@atjDK2>o`r$!FWz%D#{bF(#`ae} zuzz*({W~8R8yzFd|M@mwYi5AVs|)gRs$v^5+ui?Q$hk5t4Xz!h#X`%aIFZAjfwHcyWoRc3*+x35>+ip zsfmx)8E;!!5h_MF(EXuzJxO)IP((fe3$_|1>W8{XIAMvL?Ko!iZ22jy&2w8w_e=`- z`ZUGE19-3IauAD}L_Cb8nF4F$4e%&`#@PcZuGcU>RQBL_q}BVKiYrE^6?uxIR zkY5#KV=SJ8T%VI77Pi2*##Cd8uT$ei<3;>#|1llvEj2^upGjxfPmlpg(nhnAXiy{x$pan3Jp%ZeN;oZg6h)H4-k$EG;j&Tlb)@-0GQCc13bfy zB%M9u*V8?}A9!8Npg;~ZLvJ2;IULSP>YNzzUVUc|4A6e78dtXk&))GipuSp$6uV_N zbG|z3@}%;889OIjkPuv_H7>J%ECjm67LbNhyoFK0m0=Px2j%MS!fEted~M~5d0AS(Ehp0SL0Bal1(y4C z25W?9f4#wh&cKqxMxGl9MaP1S427%_2rp`q@nX;q2dt(7>~_-_qY})2x2WS@{TXfr zmlGBpk%|IVPN!+3LUHl+QkwJM2_~k6%~QdLL0sW=nM&zH`U#MrZx zQVBWHpf^w>$Ib4sJ-T*iw08MAyEj#t)4z&KY>WDpP#$= za@gP#mxt$66e2wo<16ZacbXJQ^GF;I z=2hmb7+`ZC#A+|3#AedAs4E;}EwW0&1_MmO3z~q*h0Hk9lOZyz)7r3-B~#@axVv-; z3WQT9M6=MRsp&@oTp(p$v;k<+VngJ|AX-V@olM6OaYCL`T<=4FwVo)rD*7icnHNCG z(lgnaN3l~{6~)f!BfpMLEj7mpB*2Kw18Lb7F&Y_S4-h*~De{6;LxFS~u=W2$+tB3x7!ZI6rCu7+`$W<- zlh+tw8=}M8C(K-bBlhsOPAXIe?d4e!S*T}PQhKj!HO*H3wJwXQ!9`6H>0wZonS44J zL~gu%vk1!oi&aikpw$Wd(N`uhjjnD8FU+zv?`jz~-_2pjGNn z18x@i5d&gU3AYC;#J5^>72_wyU?z~WCe%AWgy6ES?IXwIAZfL}XR?ylS8~ruy(42@ zMR@&epvuXA-k4l!EVwfF#P)l#a;TAefIXa0icLP=Bs7+G#aDu=Xv+!IkKXbb_ZBWAET%zQJ zXPqp=6u>#cF;TJHs}lBy2YEH!cwRY2Ms2PVs}(|j=(Sht*m%tdcT+nDKPGknY?eU( zvnu;f9T!N(`4X<-PTOj_oz7J)oFqc&)T^g5N36Nz(&#z81SxuOPXqo|vsuiP8qKp= zlWbgI_c4(=2Vi8_S)nsA6j1c49^q$!%5X+{f&_&lTYi_Hh>TtdY&&q7YKA`0DeAXJ z-iBI#It%aw>d?%=a84#sJ`1UDeD`S-yE)FB8Ao8;^aLZ329S79nxpSnI?Lw!!@`(t zM)eGA$gKz=?S4A6t>HR)LS~&(<^;0Mee>RTgYX4$v+4!iziR@`fr0i>L`p5VmdnOD zDF~O(7dlsy`WXhS8jtL}vuOvX+C8hjGu8or^As&sz1Em&a08AhK%Fv{eScmn-&=N< zXyLE$9(UBe-BFh8S69-)x;p-xCO0QIhx*A2*X~EF<7@GH=_eS^P72*9nf{L*&PAO#R~cj*iwF!AFSo4Ib_XB!iL zgT#{G0N;rnnU1f)>6!9vDd+o2DZhHC0RPJLX&Jysqh7~6?e0Sjtm~8bJMXuY{>^`( zm!fB*XZo-7Qh#@D=-)_BR+fMBOR+Q2vHWACCz5jWO_#I9`B~R|c2lC$oI04MLFf_TPSQUqEskSM)17NxpuG&@6DY4H#&h|+cWlv4&{c=KC#)p!PhmscH1xdTe^p`#6lhj?^Hf^TJ5%fPt!v$ zGB$nBh$YjMYJA_t+5|%AQPo^3fsRD1G}3h6%&=srt>03Hl^^F-N&qjtINzCbrEBNzwU& z`q~PH5yuUIBaais3Mq9iFM~EHZgZTjqJ;0me@l8G*n{L$2*HWm=JaEK>7&vo+TGRE zB*h54d%oQ(N|~;Tdo7(k7QEe4`OfaHc_tfX%gJSOV#1hoGRc0GEeBFfl z@y-Wy{JE{gp)Iooy}Rjuz0%{K=_8+n7aRmn8cr}v6hjmzlpxd|$26DhG|-~IBI7&; zsRNfJ-%0dI?x?*`l%xxiWJ~y)cWCK=bJ)2Xk14V+R>d9(HS6-Bq70`;XI?R8sXh}Q>91Srk*Jr^OJQHG(JFgk>fdz9SKgebZG8gbT()mw82fzWH4tjUS+EJ6X(!423 zY4OH~g5lzCfWkr%%S?94K@2dlBxR+w3TH)!TQanKU~RKWY0;z@NaU**`ezNl6O|FJ zpW;*^>%R+FfrEvPV$=g^%FNC;DxCORdyXTrs5~hTi4tmuEne@|`=Qn~XvUCP_dc!)``s)x8AE0>w zrq2+;pimzWWchR45m)!0+GkkdH5DT1+s_#&kR(AE)*WJ>Lam0l(s%O;y3RC!ZPK@A zWRXsP`;$Qta-O&ko-@62szdJwGY~lz%LU!^1UhE--L8v)N4&e2!;?4-ums^OXB6`q z=7l=x^wh*NXt{HhC1V-LSQPkFuK_=vNyo4P(;x5NO7i`8>ra6%u=0 zS91_S$1mP@%i>p$QA1RKQa6H0kr?B52z&6esQ+m?cboh!<~tIOHof0;P-Y>w+J~l0D1mtNaDVgoi$VNuB{v^eY=p@>2a?hE9mKe zxkW@TlA%T0c8;%Jg__uv`(2Tm`+UN~V4G0VZ9Td(Qz>`{YPjAiu6Dp|HT;Ah_~`02 z>p_n0J8*x_J#XrsQ^&o3`5a{}(u2dl?k|saMzp-+@uZrkvK-l9qc?eF^o3q9V=&M}{S5c1e;eOx_`KuFU$9VW(qUI{h!3O!;c zZf%Lw*+;DN>DeS1eyg}kA5|D2A$+_lC9&RL!{au1w+m*Rs0mQNi3^I@#Spe1Qye)0 z!dCPt)r9pea)COMlC#W3+Zi|ltcw`8N(eP9&j%Cqp$BZTY6igNQU~F~>M}2GN<@T5 z_zKr)!Ut)@4Pq`?pC*rg{2U9ySdv$m{4K7)Fh}c}m*O9*Zwb!_MtJ`i8|+{DK|rmY zEV-8K_Wf6*F#Pl38*|T%ThJH|Wnl_4dm`bpr5;y92lnpb zPT~^=&lCzcO%xu=6BN%n^{m4@$x_lHMD-qtJkN8LI^SOJHsU~kisfsE_0VZEY z=VU~NCVhak$=`byiQ`k!Zy6k+Tv$WZ%ZrQL{H#xzs*E4uXIbyOts~EmnXwl-aO2&A zKKnTGmBJd3rP!38bqi1?0D4pSdP)EY!O0*Nb77h)!im+HB7dkv0W zw0}IhjStSWU!%JN7D<=1P8(`N?}3c*3$fqai@ zl*+|n^U8_e9@GlOq<~)~Q;njdA**I-&IsM%$;1Z zVuP0mR4knG5+_2t<&>-StpgNEV$r}Nw3;T`>nq!5o9ylLXOfN{9y!rjtRw*yW<=0QU@L`4tZA;TR zNO%OQw!MW1ocG%kxv@wWoNi**Um%%u-+)~3dyw{?GH1cZ)`gYx32w`qmwrz2Y@!E$ zZ@~A^i*@$71!#NLEbcYUv7G_AhRK9w7Us@GJ8rUg8B9p$d=o!7TyjTSoS^4Ry|u zNieKH4`SQIH%ZH08 zAsERCq1wxrz`?*9Fo+kA2|yJ7CYOlFFt{0fEba={%AlM`NC}fHB<;^h$yLdJNz7H; zoBch@h5>&RoRhR~DSKn*-YpEF=BsEA69moN4FmgC@?0JWXGDsb!yQ3agOJF1X0WlpBG&-eXiFaxEY+_%3B*srYEp>A7 z)E~T$3R_9FwYD-tS9*F1Sh8-Tuv?VzLAY;DPSUh+L!o9G|7!OE?8o6K9m5!Q<%6affi#5of9KG!#7|phy5zVRj|Ug@xJ4 z^KZqHb@BJp5*>4YPten5SK2w47_iu{jrx{aKbT^Qtw6{$N}+BzEn_ii(r|pp5Ao9I z5Qj(ev8GswXp%P#qk@9I#wqN51f^675eS%bxE2_L2An+mJ>cw<$6wwum`ELXEO77g z7fovBaOgsy;DK6!28CA=?HM*F)yRnk8dOLf5r$BTs15*sq$m1*GMbw@1wY6*W|8YE zALC9ZX4$W$GQVcET(Y|a=QVnMNd(rupIS$w?9P&pY@aTk>n&e3(k_~zb>)7xwH@8` zVGHp(Y zMW2ac>0Hf!qkJ%K>}jq*OU)zY-g7-{1E6tYqXA+_xvK>Mtg4i%*6QayE8*pvU&X~Z zKZot6;cPjY%;?Tz#{R0+f3SRKuUZwdhnQOvYmbBXu}vah+9895v~3hHjA+4c8k;fm z=H_4_jwmh8s~^}!Pbl_${Kdwk3j+bfIZd7&tuX6z@a=BIS{(Lp5>B36qQn1sTM$b9?b!GRISK_n1_V!)^W6TN$xc&Js@4Ef^Fn_&* zdXc%(s@mb|{TUrvB3tk|BOlSY-FWJp)qV0~NFckrOYRyPh)Sutb_G+483S2~AwzRm zYt#OJ)PmO!l*C0;$YnUs3-wUpq6w6LSfCgOIV3U|T9&)%kk8O?szOqxn|h&{o8V%l zZQspWGx)-4oxb1()W%+BZ%@)*`^vH7mHN*97I}_HObJ<>#_VN6G%u-32i+BoB2bH8ADzW^`Gfg?s52kpG zz^EK;kxTqGk(lK+6K);c83*=!LXGe4f#q8U4$jPQ&JNwMfZ`CyGJ%W*>1;A2aK=T1 zU2yYcIr2}9O(G>p=CjCv0X%f0KCp~AV1*$X1=hCAJ=VzpFGb=?c*>&3I}l^c1v2M< zn*=8zcga_^SMcgS**#`@k28V(*@ASCGvRg|OQeJzOHQ(huB9iLf(xr`Y1$@jTTVQ; z^{=Rxtzz9hc?Ci{E1S!pPm|1p)Vo42bAKo4gg_y|fU&7uO@T%Q19=z3F(luJ3pyc< zaIA_}DdajFvoQ0Bdh$UVWo7p1YK{AUjd-({E)Tnzh`Pw`R7|>^J7@2L0pLE|l=!67 zM!Yf#a=&Lxan!`ML?|-!cB)p6@eyl&dthK<>(Ov!T3lPLMx%k8VPVzRI<*(M%t)iW^(nfyX7d^ljbac8q%bf<^w`~Hef2TjFw%7=sUc@L z<%zT-&^n=>cbEZRrwwfJGSc0`8F7R?1AVDXEL%_e0^2ax-7S~DDi@va4jwvSK#>nH zqxeoPfLP7z;SVRkkW)Ye6Z6L?P-M{DHhddI;cWFSDvTk2|7Z}eMH?9!DFfl39UdAP zIW!w?b&FCTH!nJ&a#a?&D1Xsg>$vNoWhJk|8gZ5Sq6Hwx_`s);UGMDTf5j-ev6}d~BAp`OU zJXU}nYqQqxWMb0oBHTH{{+#B3Y|hwah$2=%(1>E)z%HzJe>4Bze}i5GiJ4yuajAI$ zgN$%S0q>{+2UtHxF;61+ekF(@`AC$6k`bZhvyI#{_pqSN69$zfXSOQz$ zVZUWB6WOBdB3iB>U5K>k@VHjGs)qU)b=o0+BlCVh@;nGh+vdQ5*?__eCL#D)A16Ta zmhIch#Ge9{!JqXDT`$07AqZTF4#PgSycFU}o>*C#ZC7dqe-qsF_w0=NxENP~1%m|$ zvs+{AQ$JSW?x+VRuesBWD4H)?pwWR-5#hOXpI}@3WApmLCmzAcp?7!KcS3K=onHKW zhc-fgupPPl_*@9&K#|sD4L21RmGsPA<7>ZHnV4oory51FIC8(rF1vZd5)qSl59*db zY>}EcogWnjf3Y+(1|4Ht5E&S1K@l4B@D5YElvn5lvI0K8R0_hO5V5*6Gm$yCNDVor zM`YKcm0}37DjG;zzV1Z)N4@TCMIdxwQ=drBknOHk7TH0#VPV~!8nG<4DO!y*oeXKP zQwSZXNtk0H`yG~XcU~{sypol=K*~%}zk{2CR?G%e zH5!qOe+R{`pN}P6)Z>LY!yu-K;vtYO0O~hmxS2cn3%D39$QVF4A*5o8!<3Zd2ZH3l zk^2|$@s{Aeo#pxQ<2apSQUgurC!b)_UsMThr#W7-NFp>EPv#UF{r9-nfNt@P6I2Sh zWm3aqDI=R6_b99@S`{nha~8`Rxt(%f>RUD4f8`;~Y@mg}oqN%sX~vAhSc>^0lB`ZF z9@5@ttxgCZ6+AHOqtTCc<0zr&qv+ly6WgjG5_j><^SRX1OJ$g(SdP*oCuQdt?(dx% z>T{PU9*+~}Q{Zhk0G89AD70=qYcf!3pZDm>0|#cSv8RLAXPnywd5SB{ThO>|hIAUZM zn@Kh@CllpOKZfCuOJ5p@EhR*W7b`icN~%_jY=K%Er^Ml#hY z*|tpc=}4fgMVsW=VhFMGG6qW*u<|yY*2(WPBC37=YRi7Ss<$5aJrY@<$vR+|mH22~ z6<5MF{vBJgMWvZWNR{}by(mG^f8Wp(pwd}FUGuy}POt}#fvoW}I=MyL`jdw(CWS2d zqsOybv6ROnkL?-aob+K;p2as$*4fV$5U)|Dik% zpxgF3glIgZm~$TfsJ0{M(O6N$W2h#P#L&Fjtz3+ zc>S+hq|e^oBslXJ#mi!#Oh=`9pZ;+}r1c*@GBaMVx9iX*?>wP!+WZ8T2X3N?vl%=) zl)16ct!E9|kFN|@Y;>*Oe`dcz=^=dk3aHAe_ksM$-|+kU-cHNNAC5U?5IJO!nq5vA zEIn?F@Uy?aN$zb^@<-KrxJgolN_SOCO3UG@#|rXo2gX749f*0!IQ6*R z7(g>{-bjSz;i9}hmcq0)nk-3_o5EqnOYLBE;kaP3S1mS7;V8wTH)o7c*z}}ABhCst z-dBJx5zLK5ZF4(ss4J8_mRBLGXziI~P!Al!GQB;X z+l0$ua$Gs665>2Qdczu?@mC1BAwxg2UmYS@KO*t<@0L<&Ok+NSoqJ=6cwW@Eaed6& zHsZe@?9a)@jR3#fYxgsGKQT#-aUzKa`+eSChSA0keBYi*Tiisxb3(zK)0Xyl7;+X2 z%ZKh@b`i)2e?Jg+;m$mO%6cEzi!G#r(_Hk{{LHC~lkfB$__-vv5?SJ&2Pq4c91;u- z;MK!k1?qySkiPil+|?c83B5Z7m@j>y((a*v(-)Iq(_*Em4>Tb1f5^IjL z`p_3KkgasJj1X=vBG7j#65=$Jc=STQ3Bt`fXSVd%PfvqWo;$QHoKMJ>9W7!HYmg&G zyNUf5s==JKo;r(b&5D?jWSo(vmu0d0I@ZvFe;*?s*D6?1MeJ-g1nL3Y#fIAe-p%N5 zVF1{Kuu}}UxsbKfOj)ihPxm`723lQwzOK()`w!|8aQOzE#|D(TO+fD`=Zvp*mr@DN zBc(C)d9>#1zGc3muhXiy+OV(BM<4I&oIN?wc=0FE24R@XU{kz72MKvbXby%z=;9*r zf3pYt38KSz)Wh-Mtyh9Z2biNk6inzHjX@7H_e*yX)AUaTspa6XuRWT zwA+8`77iRk@fh08f_|R{@v07#K=Ii3=B5k;FYm-c@eE(#(uUq2he#Z!4z%UjJ@Fb8 zHb8w5f_(Mv5Q5+`yl@9HF5dK=vYi5glW&=Lq&%E38>VVMFuMFt509v4T zwvH2Yodjy5yh!5%)k^-taA8=ve+>{{>)QcFwDRSVSknQUwrvof7VTrfv|8L@2Vq_` zDGT9mLT}#rMZH+-o+!Mw0nDPlvx-?;cgAc^7+4OiSX93J%ObdScX@0sq^!1&_KO33 ziBYYkxuh|sZ!PBrD*tuDTkf?{zTByQUZk`)Btn2pR3e#l0E6=*qVG)Ef5n_RP}xOo zObPXR<$V+S0OtGW*XB+`~|3|h?z5M|2` z7%BpN!t_QBV8WPgB!_Sk9kc;`LIZ}ePGL|FBv1fk9$m^%ZU*+wwzq&J?e~s8h$5O) zPN8_BZ7&ft4n%AM&Rw4ne+Z)8AvinT7Vv(lGzKcTVK$wKP#RMRe-1p35#8I*Os0%J zKo+ziJ)&;(F}*>%OeO-Tkdcg{T^2NxkUS`d3VktVC=^`gXrz_KtTsKePG9O0Gki>sDU5>8_2a780qwF71_|x%~pzt>(qUt);yKf z{bPA5*ctx&D5#>SWr)BZNP@79SIVZF8ckE*-@LSYiLkx-Z8Jt&g}71-X%FL9FEH?7+myPR!8MxEvmN{w!XPzu|$)52Z*cV=I5>{ zZnwNqQGxnd!>=*`xL<HjMtstw2WTJt5_i)1SAtqY`*#wd1m|1=_S%x}S(~0l* z2ju-*_E+9^D|>Bg8L-PGlKusL1* z7Xyp+PjUDEKghiPOTv!7F|YK0k-!Rg@e=qtcbU!k?G$ESiJv+fQ9&b0`_mq|4hIBHStRSzmm`VzmB=m|1Zpy{(lGN zN-ItGuL*Xf>Hk+I8(QgqC4B$O1UCN*F#5L-qlm48^&f-&U&b`+-+ur0qy29z4kJCp ze}8)vv5w=0@uLF^c9rP^)&W%eRuQe)HIVB3rigAfS40)qecboI6Yl49_q&xw80OEH zIxXT4QRZHxqpgr`P|X*aPm|Dfkvk>ur4!jji?rlA5&-sG1YV?byhdc@aunwNnT_Qu z=Tsp7+JBDJl(f_>q|pD)Seu`*YHx|2wT?{D)fyU_cnGMWam!jQ4vt zKmlD#OGfL5r6~yTUrZ(bzrm;f{gC^A6Q5>fX8cQhn(6-{KFvhO^rtdL2D(2Xf4qzg zjDJFc85!CBgjqBGQQ@D<82*e(|201SAGf7rU|{-F*L1X)bj53Y1#HL8jY58G zfkk&{?iwl?c|XClWIipO{mKF#6CHiLd#Zx79b9!Rb?-Gq1(+_&ih`a4M#csq~(ZK)9-UfA+#4e-doK-sT?~ z(r!!a`9aAiR1I_5%&-H@)yuJ}jJQ9qlb!qA_6AOqbirM#Nh+)mjvi->vxu`6lRTCN zi$YfTe=;UavV@=rBfH4=Hp?VRraO&-%FBTY6Bt)Hjy02&*G3YsCpUusK)BFHDhupC zDIA^YM{1~y?87tJe@PQw=UxNmyZULTp>9YZV*!~b@s39&fWZQIbv5SXul8eg_Y7Ig zui)fIB8{JtIc}KjZ<5sf=2cvf`hi1AIEUW{Spk|q%I0|D(vabeKkkQdfDAnO>Q(7G zpaz=x${S&cFo(7>06!k-;98>iE6>m=nRU#sMW!3JuDPv+e-}Lmb3H#cp_WT((;Q}{ zcwg?GcaL{lcIWXzab<39H*cW!uk5-)Pdr}eI{QCiOG@@c>G$Je9dh|m7|cEUHZYE2 z?!Gj>M7tABr?tn-DF!IEB|~q>qwjDm=kJ((m~Fxc(FAa83}n`fEdj=^X$}}#Kr6Fu zlv8TEWWX_he~aVY`b+h-i$q1ck?V}|VpsphP}d~<$p&J{zTpPQ-4423Sm@BGGD5s1 z!UNcRcOKuFbkVcW8t8gKfH>nsCFq`!oYFyp(&6IkXr{=i8~L6&=KRuXdm0@nrB&O; zca)GV(8={)lA^mcS-@TADvhco>z+9uU*OERH7#?f)WfO}6dg|dpsF#^|0JV-OpMnD}* zP6o$Jhv7bCI=?f^bvlJ*gnL@8Xals~UnQE#xaDPt05;AxBp}cDSe)I0uB&vqmpd`T zi=xvVfA*kbe61MK$-2`8zCf0=qATd60sM|BwEFyhawL&wl30a4tdVC@-%`jH{aX*7 zfY2P*ORqUDUa)amIWM`epCh^xs~dwR9tkIevM|(>q^Y$-@ORwd)lc%k>$Dq*gQSjK zRj<8w`vsF1CyFq-rc9q-50HFFqI^KN__=1Ze*z$G%EhnHdRKI-C~Nn-m|Bv2zMNBZ zAVRvP=g*iSC7;VT)^(7wzL1!m0Dh_ivRO7Gm2VIsZ+;tZauZMC4W~J$O%7#OPIu*Y zB^=!K!RnJ=Ui``&{Lf)&R35UBKdbeJnF8ziFmVApU<JPpMcat2naEw-o34e=kXwLt3ynj&$qVx9n-1%o(?-f$~RCNXfbd zecpz5^+EU--OTYFrYG)6-(+gKO%2?flb7f!0&I;TIi=}Yr>(6-?u`@)*RMGplk zl2wzy-ozGVs$`Tm&d!LLYNn*fPUrYL&i(Y9vcE}M$KpvA*foNPs6FTOO42;9f3{+C zZe6=d&uKU94=58!?P>t-n&xR&DA^}jz04=MEk;(=SUY>kno@novf#0j8s%yc9%hL> z=#ZcrH_hY&OKXpYfsG~n$d&40dOV+NlzX85CBE6|AiWDO?=*Ob%OELEf~XBNhDru9 zPLMt&eLQ6s2jcw=+XJ_RkbNHvUt(8}Jle8AmZ}D3uYj=v%r2Y>`mrj$OyKWVqf0gg7x>5L_ z#a1RZ!P!fcY)N-J%KIW*b*~q2?18k%j)1@5pMLBN8Y*Jtw-G-N9x;BH8y~H!ETwN* z+QB}FdM@leKW1a!*-hb(#I*%~9dtW7g0)>>=y+FrES?uaWo15T@GFoz|M&Fa5SR@7ZwaLj z_Vbl<*)PmxRzha?K!#J-8I;zhKAU^l9QAF@kDk5~(~`6rYeNVUf7Wln{nz|Ej9L*n z*^bZjB&2#m4tFozk`cUQy=ykdR_thjTlufPZjy__(d1_{JPMRHBQH>cJm%s4JyZKj z#IN8T-jeZEzhhMIp#1bZ1aHD^aS1N#`PqadAjSEG+rq+Aw<~dXKRYLuPsGa2nkwEJ z&GxG%7Vo}{__pA_e`Gt7pgyz8yt+(N&aR$czY=y_l?dE@8&&jiHle--bRL&e?L8-h zwL`n&Obz*(=xBaBUjSkd^EZyrQ8=yJ;=nv1e@Y0_iN@BYpU}0_or^lt zq@JwIH2H1vkQY+!if8L#vZzkELMxiK`6T95Q`D|7c5P5gW(!+-P+UHo{9M|EKGNo5 z!L5Y{d-X49yCrcX?AwCN%W@b0PAc^uJ)h z85rpq{!0d&fBsLCFM2$tzlp5=^`rwm9pisL5!+PpbW>Vv(tZDU${86WN*5==#~(97 zK#N1r0pEcZMyKl}2g+meo0Vcn4-&;8`6(O~gi)o}#v?}ok6Wa3W(}{nwjK~3k*`W7 zR7m{N-L>LzN$A$LL?B~da-IcL`MGx8@f_>(+1>Gcf6=sT%l)L`Imzyciy(AhNHE6P zVRS?#K@1u{YGVF&F()h;@~!t^8AZy_?f#TdGllj#*fZ(vlDm%Fb zUAMihf6$dsi6o{~S`{O-`AECPchK1zFxEsR=CS6E7;nk7zuV+Y&01oM3>=lpFr~IG zhnyGr0Jh&0m3T(L%F1!XIZW4`Y+uWxpsKx-zBQV5kJw(9*H-jxZb8UUqRhI5 zvHJOQe;@0&^P=?Q z0mcJjCF8~F(7m5*T(#cTFG*=S+Do-1xaR3ztD&x39K0Uv=M*u;!h|(KY3IiUomC`t z;Y;QCHC%+rudLw!tCEq?P8MD ze@)9Zbwtq2@f;Sm8miMN)OBwd#Z60ZzRe@knQ^cfn^}})1Wk$hdV0g>NQ?@tPT(}X z_|SqdOFIsf(UVbkS0(dMp}OzFp$#%0caSV-SgTJ{;gm3%nUn57mmHFq)3uBlTie^d zZZn=czT@3m@=F>nDY0SU1n>nmZ_t$0fBBn8qSe~pXp&X#HEI&xl{H4=lXoz6Nyi+o zK(@~hy2(EIcOr-BB5mP>G^{vd8lm1e`#qX z9D}Kg^v+QpWvWT_hdS+JbVHPTg!D2=;hz`IIa85ZB8gGrdxEz=O+VnWh0+RzuX*}}!Z@e03#mZ03_6C(kuCXj6nUDi1_~Em zb(%Uu>%~L_0tyYe;gSHAf640vf~KkPr4s(vWYr0j$|6XMM+;15{h}tfanz^L-RZzJ zRNDUYZ~BC9XHrKFtYWNCha)zu-xK!fzwaQP2Qe=e4FC+*#470jP=oIjK|O!?;ye|giW=g{_BqaU$3 zrlT{m&XUxPHH~c=*f8TV<{tY5ywghAMo_M2pM$q`AHaAV{Z_YVwv6e38b4%sWEQ1X z8A~g-S8~%ZD`*w2R>ah%zM{U9aW1ip^eoxP!*OzM^}XOY;Sr&YOpP|2Le=KJVm)nS zv+g{-aPB(9QI$C)e}HxGzlv+%S$7PyVHOED946a*TTL`Lw%=VmRLoFJ;1#$O{~sW`}{-(o!4@JRI$&Hb`!pcm944;ifOa@tizIZx%Wtgeg@dH^AGe2HA& zVk`3KkX^WJloc|3oq~<2VXzU)c?13Jyehghww|!n&Ez}hWUuQG+p*SN!PApDs$Sw4 z*6E(hL1LMVRLl9hDRG* zC3aRBKhof=$z~qxyeG+u8QXFV5Q&;JW#)HGsy=~nz|UW27nkZgO!*t`F|acbU{9t! z1fw*bMGmnlAEUeo*|%*9!86%ps?`6oz*i8nvr^Z@ zbaBFxe|aSyg1v!|7-n){KazeOUj6H#tz~Zc-y8i9KH~oV9=Im z&@Nt7Q7n8H$&@K-F8v7h_nkFoWGL9$o7R~D#?GkM!s?pUZ=($1pT&aK;>FU8Rvd%H zr!3NlJ`#J%^HxT;aAhpH2J=SoLRFQr&GAEme?$KJs^ zhUx)y=0M|A&}kv+CgOM-%_ZZ-GCqM+7Vd0MzLu34ITTn=-AU3#*1(>kA=L0JMRvZF zZ#5xHtuG@0&hq=Qz+skN9{SC1jmW4b`-yB*mLL54hSA%uyM)o-vuq`|+dsWZ?VG2S ze@vO*q#l9~uYD}bw~_Z+NPG}SYOWMk4ac)YNLv{x%e|($`fOIa!57uuHSO$S4*qKm3;eUR)I5`XxkvWl3{F&%ivaT(C+Fw^#W?dOsS3H*>d}e;bAtLaKpf1 z>GS8(Os1vlzV;NVCYqQODyEUyg$PV$3`__m!JlXQrICi#Se8sGmX&j*e^qy-YXS>q zgAnrA%*st73qnj(6%^9Shx|q1V!{~_7kt4tWf4ppdV3zx4wJfBJkHs}zme4~iiROX zDC=vr3aDag%&82qu5^oUm7|M(Qr+&a4W*y0v7s~=*po=08kf(pp-?}R38$-_45$LA z(QVhD6zYF*+SSBcmGgOQBN1fs`6ULl& z9zDSI&nGCW8LlmgTWBy;%k(QNA}%VQre%cDiqZ=gti<}xwK5 z{zu-WV#v`vr93z;8T_7K*jAl%)Ytdy32t`pcHQAj?hk;Uc$I%~(ecmfslQ#&{%^l# z80w+f_4fBd{P>aj1L7acUcm4-|LuPp7A4nVGl&Ow@ftx*e>)W3kUS?4K@%Oxg+!{H z1TbKLpZ&ASez#}3UmJ^6jx|I;E`lo zu4}R~G0qC7nRtH#6jgZH-^=*K8(y%>*_-3uG+a*4k5X#isg6lork zniJQRc!?=Df8nSF&r^zfZVTbM5AwC9fElGtb-Yw#E0J}aQ<|I;DGhM;z5+nt=g78J z!TRAd^mtnn1kZPGjlnm3iumEWr&I%ieSdoT!tlj#3LbQ0^|M<+4- z>sbbtzxnU~wv?6S?;>{?*#6?A13d%t-`e#rpISCLCc1y%@{fnRm-5i&v#rTgmg6&b z8|%Ave*;>A9zl3`k`}UnA0L2_7@mI+5SS%_rkEK(1f~Hb5=Js8v4(^^{JmU6UzJ+H z66k9q#M1JTispXFqqT+i`-}=X`}5~y8>>Iw(g<&kC&Tf?MEYgR)H5qP-;drHR&Xmt zQd#mqgER=fggO>?3+2NNKJ^n}s%M%?%qEu1#V5Ch)yyE-Fh!cqcQXt$i#rtQ4}1O*7vp|A^`ngr{^X!O!lJg) zW}_B`YgF_@nBYeN`)op`M!OC5iI9vWe?kmi4#%%TiAEyoLe!B4eQOS#I$a}VuhSAA z9CsFn?bP#;F9KT2=V;YMp2*YF@qOP(v}v7&C9HI;bS{nG5w&0griO!2&+2Jg-x>0x zzLW23HSXIlwz`N1`7jS|tp4a!I{`>H77p`*09(M@=nh%Bv!Y*fWDzc?`*2>Ff5nf6 zb*>0YWEoWQ4R$#63HFU8Klw7z?QAHw*mdN!-H~DfB3h63=ag$ZitMSI&DGn1ELE7Rt%EizFg3gOVmMrv zGlrq5YdtS1@gJ%0>ciQ5qn4Es;=| zMoeUKT4+;0?U_%^b660Lec%7b*8VN`5Jv#&7BS>1wc7yX%YdAi!Vn`va2Mn_nFbT{ z?y{u0~|LOpHUFhH3I7BTZzuBBNcxRXEl9IYWVC9DzE3GYO5 zr@zxWP&+ssogGz-CnXdeu71ntDZNGuORijtnk8Dw4JW50iM6jIVyYub34+w=Q)KYwsO2`1<~>XkGCo$O-`-PLi6Rb=}KDn@S~Z{X!A<1&W* zVdDd|y{y0OfOW{d5@{F_|E5p5DNteOU31@1i(8{o^~LBGV-@PUe}m5^IKGQpo2@)R z)A=;u!L;fc_ERYXI&@IV?I#tyr78Od_T?K?b3aPqciaYT$0)n?V(1v95g1P&bFQKD zh0BHgkC?cQB0nb~icsy@$9j+z;g-Ztg+Lc;{!gn=Km|Z|1(&`GgDN$}U`lNeT0M)< zh654@)^&f%bpn;%e{KJU-%jMT?G#0k(tI92=IPmB*egNbEgMJ2cYbJxIDP`EI;2R8 zywV0f`W*pq!?`LPArt@&yIXrRF;%tEa&-*9-)rbjwN^yvWbR!Cmh(7wuH8RBc_B0+q7Rp@;2m~KRsW7uxTmPo}!a}?<_EKOGRe^rv{GcU=eXAwRmM{I&$ z3aKp@oNcd2tonN-Be0CinaV@eqo)n&)n)fE$!T4Xb>mp2Zehtf4PH640-ibrK!a(P zeJOsb3JcqA0_+jp)(d`)-OQiO6r5qg;u0&u#8Gi11NjME$$g8MnistHtHhmCF~rz{ z1Ol7ZX5*qce;{#+nCh_&Kr&0H!e<UyCPTV5iL_S%JzZj%5oY7eXu%t&t9YZu*X zR4oQ^oKTqbOT9AXw{g0nNSYl^1Wc;8&rKsHdJ?M>e!G*V(gt$aXpf1KY&7BlBT8i?8P6e8=dte0yXq%b+6Wid3$l?txzLj}xD zU;7a;ac?(l6-I|DU5mfwFK5PD9kQq5z-(SQf5<}33$-#RrjMy>vs&)oot8XGNr8!a zy{LYX^1QXgif^=u4?rJp>5DFSU>sa)O^|3Va<&??@t=sm&$(flF9w#-yEb|)O}gH`1FPmmJSD4I3Xk|f(;=QgI!D&xaA+8%UBZU+Ir063`>yNv`~mnY zx?x0IV=P;|9l?sE)j{u&>f&~}uX3u6P{MccM`zlkGb%6YmX*u_p%45s7-xR+f0%xW zk3S#^(WK)9j7FkjmxDtVyIBWa!E}I%LA4HB!8C;pOJk~Ht7V97rSQ)JOSFYSFE1~! zQ+yxwx;QpfFMgj|GlXBiJ@TXFI6|wi{xjz z3#u)ZTM{;2{%oUzlh*4CjGcRRf4J3XYaiphA3Jr6zN-(Bu{1nlzs2VEji$7}Z)@N3 z-^S!02fILV5IhVbyoj;|F&Ol8TStW^L~=4x%1Jo@WhEu4>CqJDi4a;Al+V@u=(VdF z@tCH0I2jk?Or!+JNr*L*jC_kPKW1L@=1b(gULq5m^n0I9d3l7DVgeeWuM3u2pkE{ZN}H~B8bFTU}gM+QQAa`@chrORej zY#oK_P*Zt0dA?5Amm&Jpe?G&W>21B}%8+w9TnD^8H1=Oxefkv^5{lB@9Z+Y4YT!PJ zXN{LmC)qp25!ah={}RJ;&LLws%&3SmuaVn?6nrIQIhcyon6(dvP=?kAEvFN9U~EAm zz~=xbAy1Yo!%J@TXPPnHK_8O8MnXZw>xMb4+1lk5-GTDoot~+sf9)P<2%b_d=mg5S z5nvIAVdcbj$xQB`%JQ2Pf;RC2`CvL3FkNzHV$=+RFiS9VJ<(&smrIyr=rgr{*N{#} zt~U+wjI1CiQ_Qg@SoO(WHz<|}uh^Kpg}#EmwZ6K%UEdHnP+oZz6s)ORUa-7+e0=i+G+MwyA*=s&RVdqvpC+8Y6qHe@SB=NZGet}(f8Qk_O_4y+{`T46Tj#vo z5Vn0pA(ubZ)J?j3*fc z;401lgS%h`d53|G9V)C;hczm=*!Oh`g|&A;{Gc@CB}`ZRh2? zFma2GF0b!)^pNoYVB&N$HurpY^GEQZrN_PZCs9ouoQ$>55c(1O~nDA)9Y3YzT9buMLu70L*wOt<+gJikbsng`sT9~#TbIZ16 z2b~|&aA$-Z^J9O>KD-AT5YU~7jHP`xYFAkZNB-0v1H6RPPb~R7$$7yRub%QFz1>b> z*sB`1e{vqVN1;p5TxaXDuL16cCrLz^7+l^mx3BL@4?A6(EAiaxKwT`tyl7=dA)<)$ zL-JN9(%%nutpwD`9*V64SK?w)2^(W+1wgKBUL0+J)Ci0wkj(0BoBFhM0qdIj4yv`Q z#WI@NghQwGt5rJ=K^HWU7t2BvkmnGTk(Vl-e`VQ=un_MEOHR~z!$RhmlvD6|IaYk; zPbPEauv2Ym_JG`F^?|0Zi~mr z4syVW6=RrKd_Ugw6Xtc;TZ-aoUJb$?F5SJZF3z4+FS+PUk2anQKH{H#aXq3<@*=

QN1P)Q5e`m9F z!NlVm=u@gi)2zSFn(2$=%5O;R73V>WC(jT>jL7=K+q+%%hO<}(CNK54=cAVUEkWv) zxnK5uNR}$T#?X>4lHG8wVyYse#tdMqxrtYVumMi-$@}ZUai-2KchYSM-z6-x=x{vF zcMtMdzR6Dit`4PCWx8xn8rfQ7e?D*0DeS?rzv&Y51^6&^EB*!gEM13hjv1e>Nex1{ znAtjHVb4)I9)rwc6kaqqc)|C8TrFdnot{>80%k>mE=bjpY}8tXFG(j)7At%o7`a(w zGYJb(RA=d9M*u*%%h4 zhPO%ER;|r#+j9$M_hxO}r^|gC_Tgd14%?fkhO63Rb{ZD;I$3)F6s78P6EQ3B$v^9t z`zSbkXj5=ahYQvckh4NdR7Vh45Cc9DIe#{u684BzxtPv$->(7JR%(JPdO!Zk4Y>fjoI0D*`EieI{*!XHBgnk$SP*)7eW-R?NP!)< z&6N`=)_RL~|k&ejd7IC1*~MK(b~^%LmTtHm3pQq$QzU7u)0H zB%y5vBp@WZD<~L*k60KeKn%YU$>_#kT}qwF&?N>98pRG+?24O+05TX@DdG{2Wx?a2 zGfYcJKw0$Cz$c#zX(|Agf4tVRCoD!TUsWzu{-$d4F+2GPe>Vky{yEvrbdCVGePuV& z-xwZ3ZTc33=ttX)XEfGO@Tp>dG(6E+0IP6wiv5I22h_?Jlxt;R${3giwE&HxE{dk5r;8_&X zJ)E4eA8~mR0O=y6uGk;b3L#cL&@QSd_MS-_w&ATju>?zrph(z?V7Vg z69@Jfx$xt8J$vB(YW3;#v}1Alc)KAw%l(Ak;r!V1iJ3*(7knABt#m3;#^M~hk97Ap zZQgZ_fB!aA9V@}j)WYglRo|79uZI|H>QA>f)~zr=14103EvrYWrzDR`9P)@&>^j7$ zzP=jZ%Vzo?v3UKK<7{dEECtt-E74}QmLU(keAjCzPp{3Bj@`NnT<+_|uiLqPC!bBH zwpQ$KbHz)i)6}-UT#nwnW2Yq}&p4&kCDHjM#6^o*V9-k)A%oz8Vp`i zEJDX_vKn55?sptn(7f>ApFC#3*B3y&xf26dJ4Y&CED!VC zZL@q}4#6;tE+c6onPhXYVZbg<)F>t9sC$pbN^3PFB~jRkKuqMKfRPFyKndt=967#x zot>`yE&Zvv5X)PQy@I60QMT^uxy(44rwclS-G-sHeF5SW`9amcLtLpY8fAnid zk`>9O)Vei%T>v}M=fkD^CUIRJGv!8fR`v>3c22K?2oue<6+5W|8$69`YAY)0g>zKg{@@3i0NdO6JfAzNoc(n`N%kY742@4;>CoM+k=k7{zpqTJ&EJt z?3;aTG1f59nLW+3)1WJnol#`0fBBMqB41+$AcEbJdhbetk+Wi}p#m%rE+yMNAG zCy^_S%s3sm;p_=J0pt!Bi;tf&bo*ByV;*{*!Zasx1}S8r)vQ+)JWc_W{J{6rWf$VE zfwKM9vS?MBdV_~Z7*&7BJdPuw6o*Phub(Zis?RsnifvS*UM0hEY#xK!)2GLb0#R(9 zH6x?Pd}N`!qiJ3=xM)si)(4nZZ-1MlSQQcLCTU9er6OpNhFX$mJIobqP4^uD!iu$g z&=lv?sDdecEzw|yr$nhJqO5L82W_vgn#~=?lG~9 zkJ}b(lt|E6sL;=(qORO!;L4$1qiH8CB+Z8_n zs?Shg8^jGUO6FYEns9l8pJ@}9jn~ijMC8(=p-WlDai*|3u4HP~y75R@KFyk7>u%)U zPR>ASFy%zNZ7fp0WH~J%Ykv?y(H(=;F>}7ZN~Xd9vtAkt17i;B%*;iqmKEU!83lQZ z=}rI%;-Ev}J>yJ^aCcj>JfNk!kVmuD_%(9G;JSN80u4c>Nl38YOzq{v`d%p7ji8xt z%Pu#h9&t8d0)c2=3V|VG!v3kywx7T=N%$I83k}QdTlyej^3004RevpU1328d+NOc~ zHxYlSUOet%pnk6a^@iCINrB&5>FGvq)dsxj{uA4~G8D#~4N;|)w2ib*7FKi_gMk+M zcqKsDq0&@YidmS{yN}};vN8^M0xGLs(#IGnfy|dGt>?Q!H425QvhbtOzA)AYS*2ql zoH2~#2djPn@o*4-9)Af-wqiYo=A#1Y!s_Fu3B^|l1I9z7p*6z0DJFn==+GcG3O5Om zxYV3|=)hf(pntrOSemUft;m8zqvKhHET~$T)*o|i;f=E~+HCkvqCK4M!#nc6bI<~f zUmp>}}#3iKB>I;Pz+<;!kc@M`K*lsr`!1WqZEP2z`9% zp6+;?p>j*U)`}PvOv^1gJuh!3`VkBUqVx?4)ib{+o5CW$e|1MMC1qkVlIlKna>fo( zoZLPegs=|3t$(1FdYCz-r)NYHb22tkovmpaTE+PwEV4ykqu&^;q#48DTlfq4S{$+J zyPcIfdVB}WaE6kuu49nCUo?&<$seCLlN84Ug%@~DHQP5NYfcbY)pxjIjo7L5t*D4& zJ!*~G^4_iIzS*WxKG0d(qg3JUXI0OQQL>lY-`Csb|9|TCj#ybR?U_pz?n>QBomr!0 zX9)8a6tcvXIrGVLt#jVn#+NSfPP`l!vV7m@x+xBdCY z;egGUCeXw#=5>x#o1?YLAE>8pqf?o|BGte!n)sRp?3LJx^>6O!xx%Y*ApX^O9{G?548k^ptpX~Hxx z6Iid~0pKE8CoJo*M!)C6Oo5=3nRBF)5f)=-#4fy&*!#fM2^?FGarLKpG}7+;iLaG$gIGJ)o_tOY2P}tL`L*pRe z?Lf^upkDM~vOsdTPik_r9bl2P+dy8Kiu}~He-gReT-={n6j6sw zu6t@Ryqv2s+wUr0>A?dDV z%I8tX2M!or=W6}`u=f|xl`L7BCK@4TW@ct)g_xO{nVFfHnVFe6#LUdh%q+c|HI>!X zd2dx`y_tG5^H%#>e&jOQG2iB9$6Y4<`~PrE$O8#74ImPG6>qCRCuX`ADgr4|qkl3p zAX|1P?(rD7A+yrtv)|V9&VrOdlc7icErC}CtZr`Z!4|3 zC_ed)IhS92VV8Y)CtZpfJs&oBC^@TetykA`DmjOYHJ4ddjI}E*F%u4Sq_4IpH4jd4 zaAqH8QJk-6PIG3PV6PWWUgCGRWm~~WT;ey28DV;?1RFPR^(}4U?475tUVmD_czc94 zv8z(jR6G)BU`Cf(MvW;~SSoR8wB-lZSQ=B>Mk5nf8YRcIxUSq8p>%WQP&d>TO;)0J zFhAFlBvG3&-4at+Fx3SQf_LVnQEMCSMwtw-+-fWgC&&J$ae5~Z4nuM-nr^wJs8aJt^q)1WPBcRZ- zOpcozP@3#aQ`#xuH)1Sae~pC8ur`@jROVTn8dqXjoElSP5i&|paz3Zl`cO{VoEx-O zR(m0!>>44_GDL*5*+o=xo}X3&KN3Hx*Js`svcAw4Ql(0?cipA7^nV|-mQ~{eAMY6% z%23T7vsYx19{fcgVxlu$-B3qmys!2%IcTvJIE&f$^+F*!k=6{$bg@d*sj0b>A=|ptF8e*!Hk#drnEsz8(<9^@~O>k;z2G@kC zK1kB(O;UBp=!&K?x(J0UctS1FA)T?o+Oh_YFjQn9d>`YSO#1Z~%;?wlXYcpj%l1Rc z-S^K-DS7X|(Alv5S!cucPh<%cMXEsf{(=(3Rb6U-+Pd};>whQ6_y0mmmDElJ@=ku# z#T;&Zw=1wd$V%{81<8*+$wm8oqsAvDEXhSGFD+tibbNq{BXqlpfBl@2%QI(+D9dK(W6#$;*h^6KEit7L5-9s&-Z)s-dWc1%CMPgw3Hz^YPUr3Qy|3UJ@ z@Yh~M|Cto&Kk3y!t43mA{<9j1@gG%E41ab^`G1eqNdI9_e-|XNv;3nViJk?Invt0v zkDi5v4v(IWh3&shki_<%+a49qCRxK? zJ*_Fnz%(|uz<#Brhm#AQMv12SYPq55MER$rG-WEBK;Vj$RE`LWg+s7`j?-#?X9+!{W7Pv5UKWt~EK(@OA zLZE`;=m344O60D)*q>v<*wA`{-}-;l-fqzwfG%#-U2^ym%->f$Z)R~)4bkr!oPB2< zak+k{Hb$O)lYJrkvV>>HWN5`!7ro9dL=bO+l%`-YM``YX_k^=_?hg~cGaBP{m-PQ29luX?US)KjI&%Q^~ai zaaZu6tUuO3`e}VeS+j9U?3Ei|yl3+za<-MLwoOi+KQEn}Ks$FJ)bE`*6Mq}Bo;cUr zmP~T?MAnAQ18yGP$-R0$^`|xPKRA1ud##pstUsirXWVQ-y?C|9Wo}u^c&eYHKQLtj zT2(+#zqXGyqrKkGKC^_^*7-U)cwbR{M!zw9)^>Rv6ppM)LVHTAcN~=;Y`8daI$MxG z)?a{@b1qlv?yp*=cD~y+(SJ{?K=5}!U<2bpJn%90@ZjU|fZO?8PWTx4@xrhA{0(KT zD;0w-&v^s9S>uJ`Zd+p1enYUR7r|o{uaDFEl7zade7yQ#?xlOjH(%D-~ zmlWG&5O2OgdGqe?!%h_-t#6STfV_YMw%n`WoUHd8xE%3y0tc>LPYRZK((&bJBd&Pv z6cM86+w<4Wz5u%Yuz%2We_lG&B{c>%Pm}7nkhC_Lx_mi3tTsM9t@!X^TLQU!ee=MQ z03CIhx_WKDL)4+}Y4LjVtO?1EpzFHKoD!7Ge#8*+1$ zDR|Y2q&#(#G54!GN_Idus*9^>36Ji>P zM@rNw(UG2w<-bV?L@BktGRAC7yXd2KsO@t7>6oD^!Cm}p0Ad5vgj@LcmaSA2OQ|Bjqm8RjgkWo?1@sYYqj1rhideZHlfTBkTB*B16yYo%SnV zn|3!*P9TqcnWy|zii8@8wek?TMw#Kh2hsID9OUB#R$|}J4;02aIUK6>9jxpD>>K1u z0#^qBWj^Xecp7VEs6+X*2fV1xn#@3GN96hY1WQCtp%rXhiCw79(0V2)mO4hkW$+hF zszJ@NOn=pEa%A|gYsAWgMsL8x`1l+>k{x+g48jYu8nHKS3=cAshjWO;`KdP9ilqu` zYCmUqto5`_aIK$qzG7`bQ- zJ9g$tz1Ay$+_fOe@I+bKO)+<@>TPmg@uu2?%Hlo&32dp2RifRJohTH7ORYKz$mql% zn_kfzJxQ-LQ9i?v6cG=MJXb93y27G&hSaO^1;w1&N&;gBl@X!5r-5yW`)d zyMHO%?J*PR*A*7n46WE_u^+umEXG{SLNUQ+o+G-SX)BG=V);G0uTI^g^PYdKKV?py zI}OBpaQeK7;$GvDAS$Z&eUio$QH3gaygS|qDqGZ;f2Ba+lVt2)jI_sY+(PS`b8fKO z7-nhN1oYT!4MrN9I6qj&!rX6=elO4g*D}#C+$Fj}|I2KRcM4I3OsL75m>L-r z;w93$!wfdP12zt3z@N&UFhwQ1FkJ!?=xI)8+cvH#FflpKA>@fUw*tHd~K_Z`G%*WH+t@d2++ z&y^ z9X{r)e1`xHwleRV*xdi%R2l1uETUNEy zW9;xGB=GQRU3t@5R23@r{GH`Sn}ZCDhA|q8ig?GKg+@qq^bI@RtQ0t$j|+YC`#pMB z_^={36`al&uJKh@tPYLN+;Tl}2U_9_E`NRqJ~B6fsT7fQ z$lbBN!x(hca$P^S(*qaJAg0gD^0wf9xdgJV9^d41E|nHTkOUq%&)6)Sl7hs-2c9a= zc;2238V@h0V801zikWyD=t=Tf`qj`$2ZvDwyKUHOdfJOCL&48wl%5UM`$uVv^!rul z9OK^G;COxCZonGjxPSNPjbC}4vvhs!r3yHR#~gc2QuPcKt`wh5f2SupK14^REA^D0%n zk@Qt^=%Mhfoc@%WM3=^2jwk_TGJ&IR*FD@P+SunW)S8?@4u9su3ECvxb(`T=V-+>t ztYyWVylUC7!5$7@;cA9Up{ER+8`-0q7c)MVv>h_a8zUZScu8X%*1%PYSEtz-O`fpg z`h6F3wzcc8DVi$MU+@LQF7KesmQM(n&bd=sFaA^2QY&w9D9|kpyi3=!CEEQA%fh)^?F;RcJ#x>bmZA0x*=&?5K!8vU2jPK0S9%&2iZCoF@_$*O%|T7eHdh6&59=}c6|qyI z>_izdQpJ1^nMkHMDZ|_HVQh?nBscf^Kv!^1FtHnO7o-$;ibbC1vx;dsy&*Z;n75t8 zuJ3Gg9flws_Gyh*qvZa4;jYL}SWEBAo!(^@+*p(aQj+_gJ6zoK5ucAQi?6J!T#`C! z`aS_GLVv4tVFo$9dR|RCak4xG5}&+SE?GX|YSl$7YFy^iTfC|S++6-M0wAU@Y!1iQ zM}a@8+drLV9J@rNq=v_dH#114g*(ZXm={tLp(X#`Yu?>mx-UVqnN*yL>}Xa!CAk|u z9N0eO@IfRa>>18MlHQP#1El8$s9vO}l-HJlZGRzheso9Zawc@MvEO}aEp-%;P`F<# zHqNTUZZ3bW9~I@KXDlOqO&^$ul-)ISbdU;_nK)bIEHnTx3vTRmA5T6ztI~K$Ca%c4 zoIZXz8>H==^7(o-`Et0dU6e?vRBJFiH9zfwPD;Ymt}fl6n`$wcUB~tLc*^JQele)v z`F}jhkWs-l0Cq^BN|Oj;GNzYP6hjI+$4JN>6t0B`1xsxn`B@4P1)c?i7Uoz^OmC{80kq1#T7Y%6ezJpm)7{;$o1l--KN)yxc-V6m?o~-b^_l+zz1D zg~wP*d^=hhK(*JKQos^T#C_Uw97g-%bbrJ)9n1cz@lhd%xvF*9jwGSpS2@9a1#kTQ zWr0#2(xCF%Pe1ntBZcs0pZP%S_b>=n=AF_moxIem|0)&G$Hbt%DhOnlX^p@q5L4!H zVLU`~c0?xLXO}snPH@<`XwVxkqLMGLKVtIK+kc=v zxq!M)s9y5sfz0(o;FMPbM`0>-h4hq88376~i|XyQf0YBYl99%lT@r-Dl^lM_Px=PK zsO5K!P^*(x3i;66q_a$=N$xTIv<(-;YEFX|X8c}Ye{u)C34lhwiuFou{Ie0Rrb%Rr z)-f)HS(T4#Uj1HB6)MmBhvzvWG|T+2{{XkaqK7- zBBckKtF}W8w9awja_?H<;uNx$zU0vg8T4KOfhBh%n2RUs@M^K~96-E#bM>_QN!v%u zc(g&F)69z{uWUotSj4u03^C4igp`y-7jF_1|1$>vN!>RQRZ$%bXKYB4_kY7ma!bxe z`^|^X-SlhGu#cwV64&wM&=|DQF|VN>uhchf(Nh;s7wTjV7coozddWM_Cjs~6KE6PX zhx5{W-7YuGHJ3Aen&Gl@)XNw3tyTcMcLoMhc8+3i4H{O-=<%uNlqe~$9)fpY?mIXj z8pnqq7@9$EFP41e5#3u9@qds?)KFL{Wdx&il8qkDI*qm! z{v%iMJCsRtZH?OL>?!?e-D&qj>%)s5`Io6rB=6Go{qXC)L%xH)y??%gC%YZ{Eju|e z`FK910fhmjSCZZMUWEbxxr)3#N3t{3nQAO<0&d3jcnoD2c^LI=(g1b18dAqd3mHvk zLrG7q}Aa+nZn1^d-lvdzP3@<@9z6=iyA?cQ=6% zN|F@Cr=!hT>Czx)IDbU6rT1}dg{RweHmAJ2I#3M^T#dOBtD7S2!=s39z@W7y*mGs= zvPf0TkCgK*+X(YeEDY*Mg8lMrS8b&3hn2$zuUGmAeE@nw`QCJj|8-L+1mv#(Dh!%JjDB~8b z-x5UqU}z^X{t!=@mKC);Cg!$d(0W9jqLW5Je>}wFqER@u8{hf8AM5u^#``o2u;e20 z%L@SH`ZR6GIQ5o?3*LY0Mh-|6=#09(uhg|TXrp^=Cx=yi#8iSR3|l8RI6Iv??;eao zZnj^FN)q{7PJax(5yQ2BQMv)_9)sCa*8jA;0vW9BQ=VQjNQDA9)h-$5B2wpNHZkDlR z(hh=ZE5rG|c1uBO-G(2|ur7E*XArc@KG={&4vpeW*MHfoWTUnlbteiE^HW%y<{hpk zT0bGX3-SmUF{YwdG@gU5ornYO)9xY3)CdpC5U|KE+fPp1g?cj9_`;eZSYMe6*(1Cl z643+_WcajN;J6A$P4p*I)@i!tR{gNO{rCR0O!~(yly_3kw7kX?W0m&W-9|7-#=}XV z!1eQ4Q-6nr%Th|3Dp*H|&sL-N^_5iLeo37BSC%U_fSY&Co-RtibCS{=zM>VA7=EHK z8gjs*HO?VQTALV=r*Yg(RAs}dRU8ghdtNN7~8-= zLng@f_`pg@J0S#S>Fi&fEc4|0 zfxP&A2v&|tfq^V5b@=RgJVmoBx;~@~9P9v4$sS56@;UB{=} zEq{$4U#Hxwk8@eLncVNa?$Jkphppr5T?;;`rIMkaV67JW^-&fK1?w%ni@d97{D- zzaQq$otc@S+&#A94CJlO3FnGcSh35ks(%zWt1^m_zR0+y;MY8VUPd6nw##v`5%mB^ z#rHumKryqapLF6EE)w)r-#oDodEUK*#Dz&kpnSD(@{T z4L?#;5|WW4{?xRPB^nS`wNXSSsW9ZGTfma!e_)*QJ&M4VQ~1gwv_iHD&wzo9JAd<$ zOqL9mB^MG`YDXUu6Pi;LD0?WZ7c7_ewo)ZrBs+($k)SEiS0rXkZ=^{jaFJ^-+hpo7 z!Z^|>w@uS5s#H`l!?f*c2Hci?GP^eOS@0gBd};6$iq2%i1ahOA7fq{S-2D}h_~3PJ zZ8ns&^o*(KGVbV{_h^1jL!S5OkbnP7^1b)m?=B|mu)i6+eJ1-M9vxqkNT!0sJi?@( zdiEvaz**C1I=AVXx+9~kHDt?KI-5Q$kAB(zRyqV00wUefn|s6t{e)%1#{LDwjcK0e zDav~ci)MRU?#+f2z5^>JGa*JLRw)fN^Vl)q5M~u7JO`zP%9ZuFU!3piAAcUtac~pA zjb)z7XtTH!%q~;Cu{J|jYu($IrTX~rkuwF^fH7CLF+Do3E2egzT(!t7!tP_QVLCDTDuUzDn#=R zcyb5Tb}1FdtEVf7p7=McXMf&?sx<{`irQG!VY9tQTfv6f5rm*-nC+H(a>rqi{OOt& z=W^9*lf@+R(Oyc;N6`03rt8(*?S4x4>&~YQ>m95sc1E#?uk4LaY^5prvRzj}VsNRT$F;>-ioPQA;@S7rNXx9DK zg%-kg8;i}VbZ5u#gNyg*AIOQA(Rde`zd$t>CEbmO#5KC=p`@auFxy2-2Qu;4j6--lSiVgSea-QDeK#@+U0&v6ixXAOluU`tZ#L!QlA zkXjeA&0@@C8izL|8-K|wO5-yJr)Poeg_ zc+G0N?Of4l**kpp?^|l=l&`hY;_u2TD{lc3I>cXuohMta74^X80;WBv7$L6W9sXh& z)E&OuF^mcpPk%^rmE<_GT@8dj+hrAq>7L}P;e)rCw(*}87|?Vv^?~}kX0@*~Y=72Z^J6_fzjm|Oii2IjUHNHd zVbNPrgHxU~Rt@TmEYdPU@|AHMvkeEsk zK(e)SMh*d_faFr`T0o(#>i0!JPxS)B{C*UHDBv-xll$VsE$OaFkuX*SVHVe@UY-P{ zPe_O%7JtD_g+O>{C~91mO7RY*7CZQ+$IHG+hZJ1D%chA_4yEuJY)2P;sF$5-K;}=t zf?YDMkYr}wt%oyVWTZ+mt$Y@b;7k~1G7=i*9GI4?PEjw6dUVb5{hL}Xa z7?!6HpSmi>C?#DG7B*%%2bDf+MrRrznLc3%XeVQC17DYn2RInf6(X|4J}_C zVhXw~#tJNRl_(PUso?#^_|ez-lN(YP%2GhsQfJ|9i$Nc4dW$DoiMIU$M2K&4@NEylDV2q7f2Ga!1WPbeMAE&+bQ6x=qh~4;Kg40fDutE= zIR+SYQ^Vnx2UCEdqrX-y`fqE8wO(r{^-4U7j738hwM^p%v)DyTM_;wF0T;_7;gCsl zdUPeA0IlW>Y-|9!6H8VX{SNjQ3eDs80LiMh&(04SH5<5ZOt5h{uAY2igBR8K()G5D`O9|dr_Fxyuh>#;}+sM6f+q= z!MhTk)sjMtcC-Y^up>c<@TQ8Yip}E-0q0oaqVZF49R(W)8wnkYX6p6qt4_?13EJ0L z+IQrhv*nLqz9IZ#kE(vF{!4EY8h@_eII%9)5#{&b|`fO zX6a__R_)x|YR0Z(Pl;U+*{n2-v<-@C6_!d*DUs(B=Tw-Mm={e)>f_Su(jHm&uu~3G zEXPRkMfwK4DPMqTu_nch#t&;H%Utvntc=mmu<3J+uqf7O%5ju2)@l(lV1HsuP0hN| zPyR1!LBxja{F?Y_z6bW=F(5FO&^O88hgH^)(iUhE@MXIa!;E&egBW2keQ zT2PShLp+O~A_@C*5a_3(x*L=1r`^s#y1n1T=+bm+1g;@%gK;x9t~D<<1t%G#bfDf> ztl=k}X&7Y75L#w3TZo5Zd?*h(!514yD5LH?_Kpii)zl>oc|Ahka(_i#wQvKs)$e6= zbkE<1P#kE+h1>RMoo;vw0*NFzpn9qDe*w{ZqSNodbS<;6g+1T`c(Re{F*m zu4db;yRAhV;P!A^l4XrZq6GBfcUTti1AFs5069BeZCNLt(p@pJs1y64E!XLDoLlqC zC+PTtbblX}sbh&G)~`=3Hg>G>ca-qMa%2UTKtrD!pPYr>p??u{VBjk=6Ahv5Cx|?D zj+Pcf+1bplBBLN@)N;>`QsfQiRok%ZdX*-xpuuC4!cd{3_ffK^GoH|r9Xgvmoo*h3 zRB#w>oeKH#Zy?eF2D;+j6-R1|=XcAj5w5J#co12~uLQ{BoI{JjPh99)gPO^M+BluX z>oxK=j>}f`!hd62v9Lw%<=EQE4Xp&qk!O+3gU>JGZJe*;94AsxkDha@YZh=bkciGL z{_H6%aXGTO=((&CmX+Zg&(Dg+WV*PZO2MoL_qHKA>|!Fk@7o!^0QB}>*)eg=;j-vh zk^BA_PL^xtDtN(|vEA3eQ!kl3nEk=>b&p%atvmicn13SRb(rw>%!+rJTc0TnUMDM3 zl)RQv2q`Q2Xd;fLDE}HS4xJRG2vKI4UmrfkLNQ)OmCX9v&J8C?!B*2wpvW`tGTjq= ztkmlljsL3RbaIAX^8Bj|+gN+`3Ul-17299J!fpYX(jHZeg>f#O&PNL|IE+mGV_2mXq z?=3Y}(`VbxD1v%AU{W%RSCruYcUuY=gz3p#ZeER8dbSQlwqIbU?v^u+RYw%$`GZxej{!Z$d{d(k76K8KTg4=&19^FTTJ8vk$C+OQpjmYKuWNQ)uzD^-wxRkYuzIF`*U&`Eb(i->Fx~E< zb=vYswb?>Xm+by=zu0l50sOugnzp#18lKjC(V5;U0lk&M`WBu4eu2tn_o}f!Lg(_L z&3=m^J471(4(*d+^Paec#qycHHGhQmb|esf4c!kz`=A4iao}p6D8stn^T=BnxFs8Q zVaM5ZP2Im{!`XX*85WKPWyi7A7Z8e{`(5Y#%~zDY|K)|4^a*PBP2jd|laT0z_-WNT zP5aFP{%r5r0rG6?8DW4uyluOo6>76)yF5i?q=J97u`BOPvI~Q1OP;wY}4t zw2y+iWbf(KAmzpl$hv;h#1A!-_zSvw%x<0%g~Mr|5@qAXKCr$HPG@UJRy+DPG>qqH zAg#Toj)BsWLuK6_hk9c*$Z~ZZxkbkg24=S{w2BI@qOwD!HnUtAPv~OAfl`mOyt34Q zlK1PfSZn-#_#t-^GFHm-z<+Vr#PRh|`SFB!5@rsg0Wq{9iaxMiIRSVI`>hIA3XL6w z1Urr0o_=%$Te8DL3-|j3<5&Y$jvGi0%|N|ojC^t+Bs7Fgf@NiZ@;_&c}WAuD>Z`lI*9$SzB{_rRksec{SNyzK+jU1%( z!3*6mIM)48Mq;N0+g2cKkT4mn(GFX*f!T3g0NrwcL8<^cVl-^~tsgx87$BCzge&jx z>@$1fEY`Kd4p}te>8NNlVK8SRJU$!OS~--<4jtvzxSc69Oj&O))^(ga*|TOpAL#8> z+{rOp?7o&udt%3i>woc&4*L>F(1#WZzz-|u3!6c2tOoX{%YEfH?S7svY;j!Hrzr-? zF6>mJA#M6pv%yHt32~5N&WY4)W97N3RO6v+?nrSN&?h?mXQ+!MX0cP{P%Q;5@ns@O zC%u$JqN4H5)cC+rF=OJ@Xfb2SgVf*jThVrcCYg|4L1-S(27jS}sr-@6`6+BChiFdF zgrIsrP6TYhsQPGj6p;pG)DTg>U`1Gm;i75MclI2=uJhPH^Y+mjQS+k;77@x-+0{*( ziIFC;q+pornP!*(MJA)n@y8xyNuY1=q6kXnLj>a7@f`4M4Ie9QSH+huze8|8dW3hr z-y*(v;dh=B{(nO8$@FK%Cq3Oi;T}~qF$&|aixe=MKo%Yz4mgF#w)65;bBw#!ms)#A z^VTF~{d$kV+eTi#GKT`{?D$CZ1`;e!xW864nv&z4s>H!q^Qo??uC~6iw#peuqr%3) z-VvzS-rnW`^!?xtIOqxl{ujaig>sgWh5et9rz&vT1b@(>ZM-7m%;OW__ZA}Xm(K;| z!6rF}GyM1pjSa7ZLVtN~@?wb#2P%2bHeuUycvTII*hXNozXt)vvSIO71&&cUw(w78!ib4<(Mi#xKJLmMXUvTfP&P z>toa&UNL0MOSlwL!$6X`|1%IFRg&dl}wTF{_N+&g#unkC9q;Pg~ zb|eR~Be=o5)Es`^A?9df?SK+2fA%eO(qE|JS|-6sX}lJ>vax)8$-x+nx-LgeyeErm z_?|B3P!{`(&X-yi`)tQfCd?M~+H-H5rE-7#!t?zTbzt$}FGlXqptS!7(mNUH|1M8u zWPkayJe85@A2gthe`W^$H_|&98U8|+`j3iP#y``E{@+FKWMuwN<*1A-{}nmv-|3xy z6tVpSt@A(I|3R;c_jkQ28y&>IQ9S<(TGc<1JR$yTI@Q0EJpcF^1Kq!BRR1%YC*$8a zv$VntcntqKk%j+g>mMBz|8HV^viw`c|9^JAC#@9Y-;uok=TM(C%>T#B-}iry{ABtk zk)L=_LC@L#kX-V7-kR85!4dSgUln-jLZYH(65Ar{e3t{}d4{@;3ZQjQTMWYpuxpZb z_Uv>^9c;ytH~84Vx61JezSwe|xlTJ76~coD+BxOd|n5k3=^D&;=>g#55k#eY8) zKX*?t9_dU=gj^_1TS9oTjeMg_ixJ(5CH{(Y@&5(&^FO{2jBNiE`sW;@(htNoOUhuRK|aAZ~A{H6_t&S?ysn*jO_n4 zDk=jT%b!S}46ID-|48O!U}a|dM}JZ(11k&LpUYVPL@fQIjNwm|(7&dl{_#BeN17<> zpW^##Dk|%rsI7mTm;E29qYP|xeK_5J{i$y(?96}a8w=f^%II0>{&el>nOW)oM0aIoXZ**0&@r#ZyJ!uaPhHl{z#D;7HXf1C&OEPp1=GO#kx|7k2)=-K`> z*Vz6in(dzmvVTmHKNDmb{|ztxKM1l6Y;3HI{|rI)e{cV9-u^xR*WS|E{{Kqu@87q7 z`FQ<@{9hJ27AB_u!T)7q_|a+5R{G_pkZCtjsijEK}H-Xqf&?`^Ee3 zp#4s2NhJ`q*j+k&hx3GkqTGJ&CJ?Xd2huc)3cuiyW8H-p5>1N}kVowLTtCY${xWu! znXVkz%tb47-pco#5G06ul`VZe(CPVD$KCRLT`l45_!vJvmRXXq>3`;YAIH7DUPg6! zCH(sQ+M~_c+pd|0;qBrsNpJDx<;M28wX5Jx z3_K2?)v$KslR>(n1uVtQn+jhBeON`h?tN`H&94QG*SH;~Og6 z%jXN^+bJam|F)>8YJVtI^O!efr6pVaJcZR}n)iM0V|4nnNT>V7-1KYkda9@WU_x}&$D#}jq=1OgYOR;>fCHJl9@@|mEWA<=NjO>Zr}OvXCd|g zxNpH~w>08}R5#J^2Z$Z;iuWSIU`CMafwCTwm%xNCu&Cgk<{Dz;NiU!`pH;;@$g$pFQDHpGH2hHHtbqVJ4S-$;tbc(PKjPrU`<~Ehi%3!XB82Ja zcQK1eQ({(YYBUDg`BS>Ac?O;%RewK%V0GUUBD$?9h*DX7zySE%%{xl@oZ91myda1B_X@f|v34rUkHXm1Q#jR}SU!N4={(sstk|M9GzvaCs=1;t@%6*WJ%sfL= zYa<`HV7*E350V&@{_>VNWZ-D-JZYP0SXWVM6%IcBdJBb`=Jn$7P|4+uq~}+$FdcRs zlAI1giG!w2#bW{kw|YNPfPiUgw#bN?P5M=v`|E}bZaOgHzDf%splz4|M{1o1<@cFI zm4B9>cf`Bs`@IUACC%X4ypSVCY^9UBSfq|@RdKQd*0hY;PgI?>UC62*(pAO4C?haC zaVtygsiqmMD&m5!DP6_X&*3wh1S z#L;jl1FJCmD8<7f6f2~g0>Mc_VYoVBnNzL96UX@OXgGz&BGvk3)qni`@suy%&k{pF@E7DIAFO( z1TQ%k{*9#O1L3X9_P(|jgX_aFU4Nb31zYHG8C@Q-t(Au33E5Zb8AuMJa2H{7N05(T zQvjStno?L*F$mlnR!QV4iZj(GD`Pb+{=3f5m&za4fdJcVhh|| zE;i5xUU>-QhpRbw%tFUj0{d1UZ?ZZ6NGDXep|bPWk4K;oX(On~`INvTD}O2<(KVYJ z(x~Ms+CSpD!Bvuydq;7QXgd1{Zj{#brh4ujx-2V}Jq%vuir+wxpUa@sU8oc(_qpi* zy+*&ZRtrqV>H@5T_;dDD96j&{Ll<0MwW45YQ#^_uK-xB*a&oqUlGOSU9-pij;%Qnm zmU4`a80_0pL1I%fr#iW<+JCR#m8 zaMf2npuY-jn|MRLzw$nJKUI*h_a4U#+6TnT9_0pH5(@@{Hh?5+o!ve37s9=HX<&&t zc{7RI5BhDBd0_HK0|2X$e^y5}d*OoPoM3#B){V7C^k2ltGzyPo(0|mA4>c?}JGx=x zgxOz^sSV$&@a;;NuF%NmbU&n(%S6H&l!;8J%1YIhdqmWWAV_jInFc$@f)zQ$={I{< z#0(@KK_auLYe5)P?fDcMDdoV)87sh zA_eHCp?ACSe_+!M34fYpO?(BTKTjmu$c+C;2PN+LI=SX>a<6%9WV_ut$K$zrV8z1k zfSA@cB}5HWmCgp;ZPFTgN@~EB6+Z6L?}qQx7mbq`>%D|4yg4EI_*4e)ECmA#+Ti(} z(EtSBX8?^lt95_^=M@h~YW)Kz4i#4CFyKB2g0`EA+E=S7)qjK?(cNL_>4#2r2P-aF zbKF=`eY=6)?t2~FQSC|^hI6%ipzk@I!5MyD*)>3kmY~bLr@pS<3Hgo_Pcn-BoVCfb zR?5Mtr_Z3Ol$g1K5%6z`Y@xaOIg|zKhaKod0-EXVGt>kShLl2DeM=Exg@TJ2!v0^HnHC;aZ<34SH|`f#Oaw8FzJI_j^47gZo7o}dsj+1{OpUs?!y8ZNH6-km}wbh zd{+h~t~n%x+Hk!lyF)#l>(B^pk#i;Ekv*kKg-R$NQGWnNn6$t;7~YB&#Xp078Yaco z)j0UfTXFZn&iUtZ1Zm(?q4f6~5BMd%xQ8PxZ7Pg!vifzHR4<;_pCkN|>P1kNT$ZQY zVC>!2BJUgyB}dN~VvOh6O2&t(sTQ3?nLrh(w-HSmvNQ7CdS55%!XLbq*_{- z^cq_Ucz+X>l{nl(X%rG_KjEj{^0mS}iz;NJpip%jU@&@AkMl@c@Jd1WK}A1jf1xX` z0lyZjOxOtC^+UR=82XR_XF`3jbiM&Ow*7G8xm;=o;`wYoo8Z|at~-adEJo)_YP z$5Fd8#U;n^vICt_(miuiJfp`jWME1+5-IN7I)7|GnOAHPxVSl6&%LU%Ew5?Z%UrSl zEote{KJ7=?@RVxK$GZHvE>P^ch+=J2S>!fWqp8_%en~yT#?s7X*P9JZ9&p1(hR)tB5?*m64 zvVT5(F#5ARdbbs9ME({y#N0WfFRmu;SToqwNv3oxc{Ei~)?@zi-N|T1>Fg`pd+{TT#aarSParU}3opu03 zK)b(7z_czqWg#t75Ohr%Ccu(I4jktzaDfd3;}Cyp?3A%rRSo(jqpKX0dmxaS`;{Au z=}yW+SyaU~pOu+POc_^rZyMQzAt>6-0~=a)rwl&XYd9l~At%P@Qc&d3orM|>_zS8D z#>Y?0Q|WDlT|1RI**u(|!>8LxA>F@ad0);6^Lg}3U4-PjXq*oVA+@bzT>&h!n4ob^ zhQxmqFg<1+=LC(#*v!r>>aInPz}uYUjWt-|k=>M9qe>^5tqIIwRG6d9APE}7R<$OR zF)P0UhePvW*$Pu5zS%I6q$>o*r&bH9l2zvb%qWb$lxJudgo-Gesh=AQP`~f^cOW$A=BN zhrkZ^A+@!EsJ4OheLtn~qK&YWq;O;CTA&7A+RlDq!Gl@@*a>#Ffe-w+x57I*EXeqk zxuFzgu#^=V+Q+Wka4MAFb7*UCx|=C6u^Es;Gd#_|9@d)it22@$1PKIXZt~y4#EpLu zb=H9x$mxn@7KHKr_U8B+zXRh4O01BU33Fa9pOP_4%#@cCgSR@x%b1i6iP8Y$5F%E8 z-ZfSyp^NJze-OP@%wdFALm!b@9bduB;n1E`hKKdDuW-XVyux526 zzR`B&barOYS)kEr5#&0ooiC%OhVOru9ztB5%9NMZVGyKJ+z{Du_%;wheyIbe*wfGZ z?RhsxnZw>2nsQDLY#GY|%11$<*=V(yD+55h-#KFei-0YHbc9ffs`Bgl>^0re^mMf` zd;h7t6Zkw@qt;K!W8@A-1cE5+067{kZJDPUwyVEt>BOxzsm*sl7QxFggUNs05{P?g z$HHTCa4BP5;k_yg3^iokK1`Gz=uH+1K`XQ#*#au>30XhzU0)TEwtyFd=ho;~O9?~` z8s}PF5y?-9E>ab`h{A#V`xt8y9JQxYfbBQR_{5%-{yI!n_s6cvhVB%jYMiby?!~T3 zP8lF1G5Stdr4~Zru63zRSSx>EL9b(TRpC~E=@s3kp7dW_H=V#E*-aBiv}(V7Od%aF z5dAj#n0f`X>-=2z%z<7A^LA=fYD$v=%Y__FH#qqR~+mnCxW(me>&Nvjq zO-m|_gy5tS6p)C~_YlZ#ZR~hm7q1{Dw6La)dl71C$UsgU!7kJw!!Vj>r7X}+Z)^&| zw=k19Mu|98KZs~`4K_>c?J$r5GL$WUqp#ti_G~lp27KPXicmEw}&t1^X8daz55!(CI8)1LN{bYRb3p@g#4r34# zK{u5ogU9uG8`%cw8dK1qtVAjtIdGLmsNc#&eNUTwd4RAIyL!Z-iymwn52CHboZ}Wu zYZjSU1X9Dka$X9ROS}3;WRoo(LqMe1z}heLYt78}XkljUA(E=%kgIwEJRx5|`(1Lg zQ8;K>0=oCNIg|HuXS|;aVtaR9dSLg z@)lO4_M?A&{a3?TZ7)S19C2}^08ldkxB*0V-VMiWec@^0U7UuqMeN{jbuwUVkDRz| zIK({HX%{^KQ)2DwvkbE@2qjbTZMc{-b0k{=( zpu>D;YI*DiwLL~`k4y<5zI4tllmrtnh}Wd+$l!l>aJXi@^E~jLFea*~(taee#s>5X zHVLbl!i|yFO5eecRC6LIv(E1_U2XYadtXf{wQ+b{9qY5d;MrGffO#*Up2 zYL^6&ozR#XGaWd)M_R0AKQbX>j8mW?yyD1!3pjSN;ZGjZiPhHQTG3quwf1iW>%9)B zRYHHT`XfWLpxiQ7SHV#g!-S@-DlwPD>&6pEal4fe^$E}DRvOa}#l*Wqlp~WkP^ei+ z=Yh{t&K`zv5Sv-41R836844!FS_b=SsDT=86n4;AX|Kv}5n$H^8YS3o9~-k<6Az8~ zI&YBX$g<}^2ygo_#1SlQrjR@OUZIOrU%!89apIo=5hAU2`dpwrW^eJ7x&truo5-*4J~cSO&ad}P%d_Ro#9`? zkp(hv;KE#DI(_x@v`?A12``1mW+#`xTRSGg=INg5kt`oZJ`zQGf*|#!&~>C@0SSLD zr&Q^;^4mBC`K@Lny?akRLI|52c2 zS1Kf@ii8`<07h&&7g;156Rc^mben&{>f*XNbTT3n)o5Kr!j4OCP)-dZHmPUL6s|R= zg5=r`X(G;h>dRw^=JJYGYXsuWmm=gB*V5m?hw&`5vRi=;#- zK4A$6OsjXdpr`>PMA$r?FQJvMiprd(y5>qsQ~*K;c;}$tx7rx|>TDt6RMIhollg+v zEbL|*b7Jgl{IL^jIN%6K$SqAm#I!XWn4#`)6TxB&ypI=UGo_oAUJ3cAcTZ*k9mx;( zvlW!_*cTJRv0=jP=TG(-=um&Dk#(W~6*#OrOGHCCP<;mot#^(YO$8=)kn&%!9>NJF zKZNs=y0X=f{7ZvH-A`{{?1bwjNZfyj==m}&Fo|$P0Z^U$&S2JDvNt^VJsG~Beemj# z-vq-*MCbJVo-o%ArAua*GnvQO1q6=`8wqR+WWo7|~J#CoOR?UZuuhUWrc5 z`$jFq%#u&Fail$R;R8k%OQ!r8w6Kgpt)I%Y9K)4;cLL=r+&ItACzL|kk@xFmsbb}EjhR9stYL?FvvgSRa{K~A_ zkyMD_id0|uXlk48^7ZZKVxwo$D5T>mnX({95qpMC=Yyjt-pm2Ggfp3y{-mTDX)wuG z_d@pd?Y9Sp)SiE__3f~g+?MQKn$VM-$VI8@VG#MrE;701tz`Mh`$mb`sFNen9(%o7 zyOe~Sdy>^2xCwEKRj9-j>a^m@P1%W+!LZWF4jPcp$b;Q?KZ+MzOW96Uf~4P)6f!uT zSqHn#TW>KH>7zO%qDaNU;}>_5;vPGx)rZ4cbq%TIJY;|CF|rRTG#&gs7prOx&e_@_ zCVI1p-YFhYE-HdaKi{Bc%XbGG%cWo9&GuKjgX2UAE+3YNB2tuY4vdgM*b*dMh z#TaTKvXL%a2N&txw&gA!V4PB{xysZaaTh8PfI8gw%B&b}3@sO_sD_l&4C$NG5wjvM z@5t_&Mj(G6_Cs!*#r7#&=UVj-X}XnLv;wRAY67cN9sFyRssNkwLh&=yr~T`iZunNq zyHQOnb^{s~1%wB})dk{=Av%xZ%ZaUP%UA=$4K?-dx(~r_I~+ZsdtX1j%)GTZTL5`N zbvE4*dOj?LCk-g!J8N)k5jw}LPw_cP4@&qt@~7jMdN74Y7~AamnrnuV$3nigvh(8h_M#%BYr9 z9pZlneea~lKMHrnKF*{`?khBCkCuOf-cy#~A%7i(Em6DH_PTe9foixO2PmN?z$iPk zzUZc7Xc%Hj=-l+4q@CvO2CRl1W3cZ8z%Jbo1u9vpN7i0>(DxDK15|ZVATYO^>M!0W zk)Yg~*9|bt!%@|R2SyMpN1;kkab#oFt=WI0h*h?2j3SKFkcd?F1`v1$Ln*Jf?)XD{ z+V@9tRG=*CyEs_@r7dM5d&GC^I|Cn5{vN$BUqxx4MAn?BHt$+dho)^-cB)@dUv#f; zYG-CqRLL4`MG31wf!vA4Zczf1L0LvFMH^j%SvmCRTqK2dIK2ii2>&9YGf&p>BCfk#J%r@D}V4!pACN{WdG5B z?g|YV_dhCUFWynRD_c(bXD3n6*ek1x#Wp}>A5^kz5tt0_Auv@6%8lZ`=kHhW0C7@o zs@(kXf#a&Y@_?^{$QG=e6Q=v)tK;~P71Zu4w$;-Lupeu%x9ulrj zU^`um@F;7m;suFq&_vC$-Y-_i_qVoj{{z%aPMV*Da4&B+%Y0r=?if7|w>i?Nx9+cx zEpX0WHHTMpexzo$7bYR>-)CGrr7w%8uq8IbhjlT-w62N$Hzhs0*ED|unI2XcU!RX6 zxLH09-Gq~_lbp%RJ{z|^lf&4T5m&UH1A-n%=*gKw*vqXv;Pj>-0XRyJ&=fWPJlUIU zA?}Mw%d~p89jH%NxGbI8^N+W1WvBc_#Ry(OfHKHB4(mxNl)T*(u9}f>+ww1rAE)5n z&n5HAb#`n#Npm5z=6-wOZk)U5G8RG{>KmdDfMF|jc-{13?& z12gOY`agx6${vnNi)r6auUTG*)>n)hfq_KNe62slT{qz@V~2qOMkM_45NAxSwF5=0 z^$}(g{q>;M@|AxIg~Gzghg0hY=Mmn?l9l2K|B6~5VC*klZ1#U+fCAW>%X*!{pIKwp zvN(!S)2i^;zQ6o@-+J#bx%Lu;i^~X;t`2+MCzW*xCq+1&kjhr2^D@3xdH>G5l^vQ5 zW4y*$FEM)18={U}mdar#YpneS#DzB_M4P6m3SXGj9gne5uQi$b6|EIGAQz6 z@^AbAS1^e)kSpX7C)X6P&dL&OF)n`UyIU)jTcpv*@mm+S8dtFe(2wS4BqN*`klkEsiKT1-$_tds}lZcy|nv#Fv^zj@Yj{2^;uJXAa>eN~GLEJ4F zxFMtvj4sXn9I(kk6U!h6!L_hZ?A0{bpfQ#M9hXpb2VPBrP8v9j1c~{#?cHhWFu~6|4_eSFt)E&W$a*q54VQ!RO zk~M#)Gm~;g`cR~3YDiLy5fcZ>95G6eklth*M=LMXy(ApMb__wU&J?kJXy=x>R;Gni zd*tjc+Ys(DKuj-+6s~3I6jTj{TcAm;q6w#CKXiVI1W}Zfeo7uUV#?cOx1}>fT)sc2 zJfl#rbxyl}N|#kIKS8RF7fE5P>uY)J9oK(63s615o;T_oK}wUWV!B)^z?E;cSRE#r zXD5TiQ+?!RIv?@q@_<{PSV72i@H?bV@|u~+u*39-YD$el5bzw`i+uQW2FAj}HOgj2 zO&e8Y`D!uQe9}?*0?C8jAU{dJL>o&bzeKK~NR74-QrPMwacSt0jPCrFjoTWXU&VhB zQfoj_uF{5)LYU7jv_%Qex}G?0DQwg(=0+T>WmbMGN^lrxRDhMkVo-y@G;0f#*6u33 z`NDWp5U|P2jVup!qFebQuc7YC4> zWyIDIgwiLsi91VLFw2kUBK%=)ZUQ4tT?VyahGuO7h&GU_jYKj8j!y3V=}mv^ z@a5=qd$;vJoM<8l)#)Z&hawPwNEpBH~xz2A%lAq0UZAv2W0ljzTt&; zo&DC=4e=4#@%{dVM}B~Pfq4x5ZtM=}S+u3UEpAg_Q}t}NZG26#O=EvkF|9-tE=Vg8 zPs&cpWLXH-4%h}0w@OLrVtV~c`f8?fLOYr5nJjL9xoMIac7w6NGbyzuyTjl)JSzvR zwdG5nw(UDo`pVB|J?ZcbZiG9kUxbpa?!=py4w5Z z>U2xLh9ulSPmgtOFR?c)XylSWx>ZL0H};Q3W~PNjYOQw1yQ@LcKqW2qI@f2zMJkH! zjb^j?W)&7|Paz9ha`~DvUVERaQbWcPgY5COOlXUM?A2#Z5l?@qZr6KNuBHvUqa)X* zr#ER6jTZZR>rSWP8!62qhEPqvil*(rOsnl~vDU$Ra!oy)I^M%$wCh8wg)y0k%3S`o z6q$bBrk{!@cI_mCp?|B%G{NvF# zQ}3tX5!;XOdRl)QEULd(HRZ!YohPXekYnc9%8N0~bJf#j?n~p(CNpXiuSs}=OzEDn zvXg3qbB5}rxpOEVE48Ltl#z`XT@ZxSeDJ1+W(cml+s~70I1rQWRi{`aQj_b7AAR7R zR;o%H*ya$Tn|!VEE3FRT^)kLS20XK=bvsrYQ<=rpSoMG1DOsrrj>11Kh{qa(oy1qJ zJ(unqyqHB|QtNvj80R14htZW9yjOMN-fTuqGSn99YRnnsLmkHRHf&Zm8!>Fh!`r)u zN*&mHfWH+wZn;Ztc{6@r%kV^zV$2sZhf~P|!4t90ZxQ_J7YpA)GDs8~mGp4Bbx(62 z!s@db^J;&LDN893Fif3u6gNsA(g}g%mXaY;+id# z{-V$xvM&jXWPc&6n?;;x$l&6{Cem6F1enyz#{W4ld_)XJX+%FA>uZ+k1_GL&VTNuq z$5!pn2Msx1eGD>2zZo>BnZi_NZQiRHXKWuRM!tX129;Hii&XuMuc58-BEX^#v8|aw z)}t0|foR7ZObMYwsj2)@L9c+LM(h1?Ui zTd_ELM;SzG@ZtoeL-)eJ>ku@J^d+#H+1Eq)k2;!)@xaTL6oAAmZ(r>Ib85EIO~4L(*N#D=l7 z4Zdd;)G}s!V3xv_Z_9?=BgX>ORKG&sMul4f`U8}L1gY5kHo3|*R8NH@HN5x=ew0{cCx%MlX z$?{n5o#8`rw?xfVNH3#L2C13iV=tdw2mkTA4SJss(R0LU> zf+^@`gyW;24KFA1STSb2rqyJgoRMypYRQWZxFTP4uGlv9qST-lP9nCyt z_h(T0nWb7*lqr-FJyp0~ULQcvD@0NeiyC?#S5P?I*mz~VX5K`2eXX{-{vXAOs_^8} z@o`RMDAsD3f$vqVGI^r6D%BcW6}HnGOl~oKn!*nAbRv0l!9KF7lWZI88oPhiL2B9) z4?ET9Wp~wR#X-JVpaPhWnXytwuCa(q0xVGI zok}orXcJ$cHiLMWAkM?S7rXYqpjO5B(zvlc$8WM29u!V80(rPM>SIvVyn(t-vPOYV zw$~#hS>5Bbp2G2W1o0~0xo^~V9|;J#m5hPnt}bZi72>)>rXv^K)3JZK6-q|3$79F% z6iV=f@=(*$7}hO7{dDf5uOeS0yz zX^`H0uP~hS{2ceT`m2A$A(lVOB^en0*9#y6-CPD;J>8<_IG`VxfSf;o?tY;DMeBbd z#$;gnXJ9$9b=LiOaGS5;m>i)==JGSl}|d!}d}bNP8kg7|~|l z9&cUiwi^gY&bCQdUV9&jBdE>+u>MVyIFCg!K5zcpwXWU0VRR61^wc%+<`r~{!J{04 zD=ew2U&H!qouq#!A*9}mf0ADQi=hpHOQad5Wfugup0dW~2fO?2Kb3g)Pi6xPDT@j4 z$^3hg3crn|;eTUF@ZZx5Rq*K9{+ts0=iI{oC64D$DZ&3ym;Twj;D60}{GalI|1|;h zKh1gkkMn~6I->tLIhd6WkDBR^OF_@h`o|0XzogCnwAuzE16LtuT?rxq)LQk$r zJy+S;9VQG34)Kx<>|iST{A>l0U~w5ge*P3f1lvXM2NDcLizdbgAi)Cn4)L=S!(`TLBO*BDgf@1TA*>Tu$njdu8MD{pRJpsj#WEs8qA1bbVsR503BL z89MN(yX}2yNap4bw+8Q_1#1k2d8&%`-m`KQ1DNe$dr-=Q4Csso7RS?j%o?(hv&#F{ zKTH7@H@%a}-Y-l-BJ;_7EfYwl?aAZ57kQx;Yy5u-us*$iFG9k@J}gxn{ER2)gSbxR zvUGqhYZvu~nZV8sX>7>LEU|%4=rzg~ z(rSNVJe`X4-QU%;!V~tAnUj>ldn`W^3lr$L8aPcmoz88hOuM;Vo8As)md(#rBRSv$ z<_d5c?zJgOTX}B+GM82epJ(&7BdPDk)q!K$ z_Wte-+VuJxxAOkIK;+)y>+$Qp;1!G0U8Uo0%6;;f$rIK#Q-gf=O{c$my&=8i-mZ#q zoxcsb>Y}IrlqIoy^5|$}y-MPOIrt_Ox44<7!ti*-!c|0if4<-wVy;K61j_6MmhFGy z*f$6Bdru9MtjdYE_&X2C$7kTP1!1rWh2Z!qebE!9j%PX8-=QaH&9j{Sy(a~+`d(*> zk6ioi{LRNxcEgRq4vrL@h_5w7z!p~+fi(>Owh-Garh_YvXqOBWe3-HYkRZ-B_=*K6 z=tRRD9$)Dx-0HJ$OSyNql|SD&zevhtZs6=Ty!o8+is7IqZ3Y ziA=imSh3;s0KEc<)bH%mRmKLf)%{gBYj2^BrxNs4hvL}!w0Sp~t%D2nLd$=rjWjlU z>8puR*Q~zXZ^-o$yO-Otgof*SAA;L;-ttvNU~*P`AsU1ztm~i++XU4)g$|Dy7>Jd# zXcbs>ouBVy*;gp(J$T!(Q1QkD?b9#veF&O7keAQE-*cT?Po`G$hhMDuyJrFtD0nGU|c!iU#)h)wCTR6*yF8e@%QTu`y!x-9i!Ia@B>VK>> z$c5H?Q$=2`Z&czbbG0Fa(SN}M_tI{&hzMKvVC@N&O{QQ^@Ufm5&rpB5MEND}5qxPW zhEbO{3Dsj=!;b*}`gBPE+KA(ZUkV_5mb^n7B~ciqVu+bocASoCb?*B-6ViqzBy2oJ z9Cc*K;I--YZ$8z(CpFF4vf-Ka>}+d(tvU*rWS(3!<-P8EhK2E=lHHuEH???$G?ga^ zP0R7gs&yP9Fc9mTOMriqdhenp{vpAU3|hxl3$+i`hY)H6lq^ZbHRM)VWGiOaf;5#A zrdq5IKHO)uNkv7k+KMyTKZi5^ksL3m0DnC1ibKpfO41Ik(YEpp^z;(wWr&^@!99Fp zrcU74*KKtG!X82xF>1C#j=%x;c*YdnLk508GD&2E7m2gL=W%~xxG^L!$T|S8-{dOA zCD9)Cx~E1jF;PNZD}+I9)jtQgCDg?h^wgN{Eqyo2_ktpyt9(JJK5}e};v8RNi#*jZ zUx;L+?c2&`ibIYB5eUEHz|SGUvX>SzJv}%1fTiyeOGAu0a*qe`Icn_w^OQsCi!YhK zST;0Y6y6^G8xeo(oN8)Y^v#XWLq+#)O{GeBeq{3VeFk$ca%j+gw=;YYnFdEeH^@kpB^&)n2B0E|{Dsqt*q zfd;(<>rcI&aq>RqZ8%{(p|(PlaV|BXboW^!S^rcN(>s5z3QRJRJocb6?e+c>TSSI&F>p5QABa&N*!nVX}gfG4+ew~Fr&0CJU^a_-#6F?3%1iswD z0*xvJeb56%w~!dS94ySmvVEI$Yo+6A0(7OL*Q>Lmqey!FRHkNednLX~dEz1W?hh5% zo{I%L*&%;47bS(BZfertqGZB&#F9#jUh02elOQ2ctF=$4$aPzHS$?N)x?EzM zKQ2B$yrAaHx4UcBAS{p1FGW14J9%+HUXU<>n{o?8Jd7NJ z?@NF0Mo3n+w|F7feu5hs#=(;S#^CoC(!9orSIt?V9z1=0*kB)u6&ajV8IWJlAQb+A zW8-_#fS6;}!)WOKmnY%$~U?2!nAdPUJgyneG7;&ybP7(s@q2ZEBN z>xJM;+w&G>u|darQ@=;HE3tQ~o>6sb4vBxOV+IL#+(f7{^>ALvkw@#7ik)VUpaCn) zCl$ly`2`@4+>Ka-(?E#|fe~t~sJY^F%Se!Y+rsiIxLCJZOk#m^WK^+@YA*%EeuvO% z;lc_MrhUMypCw?{0}l3zYMa8uwa#YGZl5%eZ#or>V7QiH11k>^6O_Leix?4$7{Y&h zDed}FnpgB9fjUz@29UWea2Z}mMN~j}W?Dcw03$Ar;^BxV8vV~|YNzA#7_7Qgk49Yw zg?`}@ao&?cT+<|pCg8_k8=a#C1G96cn*erx4~D9Cp)i+Pp=jaZ*XaV;iO0c5A(M90 z_(J#+{3(Idc2=b8{g(TqFRDZYO_YD9&bU&S2*+W`PG!%tiH5VMu}%zpgdUA{boKPa z9wQY(F*#IiOP;2_KW3F_u}#Q{5tGta+XH zac|Cs$t{d_AZXzaaY9!=a^^4QYm&5hp4F$#-HPs$0)MIe_$d-fIaSsE8@_)m1(6LA zfKlUg+2Vl^P&O@q$*Q`?{ocIm?a@hzGmcVhp<+60BznU6STM|xD=Jqy1PvJ*H3!2l z&{3$IV@A>EkmfG;`AlUc;DI0xgMnVB23F3!m) z=MxFj<0AVYe2zECCWDo-2akW`3cBUJz)R+O&x04VNHdIscn3%G0=mSQ4)KQrE`V^B zOn$AZj1=8`y<|Ab$0;zZ=qytXdycUk1ARre=z&bH z9MPvMzYJE~p7sldiz9zI89tkyw>+#~#?$74vMi$($EtmvS)Ur#z2345KQmY^W5NWi zQwW}r)49X&4La$6GSH)@h0@ACwE?u0^C*Ztm@kj1Dl~2(Bf#(YmyXn`GSQU^sxTuu zG~J$IKi+cCt>u~oHP|Y1QD-H`U3tOxxpHI+k4uZ>;;NioNOFJA0yHA***Q2MC1H^% z()gSf9jcak-#`Y)WH&smv3^QW)M2KmA$CYJny&1i<}UYA)Al?rjcdR_+*cUvt;k#q zhYU5~_I#&%m7te`J6Zs>$gay}6z57BGR$I9tpdUQCdu!LmBqpPDWpJkzA(h-l&&RX zX#iTA+iv?aiameuG!@nFG!(|j@O;Uua+yasmmyG`yYOqyu#`Zl9;OKxSb&?!oD<8e z-7~tr80oa{BzRCB6bHBZ*H`t-%XzQm-A@}!I(t^_Cn3kXRTYuSNFQSI6{PvC!$>=| zN7Js^K#hXF_=TbNw+yF837N_rw0=(b*MT_ToO>JyU($kFh1Mlq zOJ7ykvIgyD?N+e0xz(`?G(>|n#ItsdNmK{DDFLj|{M@H)*SEAQ)aBa6zpN~2-_37G z+;t*ogz}SR8Jl9f1J(Z+B@pwg3*sl+z}jx5gw}t@Ha~F($%oud$AQD95s5*olfc=F zP$E@1%-MX~7`yrF@HVYhyDKc0k$jQ+QOkrYa5Km;S32@%63~oaJ%j`3Aewp6C?Sg^v&d%&B3p9-af*n7fU@q68Ph79z9_76Zy28z~oi+ENQeIv; zwjO``U-!>CmJ??2a&d^__>|n4ut=^J6}`btA8kTLVvh(*D+F#H-kg+durb(~>`bro z2&{jzy!~RA*_RcQwepu&uuSg=kBQh79>8S+4rgoCh@D6x-@^UnBt(oDAlt7<01iFP ztyNoY+4~`nPjMME+VTB1T0NTKo>aZMw$6VUq*Y4Mn)$f5^?|NXwm&%7BaPk?tL51p ztVFA>*!Yn)=s1vf@95r}t9OC@6a^>$CVuEZ*`i$kB>?i_erMR9+bfZ=;g@rrrJYgd zIQY4x`E)d!vf?sdJmG5Us{M3O1c^HLGvsOB4-zus`Gv$&>$kTN?wDswPT%HL?`D5B zDY$lrCJn@)VcknQfgEj)`U+N z7N>o>`-~b<&+Xs@9BtOPK2!F^Ihb){}t;}KT*z4n~s&msZ%OGsdL zs~wf8Qz;+~=1+gf@99eO zUrdR?vtLOUdd+Fq1nOZZs8%%keMYqqM!CAt>+_hB<4W??I+iNX;l6IFp=~Wutq=JD zo#50HTc~59nqKdvMDY3E<;JDUFRoAdi9YA2ndn7av8zm;^4iY zhC)SW#lr55hTC`SGgN=2mjq zeTb5&arNf9?IVZc)o3H%MOZwX~ygd zd6cRTK;Tn6pFw}ibyk=F6CAFRS-F~%zcPW4CCC~8n8?%@qc&!Y8^Y@fx&Vu}pki^6 zJK$4^Dp_l07!mZxTd7oujLT(v8_d(rX4lEdynt0ctKPC4nDo?)fVR|R?qW>nOCX3R zNZ1gQ6PxYY@7mo0vngD@jJgvx2{aKj6f|yS%xB1F+%ta}q=lr7CX1$WMirrspow@M z+zyQ~&06CqbK+MPR5ncJq%>C^s2owdQj?1yKoO#`rwxa&#+==h=vd_{Yta%{T;ITx zW&3!&8g#=nFR$=errGvtKSe>-*2-z(b(q_=^Xb7aJV22g>3I?E+bJI8bZCgNW_KLW zZq0W%&boiSjneeUG3^Fej#a@gY+;Q=nmx=f8tL4SH9)lRj}^<~=tz5Td4N>Hq`$VQ zv%NTSdOSYgC=D#ri|8Bc^VHiY-B9MeTFz9L+>K%1U7JB=)aLd6c#K31fm!9~4)c=a zlyh4^p?7i#fl-;t?wZT%pGrk@$9DargvV)Ijr#0sqT*76#H_v(t-f-f`>G5Tg6{^Ksnmp=2b#nJZGN1tEoyH+-YSt{ zO*opDxwhiCJxga^Z>dqcr?^1H`E2s`kY>4*K<0WkW;{N%Se7<0>#cxmJH*pVB_GPg zo@`+Ol~3n*xuth`;?+|zf&n0M0G z+vk~I1)P_bOnupPbWX_11&>N8t5#oW_cVWq5^{Ci*hSQ8zcAa5b1ngSaEGm(K3nRk zGhtlz72II>7Sslft}KI5~^kf`-zr(PCRJI4C8zwx%?ZEj8G;QL`kiY zL#A@>3}gQ6tTB}0mckw9k>e;BP2l>OgE5nblvOI!knPdNTE-kgMSnL(2{B|w@JWA8 zN@+1$px!`WC^4)kD(pt?bf(4GPP<`tkL{odzTwVKTPBoT(}orTtg?<@Dzk*6TUp0| z8wmpkl^zZ_4%Mkf>cJc{ein< zq8mEKwaegLz=oI04uHcs`~4x4&W`k$Hn>|qdT5(w3vLFnR<>K(N=y{R~P>1&1xIa8pyE7bZa{|W*wxjku(g;J+Z-zQSb>fsWmOl4QX;-DS8V{A{ zx+~PypFN?h12eZWaMCzZvj!w)_gIzi`aL#lgpxh8j`_mQOwvUv>DGTNmnd9PIrv>J zFQ{I=m8fjYE!)WMljR3V424u?$&Lc?pOON>qtQC&8m>iZ%K&av4usRMeMeQkO=MDnK^B@8AJ*@$%I)AZ0E#$FW$i>hHa}c+Oz0(UyO?js=j%Lp_5?;`hqS zn^D!JU_w=gs0uRAtNMKf*#@uQ@xD2tMQeJ$e7U96?fiVz#9ErA)pN`GMA+1Oyeys7 z{xms?xNWUZs>c2PbJ7g9F&$>?o0vP)xeS5+p!<}^MzzAGyt%S4B$oy2b1on~M0}ZG z(~!4XMPoBT<@$eFMn_P$56vTkIW)p|s;kN_|Mich@H0v#{^X>^L&3u##3IcJ*oHy?i^Pnj`WWC z?SZ<)1!q0+1GXFK!(eH?pd%8yguk02yoML)6E=LYs$)1|yAEgc;%kS#PU9x;Jzc-I z%f&jzTfD0^opCD^O~LpAdX|xhfrtXF7p(xT0d2yp)CQBVsfnqHC2s7AsADs%CF(2< zLvI!wGNONuAsGWoTKL3=k2eKaJs*od_=N6H%2GIhJU9>3bA#fw` z?m$fN-c)|B+jTsRrw^WyBX-_Ow018|vC%-MuJh)vyU|vIm}8u2?Rxt;n(ON09#`IH z=(^`@=Sv30F2H=MOV&iowd-3fX(?<@Pg&tg=3;-((WA|^cV@}Bpq#?VR#1A7deLxX zXE@9%rEqBB1NyU6^OhU#(zb2G$@iuADiHThFP|q5*eeHYzphr$Xo!}gFWHX(@=Pl zpvA66g0OydK9=tMxz7IhoDWQa)DS9Z?Y6Jc3587L4ai55+6GVX4DZ_+RSgD25hghN z4L{3|v`Yah(E4YRFzs00zyX= z)}8toDoB<-dSDhv6mCrl(FC!B=}#nG<@R6N!pXA;WPCPCL@Kcm?b<6~(-G4W=ZBYb74}K?g0Grp%2$8; zHr%r+MY7Xf!zTp~vj!J&AoLaVh|b%0_#wiCfbwwyByU|VzD(lD4v7pwNTU`XtGh>2Qtcbtn?UgP{BF=QsqaWGTC7GkswTZN+8+s4$2F^cVIg50Lor zWTBG!66w17Pnd390CDhf@&{DO)uWdNDQ@4;1QKTr&VFhQR*SKDy-D_y?F+^Wkb=+a#CA`H zjrlAX6lTvZ>i9U_p4XJGmY{w=%?m9=L8K0iQ{yT!N$+=nbs=vHDYy5%`%1$DV-DC_#iW}7DjXls~K2p$&5896CS1M5(C-B9j=c|7TRmfQ_!U!dF z)H!7|83Q;ut!a=VOYwP0Y<<))@x9@brJetVP7@8U1a-*8AE`{|K2p5@(NEVed(ZaC z*3TqR=N9KywQ7wQ3M9U2O0OPJsvs+*1i`=-38PR-ynrF2KeN>#)RLInI6sV`A4y;A z>fj;aQL(-R{cKhqOGJMnwh65jr6sjF$xX#wDc1ZRmf{Xf>ndau@$i1~j*i0wTMJd| zZk1-phD{Uc4@ED4K^-#}rHnJj8;C(S#7LoTh%HHxDkJuTIsh;_f{fk0eRnA z-g&n6`^(NmBH7Y-&tgfYW^J91Trpdt9X8k0rJj-kq2b9A-Zc1QVfoJ|UnBrig!`CR z80-7GA-f={sO?^NcQw)cnFkl;1;?&i&~L8v8p>Ur*&DK^vlV)+QQkY|&EL6--_uvtxtcA*ZmAR)dw9zRTub2x(TMd?pmP%2iM*aV&&n+go8M`E$l%`LHnu& zHtT%2B1FGx_mgXDi;H2^1os!NXO^qyj*@O{TwNVgfzUB1)==fG?8NK{WN&P9`^pWv z>tVf0mQm~tr&zqkE-t9w0IEtjcXpF6l-9!J+@Vgdp+kS?vKO)Xs{oi+kF%PbDOu)UpYrCD0AbmVDukRyJoSrL5aXdUUG2FJo{Yp8O7 z%L7!w$jRZ&;K9OwbOq2#^pDJ1sLY&J3?}dN`*~2GKNc$IW38V{zVp(dC$t>V>8Pr$ zc!71uCBV$m1t2{A8yvK*p+fyhv@u zRE$Tpx&xdBt}58-yUT(WbzvMS@~@wZ6;=)io)qkZxODfIwi|j|Evz6Tp`5P58x#x{ z&hI{XDAn`GqAFk_;0z*`9~fFEExZ|iycTT`Oj>JoY#J+7Ossd7#>RSl7y+YR3sk@4 zNyse-1bStEW>MFJ9*~4@(jLlH`?^j|C?7S)lQiBb7tlmQJ%Th#uu_`q)(Xfye#ktR z(s>n{*m`1)G4(MqgE_ak2(AX|D{Q!z(3CG>VC<5f8n0cSo~GYnK7AimMd-+^kL3qN zuQOekQ>Luf-}=aqMQ+CO?oEshSDNuAhzh-{UKf~uP^oBK_jqowtuQG2lpSVpef!-z zGODuSqdbidL-$h~_&hV~(%x1+ZDPSY>#O;xNfMRy$&+j1v#)$9LJ$n9b`Qw=86hTP zQDxL6>aRu{0Vfak@z?{R(x9-$E7(Jfrhq1R-eR#6C+XYiekUD)#4#;LEQ|W3N@V}; zCLM5pzYDy97dvnom4<=f9+9()_^#%Zg1O_lrL?->LkoR`A%f*9#C*Djw;1iZgJ4MC zo~>2d)CyGkprIC80r8n#%!eV`67q0ih?^j-7y(r*9zUt+Gu@T4?I&{8xuD@80v3Cg zI4_K_5B&(Ik$;Z8vJ;XGCc~s7!!GvG`YynKACWb8h2GrzwGTOTEh$$FJ867RcaNH! zQ|Z(6rA1sWaamnlfw&OBl3*2kncMZ~al|x2ggt_!-^X1N53Q-5uK+f{`K>K0GIVP* zAjl0PJ=fiP;3N)EcUS`{n0HF;WI9jVTqZ%33S(6^KEUTN3P=<)!EbJoP-e(b%_H)E zp6^I-<;*HZBIa7GYXuN(o4~_n3^O{+kReI1Y6x-~+66cwiM5Sx@tiPx>zDxFqIYOf zCFYtF=#V{N0Ta#i0b1eQ)n!gn{XkSb*aMLnR=|$4%68nNQAfK}yxTczNE4kJYS6%J zn$U>8iXmhj$beYWU^@X6$ca4MX6KE6Q=MJK{Z5@3hmq(xKn6Q~x-zaZIb7$4QD~^i z8C1B%T;U`=o^m@mZMx0>{Utj8;P2xz-4m`7cFC&K8xl{qN3c5(plK09dF#}!AH4>u zB0&lp|3#MOX9$Nb862-i6aXdC9Z6IG7ZNgp#(<*Y6$r7GAqW1bDS~Vth*ji&Zay=a z#m>xLB$AW-4oLmnP5tam{p|St`9p*fybe+hCUz@}mmCdY~_6dW-gRWXB!&L-yw*^o(I`*;mg^X z=xGH+YdBy{bRcC4Vi6^@EjxUFx8V(Nz;wc{5B7~3u%`Y^8TOY6zNRN6kKPSE%(pAP zCfj$eqaJ>y58Tuzu_VXv4z1n|d;l!E=kFaa5EIIsiBqsssDOF+8*PS6eOu2oDUK_D z7r?yeUV4ygI<~PLuz(hHEMwbHKI(uLG^|EGDY;6;{@k}8M<7rz7r3;4*shQ;R0z*K z=#-bZ*H*!w+P9Oct^vL0$h9%O+8|0$UlY1AFrCJJn;rT^o9wj~C0c99kervbfN@*1 zKi1JZ$8<3Rs+>y=Z^lp}9>f_e>wb^3@ATt$L%4?Z9-&Ij8H3QK>&`>s&lO|2>J?CQ z!2w_@t?2q5qgU3@Nt~m9E2idQktV79Zf`;{)8bUM@MDIqZNH#4V-BdFyOm$nkZl&F z@}g0uz=kc-zHL)g9n?JxpPCZ~q2%aq5It&m78tcJP~7u96eb&r-oql}mqs4!W5E0f z>zyNJF(VHkW>KBcg%98d**Y>op-IfvtF^`F$^ryVlP<$UL7_{3o!%jboM7xt07)I! zl?FUu{e1!An*Y<7e#C$-p?6=7E~}SG1J`U#G<8gOj6XFpN)sdj?SL+IL>D07B0F~R zM@+gdS3n4Qqye3CBp=554+E$hS>4bFS%yesItUe@08T@jl#9lamWDrKQKp7BvG`3$!ARxbeZ5x<=1$`O2#`3uXKUDa3L*c-2h`}=b z%4ym8;@dHr$>{ZZUMqUXvb9N9YF~yi>za^HZ&`)tnwUx8g9X{_U@yFao~f#!aB}#y ztXQz16r@P#80L$_NVIK`7>~YT5LhT!kZ2ezNa%P~n8HziF(U?xoscK+u39Dl(v5>=(k9z0{@H&1{YW4jpd;*fdjjyxzrj}^p%@XB5IBA+`O)PrGMTJWtnGe>1pEti z$vyBJm;aTq_Addwlc(jVy9<*fAzxA4op#AVv*PA*y#!= zNNA9lS)APLpQk+e{CP|Yl>8Ub{)LR1k&XGE!7h!P_tU|3zEe6{!uHlxiGl+Oh|#h? z@XcrQ!$vnj1go=zJKSum%!$%0#6mSRG~k3ed9(50O0m{`Y{FcIkrZ}7WrOGtD{kRo0|}y=q`kMHJ?8h=)e`jZ`<`}(xAXq( zvX|j~K9A&@MqhQE8R|>U|GfRSD|_;0VK#bs>RL@_a`N4ehKIFh?|f@Wr&=z4tUH~5 ziS+lx-bAZ=B8kZ4`I!$HHrTrJUU&EViI!=2up`#I;pOq+-t|}0iR#TNTicx7e(0cc zrN6RqI=|^i)uk;X2a<#5rZN>(m5uwJkKc+8)U}=3^|(Dc)>$7JN~N3jyuPdZYPKf! z-cQ7jcJyx3`&-)!kH&VNnE3K({QKO0vnMmT)QjQ#x{b31Jt{Ic4u56W2j+iOJk8}h zLhfWZ546S@CSVI|u0i27*I+X?P}Y`ONrnkHUTzKcGH_taSZ=)*877d{N^9;v?>}&Y zD!v4VN%?R8&0rG3U$^b}7E*E70h3Al3(RBM_1&P>Q6_))s`%M|MW4xg z>H7RqIie|EhG>d~6HTUgqTCz3P?Bi+3ltsWaj>6wzDh2W3~ngP%=hPqTdzvY=UuCn zf}R3jy#JtOAozTQ)ud&#l3}zAm}Sy3l4I^e7`&bLXN^b}$quJ#oG|=gS!IXH^tF@K{|+nk-$U=eg#YjzXFR)V1QqdLOH~*P!vT`6h%=KMNt$* YQ4~c{6h%=KMNv!aHzt~NGynhv0Mr*0jQ{`u delta 167065 zcmV(%K;pl}pbGz+3V$Dq2mk;800003>{{D$+eni2v%aDtCVEGX5ou93yS?Gvpzf9^ znJuz?YdLxfK#?f3fTF7aN)SW(mwB9rdD>6tAKQ~v00>Iz-mu598xygNuqon_Rh4;8 z=E(xST#0nEwcg$dS~kn0qMeEBC*S$Aj?d1{7JYY~Z?8AM>0$GHb93uSXM1B~b7OsH zYh(M#dS`2WWBmzR|K1aM?33G!8^)d#+{VF=FMjvB|LOd{=;sIa>U(@veqgL`#MH<% zV^-%T6m7Prlc`j~vMZUz?5~*!Go8wiC0wS2v%fNt;S&>s1u`iAY96isw&F`J(ois+ zvZ0d)0Udv5BawyiuehQ;E&iQFnx#5pF(%Oxk<6IPoSB1}Fa=BVBoKzl)J-+eQNc39 zLo6%POw43NQ}{LEsT>KLwLe>((%s+3tt*$24)a8$S*W2_1FAtfF+wpaXdG|NHpW?6LQKYR zxikNL7zf@DWrnezl`F>|5vGW&6&h{7AGLokQ*jmQ1c@V>Ursp|;>Nw5Um%U@D0iHS zMasHaW@G?@ykuSE)HE|%Dde*O)oZ3ll$Ta|y)!m6hzc>Yhg1|6@FitNj=*NEgKHjU z?5C>Q@3Lx?5*Lq8rcZJHgBd+`W7m1@cK!hc6GPiL)$+0H_Tkjq`DLafUR+-OvW0)d zcA09EaAoC(1BbTSB$-oyA77`SwJup-N+ z^A+ZJCR$eXpV1?%7 zZd{7+%B8>h-o=*$v!-)!b0Cn3=T(0Vpd$&N2xqcpR5!3xkg{)1i`6i^G=!j)Sa7q^YuHro&dzT&sW?(W8gY}HK#ez9we&f*bEX+3@V zz`8@|@|4K>7OY<*lPv@we>j3^vU@SYcH1Qb;gUh~8Vg5`I9N8y729CLexC*03eZJ7 zust;p1U}O=eY4Xh1=y-iF`m_7VXGSTCjqSu79v{k(@5uO#HO8R(~XC-r?uX94@P+2 zX5a-!#FMGE(p!!~l|Z%SYixV{+4ee1?iHshb@{2Y*lTl;Je`zY<*1xcWy&mdP z6hZfSfQjqOh4f)4N7;j!!t#%8HaHxznS}(^t;61%zQ?I1>!c$oeyvPTkWCv7c^F4E zGmm(pvkv_B-d?-}1Vqb#y*5MHj38E(Vyj0N{Ws4HQZ4f3#o_RwKM-n_8)0wnnFYJv z#XNH}bO>f4&&P4be^$#o@Egd~<6JxvHEF}#+9u+hQE+X)KK0-(1zcp@{oM>r2UP1V zPO;UCgP(qWeQ~h=`&gHOR5C013JY89y2zEM{>VyTAV!4wRVGtnaBYPs<4>{j)E`+v zAVZZu^uZ2wL)EFr71uXqO=_4*EyiB4TC-Vgket?-7hf$Re+>Kx<<@50yq7bEnxm;@ ziZo(;{$}X9raYw?;V>C+7^qsd)}o7Q2LUouET@)QB5tnb?AKC`wb4np$2Vkawb_D6 zT6lzeA@L|@%|yz5+osTcrKI>8&6hwH66R?eRjJ_~rw8ZNT5p$6&S^9%j#Qi$#q$5U zxBgf7e7V2>e}nLT?=k*(|7Y|0=JR_0XKVfW^C#<_&F4GM|9}7If8$e1KTxhosN4C* z4md_!h1#igdrI3biU+s9pf&$f=aDO*oZZPws+d_u7m;SXp3(--2sP?bj;=35oAN}E z%yzUU+WJG6B-gH?s2>=P&_s}7e7|I{dMT6@zjvX(f66ayY&1mFWi{5^B)6A8W0sdA zx#F@2tM=-~`o`wd^4J>d3DFutqEAV@+@0ujyv9C?L=`->6Isv& zf%)yfQfdV-2R7j8IOg)xoO=soZTyAbuAFEa=bY7sPq1X$G=C<;SPHcm=0k3Z<*^P$ zL^)n#f2?X)KWL6(SqqXw;Rm%W{vf+ib#{chj|JDE8K7f zWjvBPlWXj>h}7ku=a)djZXzT?eDhftm7gT6Wsi$)4F5iM~rjVmxL2SYwgCjTm~_p zHQng}PS7m<%J#StLBqH2bDp*aSYPGje?9fv->F=q(`Y(eq_EO-`*A+j3*R4e{TXt_ zhhVQTQq_Ur87P$30ol`NVqmKb3_OBp7=1rHAYj6+i&7KR10SZBki`fP1&xhKG6ixB z!RDH>Rw$?e4O!hr=n><%77nK!2QV_H?ODevN1N@hq0RRCMtgk&HFAkIdo-N|e``8V z>A)HrqM43I*6{giL3*7#9jQvl&EGfwS+r7 zBj@yUL##5n7?bn+=>G_rr?KLE1U@19rXYJ~(#8`>UXSVKBSUvz6p_1&&lKI&*H`rO zluPR(&1c+5y80bvM@NQIDUp%6e}RMiUY;ayD`weOe2V@!^Y*hFNi*jrrB3@;m;k3* zZ?(m*qn!RfM>^GxjD3U#D<6s*4)+W><B1Tc`HH=h44O2CE6x(BrGBNxG0BymW0oOe-`qAJKy1uL8#3mFxZp-Q|RNdw|GA+Dss{f-s6Te#5)85R~r{(EEHQ8mLCbLdxP~XaVp;g`OxZwe&SR=km zgx{;2dAF!T%@;aV{q=WF;|9P-;9mUXQh*4BX%dOrkDZ~+1EjnLQEjsJZ+v}TXsP0t@eHFzd z^A;sz<7s^qFn}DOsPBuJaum`iUoOAJ#`*g~6UcnJjVV5Re{oJXVEGwU*&`7fP}~tN z&M9xj@YvAKsBI#4Pqa&YsTV32RJJP|4~d&V(~WQuF&wI5LK8xo5E6Xf&#C+)NkBK# zV0}FmL~|Ymxvx82s1%y_Ys(1_N4aTrDYtx}gZuf!iTIhF@B*tkibVn#Wp24;wo&2= z{B$V`5<~Jjf45`d<){$exh2)7F`d`bDRc#;$DT{1Jqe$D)KohMQ>3Rzr^qQ(!o-wY zi>d5umyfz=u&zkuy@AqBk&p;wINb+*+_%7Y=7ifsUlrkvD=(Cf$=V(gOhgV)`i62A zT8C;z?_2-}T+lZ0zT~*^5O-0ngh;V;^qp`kJQI*uf30H-M4N_~(}A0Xpy^d9muk(S zM602w)RZ_O)J(ZbZdbYpP&$DyV+{}?ga}%Vr4t^4N)@8cI43PvA?t{Pjk5;8x)In_ zqpVNmp@Q0R0jS-*11Cz*At@Rncm=+J2(UopWC~vm#6GGanwb)hr^wV8-oX-u12ew| z&ayXKe_*s5#F(9!c(9h%D$IV6cMi7ano?ec-M7f8(Iii}S%NVMtwgG#g!4I^M35%Q z--I>JLNb@=_N`%!aqrCr(yJO-BfeOm9^!vY%Nl$94;52K-b!fvx3KeJW-$e{ImFM@6CJ&b~;~qQvVypQ1x(5D8UL zQUdjaIxd7pd4c;KwcvNigpy%Yz&K^=Cf88d;nxM-N^2$U5OUu8lPYkD;Y5r^Wp)@SaJHEu0ROnm6U{2sN0N96=5ggqU9-s15%)gO z6k}iU!tW$(d`*$tSl>cRwAtw_7dagSe`2{Oq9gFC21oekyxD7%DwzavsqmK6S1p{S zavA2F@|~{R0k_|RyrPh4kRNFn8nAxEv~r3B#meE46a=8Vn8hPfvZ36YTHR|<7pGE% zmQt}X)n%pclWsL_WE9#^LbqEkz?G0%l{J@hqT{q)Q!^O#&Ji?@904fqy(g>He~ey< zKpZi0M7#O{K~0-aC()z34*k9aa|_t{ng#5*27P;DV`%}%oRKNQiCscu zebjZ_sU`G9KtV=X&1lLIyBs*lf65+}1cciMbcoAPAW-_E?9)8143XJHaEn%ofVU6k zHwM|$5anq#*1#9t#*h<~*>YT1pHY1-ho{^{Y0%3;ew8*R;z72UPSaaw)m$7nP*XDG zKvm(x;ik?2^n}Slh_+EI^-EtbcQ+w-U)+6^h~C_F&wu0}@HW384-Qa%2Q> zP_BxWxJW-Im^A^RUUIFFe;SMLnk>#u?Slg)P}!C|F$A$(7f3Gg4d{|v_N5r>d(P=e z%kR1=N?5~9Csc%?n|z~19_vb)VClD^P8z-=$p+&2a>Yagl$bFos$2}CJ zV%+E-FnD7MgidMee`z#5b3yZX>E2HGvvrxn(5o8NyBYM#80GYWUX)bGRITbv<#T$H z;k^Ez&h1-Y~oa%A$6*LV~zGLB|TtpPm9QXSh-v1anAiQ?h10b zd&}5AOMbf6Cw_;9hwN>09Y~EEhs&yXUf2Pv+I*e;_E6IH#E+t{-v$ap;8Q-Lcn1Tz|<^H}@l+e#!j`-b)HnwGLUX1!H{>Bu@RBKIGT#vT|*Z zmsEHB+~3R8=tZ2}{dYYAk6W{sH*v133(!)g<3d(WfJN^K!eCJXM*F=g?-u&2%-SF6 z-mAQH(mAcwe|w4+=iPObYi~vP(uOO^d9=2I#;ROM$(w7dJ;;}8oOkF;_H#OO3mrQ^ zPIX4lF@Vb76Bn-R!LCI~bnnsDa)jQe%fwg;`>p8Q{BilgT2C&alUC?MHAo6HJs9Wp zY)m`eu2xXQ(E*YCNG)$l^+6oIH5-yXTpST!@&tz zw4*ug1o#H{okn9{ZUP5bJ<|TJ^zHjO`Wli97@->kuEP2ha};aq9(kF_Jxe-;uf4Sr zDFX)twAxwb8rgzo{#qgFgs!19-1_H&8P$g)4K?jHv1_0q%sJVs9!9<8R;thFU_+`A z6HZ%je_r?a2dgvZ#c@ns_M4cfmt~MO+U+Iw9GwqT>s;pn?MrJ~KjakcgkdGrPQh`&&a7}dN2kzk-1yu%p?K|alaRk?``GQX(OYdL+hf`zhrXYxPP!thFy7ZTgg}R!)lkz zf2{(@7x#*=F;Lypj@hH;4LRDj1jDm$h&NDoEsvWB<~=_Qex`3Xb+cKiw$<(~1b=yN zLEP5m9fkpE4EpG;CJrZL6U5mwcU-Bcx81&_qL`kd+J0T=gB8J1H+SNQ@S1o#lJeWP zCimr>p-QdKk=ql))@qM-Il|dk3Cl;De-+dLi^3v4JN!ZWc8S=SLu&WdAT%QPFlTwy zUPj$41}T&y@9I4qfCS|A?w;ugA#3b$Ar!}J^0s>njYC@0*zrq29?ESG!G7P~7Xuv8 z2Ve*$SDu3!M2pmEz`S_{cY!A`Q-ma9~~kA=v|9{o7?<0K{yle-i`y z%gN<^2S>+x^6n1y`KWZqXE zHoje@Q%4V!og6!;u=)^cO{LH}{wTe#~(j`#?lOH-Q9;jCBvHuS_`PTZoL2l3@``S8o!A|Mz#IbJ&?gd*HvshB76nZ=GWTHt2OTz8F-5E!`S zfQ>Lh;KshEHt%g$643*NvlA`khqa*rB_#bI!_g@hpDqJ(b@V@(Lg9)e<1MWz{}h%W zqMk35Msbk4)v8suB4#tue-ai`q~3%V0JBvq7u?*0g_*R#SLX>4Fw7V5ogeu%TUeg} zO%~S?5~<=2*2D=GE-IWGc$`Gbdw85gOClk5L?Z6r5nZJrkZnQ-a^PcdFi+sH<*P^3 zd+-9dt0Tgjov>yKg*oizDP!sTI$(Chs^L}4v32ry0WuF3By9a|e?)Ibo&J8|-rD)% zfLOk&tNb0|tPfU)SQ;4Et^X94l}*Ltz9(PQ*;VoxknjT7F9k#;DB0>0yd@pQ8gd3H zJ21bdJRWINbUIKQRz(|T%q=1yV_+qQ6lUTUMj2Zj<_y9tp(aPj?&E;L9HFYfQkipL z9(2UqDx_3FyUee-f0YAjd33u?H zysK6r&;X>M7MBhzA^=bPDl{pk!@I%YDAzD@jMBe)pm?I;l_lZ4x)pTdaygeWq_-b%kE0m4M8(Tn^B9 z#+uY=M5S~=AGy3nkW3mD_ozB@wS=#Q&I~LW`ijB{}; zp8;WGmumlk)sl$|Asx>K!e4}S^a(2oe{6_pbo>BQoI>&$6nwKDxLL+4!o`IocE{Yl z39BA$iBQT((5D*a=`@0Q;?}ARI*{em!$I(35W2W#?{$JME*0h^k5vZPy>?KaPO&OO z7VzExe;47yXT7S8>IFV8v=N6Y-2c$NY{_K&s73I5d=?ftLY;YzD$I9v@C)1rw6mm? z=^~eL55Rtbhqlxkb*qSw)!U7wJesI_?=D|0bZ`k#w`JOT6PC{r8>)@hgn?CsL@tg- zx~LL9hU3~dUxs;OpoR8wK=45%c@12xAmj%Oe>*56mtg9*ixmg>Q0hTI>N@&y@chtG z@;#YfJYg6*a7KN=Ren-o7$H_4#r#X85^(sk<$ zfCyOQco=6N{)E2<3$+Q$1Wovm>V=w38ByWrF%W62msP?N)ZV1sb?4k$-Af5rN`$VNgBbl&-dO*K^2}V-abV`*u04f8oG;?S~ z7^(Vtvr22!CHJir2UxfGg4}%hc6)Ad2uh@KK3qN~%_^m;QyI5a0i-K7ST2V#&i9@@ zOUL4Ety#YlxVPhtL1aNqk+9wQH4~;Ke_Ua3nD_Dy90qs;WTHz5jhSB>Apng`TEhba z#KEf#4}qsNN*99*D}z^<%^tvQY;`9;vSwj^IIr$0w4_j}#+^G0FcWltFl!*W1Q0_S zN)XBk<5>Xe+ZdHOZ8PPdpmq(SxZ|aoaLl2e?mc@I;&{YUEm{Bm+sE90+3-F!f3{0W zvUHKDBGGH)EmEMH+)a$>EML6}c4T%m-$&;fHsunbUG~A9m|vCWYq~O!zPoshC6Kr< z0!!vlK0yqtNH^OzkCA%3r?}HT7`Wm)10+F3vdl@?TMr?c>T8D@=0d3=Rm_06Vif>G zr2&QzVC1{v0uBW}gqGufEf+7ue^EA(_WLrJWir*yz7A(h?0sb_h7vP*m8#jd22r?@ z^j*O79#wDR%+kcLeGCbe`7S~1z%7>;bGrW0n!3lnd^LIFlDbkStveypOY*BQjyW>H z0~nwPCo6QG2)9HHv)Mdm3fk0Iwem<%^`JjuJ~O^D4 zgcy=V{G5tRt*YS?%^s?)#LN(%R2|4H@_g=2c*A|we;*k8T3Vw&2)3o(gOjH|DUPQ69g|rloOt-7;>z0~@s!ul3_>rth`);O%qMdxze**_gzbdmgv|!AE3oi7WgIKde{&`DUSA%^g}ZPc z7yzy%?!J9s<@wh1I}ZtJg~$iP&*3q~tqQ+kc?fLl{vjiYyf1ox>aW@w+`;?S4#=0} zCitp5Tr0{$05Njd)VM9&Vx^{ccfRb(4#VwaA)&Inf z$_Q1QYW1U__VvdjfA_5!hp#3Je7*`(zi-WDr7=$(S@@4k3FAKMBp@zZDv*>ij(YH) z8vlp%kJJbNl^)IsVh4LC(ZfXXK#yMB!U^9Omz8f%%YZq0+^;0FoRUh7!{y|&i#MZiLhyn1 zb)tLbi(2G&f08%>6L<|R+rA}=OY`!f;J}+M3s?Yi#7edcNpF|L7@|Z7W)})0;yC1s z!g%e6LKz5JIu&v|#yzpUg@eiNr5MD25`Ipk(=FM+fpG&T##Isgn#5AbJ*;xOv^7x~ zz}|FmbCz7;C8Ef|PtvBKOM!O?mm|=yLTEUpVMn1Bf7BO+v|O2|8*yw;Y)878ZwjHR zaFXc)q)TBOXcWLM9R)$G%_pGb>IZHMTuU+Op55n;m&ARLu1>i9WKHe*p9rONERkvr zlpf-Cp{%o8ECO%}e1taP=b=o8-z%k3b;6pJcNo)wpYG20m(*AMUQelNk3f>wJD}!N z0461ze_5Njpv-`N+j}v++aSf$lFriLffyoK9q;Eo*24dgNu@H9{~?)3 zr9Ki*B!$!TA^*b%@$;3Dyw~qH-a$?w!QGXf?fdsr5yo$!Sn?v#psZ^ zO@By#fd4|5fCJm$706%NG2rwj`3B@38}5O8!I&4rMNoe$FG0N`ZF>0v`wILg+oS3) z=`Tq4T)GVSxAYq559v7YU%vZvx^jl42Z83nn{l=_!mgn9uT2@3J}9!9wn>>;8@ ze?fXctaF!Jl)sZ#!6+u>Sn%J{x1cz;axc`M>|s!xo}3JD0q}31mmyzaN5lA2583B# zP?(NA4*kax8kx`Q;8(Qc!681l6aT8*4-QRB4}||DCxnp|+6&m<)XbJsCcPjBhFymMUE)#-(d57?;4)Ype~QTM4&Of{@NKp9}@M@nLS(f4+ef zOy+%>GkpVN&Kjdce6y!PAj=2@G6z8*-98XVV9M48G6~E^q9rgJ21MljnGNFxdNvvr zc=)D5l^o3Gl7mHDa?pt-2U8!SK%od*^}$*Z((7iq2)twA#t@a2$kM4{n<`82Z(ngqeQinbgeHS0R2PK2PaJXF5sM~F{ObyGqYyFXN?D+RcA-v zC-JUtpr6$jBna%CQWfe4#fnoLpcCT&3(Yt{;Blu4F$Cp#TTq_s2*LX#&-D%Tot!dio=zw|R^<{1!!nGSA zXO=iTh_nI1hHM2JVi(&$lLsXpAl|3d(l^kn zRYySLTdl=TXO|`$@Y_48)OazI4Fm%h^^-*ZUNpHee(+nr< z*GrNU`dU5D3ENW!)d>yM;5socO>`&h-~v$({%R^2e4#OupCv^BI+NukF##Ks7bYSB z80vI=BXsCFOl{4de~9sBVPq@=0qAu3F#$YTMpdEW$$9%>^a14-VE? zlf5PzEUi5>pm1LA&s-ZfFhT=nb+>Plp+I@|;;vwU@?x_=)u0?0jOXcWtBF4>lxKWP24u+a63V+`WHF*QQBkLfEzFi48!KD%> zI4g(~oM+$!_trSUZUZN{PLrc2C4V{up(Q@?y|2p51?v=qmNzh5&H(4MZ&85YxU)+V z9K&BiGJ?@X1jpdcL}xR>@lqebaihfutcBsd%$&k|F&VB|JW%;_9mzExng(u^pXgc{ zBtEX|b?~<&-?d%GcTJ{Tkn-BJrijfNR=5}3w%XQw?LLMm9t2rDL}cPTrY!l zfo)Mmyuc%u+zadvf$wIJFR(Ej;D2A>S<(~?BvXAX49X3F!WT`$Kwni%3`!GgG6qtb z1|Neukdl&tFAb~=%*nuqM~iKqBUuo{bEwD}r1Piv8GJH|DH`ZClcRyn3?EGcJ&=#7 zLD~<#RJ5P3K{`_%f`Bq?~m64>endcrlj*{_&pE?$Hn*+l@7T|yvPXxrKCLw$Bi)aMcfl5G@ARcF@z(NB4Y}4T#&iW0@Y)=h zlBz!NwY1FqK&_P~I>SI=v*3yD=~G3F#HQ!0A%vR+dF#SFr1-gMR zvFWS3^}8OjM76s?iGQjNJ?Iiu=y(`720^WjmAtm9DWZyB@1SMJbYjc;S67SzA%u+p+7_kD6Q^$kp+r(pAbsRz#FBoeNh zaP12WT)TGnXKP$LNJ-(5YE=6yK((X8xocoQNZh?4Z`83S*nd9hc8%QUPipW!pGo67 z8p1CnAcddSVI?Gh{(V~sc>^OX=%eoRXV~}ZoZIaq(S#+yp3>kKEnyPPeSt(1!f{dg zP9WqGob@0(s60GO5Gvgq%@HcCRUb_#TYiculuW-i@L;tK+6>!b0ei8GkI`F}k5$h=9cDL!xMSZPXcd<9W) z3Nfu{y^RIbtlsMYm6GJ!@7Fr08<<#tVBag>8W z!tJ%>hktz?w%B?!n{4s;^SN>Evc>TXqtO;yCuXlL62rRPmT6*DBv?Q--QwGtUAN@O zb>pqcgiu3C#vQo0{}v6<--3&E2z24HeOG_q6I6oQ%UB<|M5 z`n=U#s(Tmj60?C9zodG2;on9ZFIsr$u3gN63yTO zNq;k_>^?;^XyOj*P*m&0c(`szKqMl3KaqfV13hnwT+cUENhH~`llDC3i}F9PqTy3J zkC^yacm?B>jiQ-m0M%gSV>NEl@+k)rK%oWzdfcp63nmpBIH| zlTy+k>mz|^^7Ylp4-L!rbN125G;JS?1AjkrpH@xNE)a8hg(sji0oe zM#dfD@$-p8rt+gTG}!!X&E{v=Ns`cyCJi|Kn9+}BLzDXPcT8SCl9WMeKbj4b-Os0| znchzcF3s>~%np(K)h!6~{Q0g&M1NA<)<^gax<6hwM@V{R{NvfLi(stP3DXRI?|)mi zP~O1$u)upAemn@|i;V=cCw$cYYD@_J|MESiuKyOAER&@A?}T_cmHkLOk7+ePH|umxKxTAVh~@2$=f% zeuKzx#HGQ0=o7Xa7;75%D=dS7T{|CvFRr zVKr!Cl0mNNj>DsheDlhGUufi?%f&ip23(u6M;vwxQk316Y? z2sgs+hv@<*!RpIbqwXqk(0oIHX~9zaXgrpkWrqgJsic^bRaVnMJ#*Yb)#-soS(TC1 zh0P;LZL|{A?Z?YYn?kac084yoV?03Ptr5F0euw81;=7ueCg;H&hq7MVLg0ZND z{D9j(-Y+I58uc|wix3wmEPo5vYelS)e9m~^tBD5`8W`4D=;mm(Mz4d4n1s*-H;s=B zj+|d{Z%|2c!ROCP#Kj~4$^hm)Co~T(2o1*}{aY#)#;q}IhYqtUR;5&QiAEAGoy!jl zAT73f)OnqY7JIbca3DhSW_os{@rsGDT=505I^gDs#mj@YF6Pa3qko<&eih5-Ma2V0 z`^JUU7n-$rR#-UtVu1)c>DX=JVz3{Gc(<@XhNbDXttCuM#Z5i+fC6#ORTMD$0qTxg?tXb0V695Clcc3 z!lf2|O%dV{w0bTiQGco>3!g^(n#&85p@t!N-~knj0R=9kl&fM2YE+CYeiOU6fXH-P ztYBeWExv{At?~zo=b#V5K@zT;Ehfo^Qp#MdK;GMdV$9(5t`dX5vjXD6wTmT~|3Qla zYOr7(_;CmJ;7GnU5+x?uViDrhETKH?jckjlxBJHJrk&!kWPeK{+hm7u>Sr&on)ZX& zvgt&`#y8uN%bOy0%7z6LvrLN`woByUiIpxztEFMLC{Hm9ls>Oi>GLc&EOO^Y0S1e! zTM|nHCmg1-D2`G<;yH98Jz+p9fzWO*nYYxcxL#R^93fU64>MhmGJ%N@!z(*@eH3~y zg)lPMu5w@<_J92D#f#wLASEo^Wt9<+rYl-;2lHUIird48lxPr=k-R)QX1Lkx5T7xa zlZcfQGlFtR%(9r%oC_NwOlu)2=f(frqQ|tDaE+V|;2}VJh(~nG(YUybGgvGMso5Vx z`eA&0+}lC`*1);PK1r3*N^!U8LKlpQ9aRy2j|bbjzkg865lYsQdt5wbtGTCFtQgA| zr%$H^>IYOu8RU)-!LSYD@wO%sAi|s!FN1iFf*SDLYV1Q}u;Gug|N1H=(5b!RkO?vB z{Ws0Pwb_4Z@gCcM;$tns{>vuPnGfy158|hXH^AyywEj?I4C_D9Q*ip+!fNq$J8y{z z%R-9=HGiXL*it0DNymHDT5cR<1{gNt41$k~wvsrPWw15-ElozwMV8UIxtiV6FO*KX z0na+9#W}M(yP{BPItrH*mv%^4SzfJATuOd9(ETccP@C>VvAH9ZHsXdR20eB`hW5TA*$L@Z$S8Kz>(Gj2<-;uC&8)9 z$`Cgdu*F01v#XLZLs^VOEYUX0M*Zdatie(xKbRMDB;;%=yVMHJ%MXeRuIUk0;_89? zU^@;WI_Q)q!esnF#B*F)(P%Oh<%;IPKL9dhFucbtTF4me2QVarUdWLH!H+uP3iCTx z0)JaoJZ;RLJ4(h-l+;Fl!O4%HwKsxA%be+7gCGp;TUsrlipBkfZW5*v80kWdR4YOr@MUO0sGFy}3=9)EDE4KrOG^o=(0|}di2H)>l%8C~L1C1aJ<^!MLa^&B1kmPI zXepRSw0u>kuiYD{!otByqftOApDY7A_+cP-pXdn8j5N`u$>Q0j8BmkvDIICn)@+E< ziaF)5GMlBD5Y=Xa5Uhi4V@tu-mIByKmKuaG#?diBv@G0S*k=WIpbBqLlLD~mD}T({477nCXjrKt%AYl4P1CLKtU0VR~F449?tT;?N9$Z7fMbK1( z_@fX!=xPms!@*|(faQbZV!=yQih1SPJEc#7VKp4}#5RYn*~4OjaG#hDE5HBKALBQ1Ki&kJi z4uPdR;!T8OfUSd@vkeV1pZACs6fJ5zn{YBoUcyDI7Q5d{WOnc_=31f=8-JPDaGuQ# z4DOc91kaaPF? z7U+k%0G8G>Dq6)#M%-bfTz|D86egrksaom>4>+jVU?3A=@>DU%mg9>V1{TM?thEHT z>~#)oS*Uhs=pff!HgF7IpC7^(h(ZC!kxsJVER04)D8eMeO&q82hngDMM;ueKIHnY| zjZCY^h!9ZelMX-u4KoiLY+$>oxO&D!qKTtV4dO^Sj3e=wsdIxCtAAFb_)(-jc4VvA zQPe9#WKBkpP%VjvLW4$`p1vW$@i&`^nYmM4L6Bfw&U=ezEBtL1d8=oav#BXu)kCug6movi}NPtSBQX` zD=YF$>r(7vcm}bFC@N%!me3>7ctIe+jEaqgro&R9&(%{y>0Y5w8jHYYrYW(EHz7>i zZYw7a0{(1tLHrftMNzU>bj%YD6KoqY5hz5+pAk0u7#Id@ReuumUWu6ZB4FuOFj`xb z)y*_J0g+IEY;6UFLcZ7!l-}@hwwh!hRJ%xM>x#2JgdGONorOJ!%Tok$aPT1qYIx|7 z^t&Homb40shr|ofgfMOkqwJnq1BFtl0!KRUiA`J>kJ$^|JkE%kG$T+8vEOmt>9pWd z6a2qr4gLS`M}OIWAo$*g+%No0ZT}_WEk^t=-6CF3q*@ZG5ADAX;z!wkNL)5sTSKN5 zh)U72<^&8T3<-^tSi}wG2Zb!HMhoDgg8PTRc>XM~8Gex-B?!IAqJI%PQ0i?5*$gJg zV#rgBt>BS1a#leYswAX0I*=eBh4cfx&~6Wv3OOm4aepKcZHq)y+KLSkQ|MDxfihg) zm{Z2uA(XzTALO%uHCFTN@w{6=K(DFjRob!-gQUV3BNLUA#k?3cq@KxNtqQeJrJTGO z+*_x}sSxF!_G)V$uK~tU93kunY6a94?RF?1wsZZ3k*IidFgW@FX9*SqJ&KIHVqywn zv6BW|VSmm-d2uzz&cvHDadp+6$WA>GbJQdGRS+12Xhy#)Q_-Zaz|1A=e5Nhi#i3Og zd2)4RLP!j>L_E8x2jU%Glby|o*E6ZCjDj}JXtEQj1bnzfenog9V!#i@BXE_1EYaoQhTey5!Mv}rCbS9I_oB5VdyDhQVoou#DV}XK8Lsm{$zvGIkMe|J{1Ue`Je=!6 z`;mN67>Mv`;cGODe|lM>_JB`-?MPOuxmp|bVm;=Th)vjwD15hgygF2ZRm{13G9!A* zRdU?Oi?_s6X`L6hF)yCYrrIzs-jYtm8|B3#CK`aYin!c>Q)v_|WUvGWNOKfu1VM&r zprEsXPaUScf-Xt{lZiVXAqSj-Btg}OTpat>wz%O4H9BM=ydIEOxHf(xo{mQ0m$sA0 zJ0gETkzXY-S1O6q28c_1oG6?;YP~BZG^4ISx*Sl;vji{&dJMM3SbjKO$-45V!|B&I{Uub zIE2t~-_X(xVj%+H5`-b)D?)0d07`#mF4S6kL*lbuHRd}aux&`IcEeA|`;uH=KekaUL}g1h{2Tq6+}y{hJwLsDvD1xFv9MXZ}o<10|S2m zjN_~W+)MbP@(Iz3DxmI>BG4wQ0p?pAx=|E7Bfw`hC`6)xJCT7VPe3wV)5K|br!(erkzeRp_FhH0QTac$DHxNd6WkM zGf*{9kis@E4McI~{CO5wDPcbkyI@tpGt1kSk+XjUc-+-;X^T|+2{AQW7_@)GY!wSh z%_afb-n`1Vwquw&ib`X>4}yuTF(teU8rtNr^92Y&GZ<^vWv^kuqVz?vZay(F6Y^b% zi4zukMGOOBa|}Qd`x`{D*hG1Xgs{w@Pr`(ekp%f%d6{-?3-Y1GU}M2`MmgGS3J_jX zJaFg?733L5bR0+ru@sfkIIMr0uoBcs*T1#DHWA^>Z{;YadVS-h!Juyd&ly;+OW&T3ih}oVp3t-Owa~EI~QCS z*hwn%OpgW|b!glJzI8J|x8!IB-?ROwa zpkx>Z^BnARUdcmY6J{aYhAJX7fa^N~{N$L8s8&}ynU-vy<4hPa# zPCQx?z)90C6ozgtFU)^4T0+AnKu`_%pO8mD$dwSu>#XNBUtEEATTaU>q za&`*I^3sEye1!gpt&abRlU4Kn2lBs(cuM@P^1sPA=6^GZ595Cy#E*{usrC0o{`9Lp zNwf?>GFTES&USxn0?{uJ=fhoHs#U=@K=rSj!92;ux>PAyM)(%mbC#Ds(DkH~n8(W? zVhl4+jr?@K$8VzXtpFO|o&=2_Vdu_in6?#0j^)eR+MKmz)rr6+oqENrYlGAkq2eaO z7KjCqr8&0@$hjguB12vv4Tq4Iid!K0Sv8_gSbqcJJzsydU}^GY4Ej(iOo+yOg}YEw zA1rjJF(xYJX$0Ir(Aa>J!sKMB0_m%mH4ki`h|KpgI*(;*A3L2jM7JR}e88y{IN&48 z72qxWw!|w2y1-39GF^-nb(`D~(;-4D9+Bu8WSSW?6BRZn6nNwMhCwk&MmpWVDk2!+xqD(xKT;5=RH0fmBV8RLmV5a}sa9y@! zPQ%I)d|RATowLlz>JnR5mrM#Gh}jF_p;2Oqt7uak?7-1HnXrta<8vZ56`F#jVlL7E zf?(xJD&%Q;TXb+Hg(N7bo|Ksm;-xX)(5~Yv7{fF@TX7E#_-S8W6~xrX{U+HL_7wY~ zBM9OnpV_p3>kNmDeLBc+=WH&~Q6sZ(_!0x*ue}MG+s9wTQ$J`ckr59x^{|6yTf;eU zDJ)|N8lFZ6T`3!&Nk~pUB5e1(SRFpxLOx8ksmMnO)=1g76G~-*PVl=Z-7Esam`16X z6){!d;6eCGh&?^s-JOiE6>dtc0*c28LZ1O6(oS%Hkv`$H+eJ?CR~czcAb=uA9DPt( z3%tt+o0&@W5Q=sH`*c)wbTkg)+73a9VpO<1(NVEzj3TnIz{qY`U{nw00x(L%u0yIL zMkOX}42OWCFooc~z;hO>+7^bi1CVQqgAcYa^D>$S3Y+1-5+$tU za1p_OZBbB5nhsh)@(sp2yBa?Hs_A57-iK)|ppV+;R>xeoXbAy~TZ-!l>|sG z@07&XVxzsl|_(B?JI@BB>IHa21Gu zJ6l5ZN8&BX2qa2*0nxMwB_bK|a%(niycMQm&?1CU=}Jj`B^7n$l@eD-w6sOC=~nX! z?M7lH){?2)b`eWPqmr?dO7j{g#WeP`I=07#N}j7=N<;3rCFG%&@35c1Vx-x2rAJb( zQ8xS30lEtTo+2H^WtL*lFgZWF9 zpUUu}dBLyCaGC}SVq$TMup3xuD!^-@5+t!N_Lv7X#bOFE8OLk{ z+^Nf8$vcR*)yK!h3afL^Mw*2T0!*BmC|1!p!cnMtliNiu2umdnlIvM2|`>7G2Mo?JuV2MEorg0;?M_i>OxG&m!dj=GzkPlCafhiDza$&_GsA+Xmt3&dlM*>mI)(cCoJ+mf_LrN%h3i^*~$ zp(_irI7LZM6|=k5T0`ZiOn?6rYa`|H3AK0*2VwcB(=?+QJwZT|DKpz_Z;#V25t+wE zBcNd-J0@KDRqc~yT>bbQd9wic2Dhu&FKY$Js33q3S3{*9&}kxF2i1uQ;g*U=v8;vC zu+Bi0qay9Q2ZgbdLzp-2K_cF3~S)& zDuQn=Nvetj76sG>@!n{u(QxO&gLo=($%b;-sSf!iN>bOjgM`+BKVs7{J2{|s z6p~L~6xM`Pi!PBDe?r40A;Lvq)CQ$hV$*{4Y`_K~A_D>1U_^duOS;20k`gtD{M02A z!gZ%zWuV0=?#lxvT&W-C`r&JGs=TAIV=z+lljcY;2`iV%wSvQFjYX3mNgsdK7-vOV z8@y&MBk0`HW+f#`SjcRX+_gLdLh}wdqoqn6T&qI~C~*|Ues*1j2bTgHBM2al0|3kh zAtUs^gJ%=V{nXi%d%_2f1;0kTFycV$L98GeVjvXwocPaMiq2cN*b#N^nR(i{HWPXe z<|;x4O57I4A~YleJ%7cHP;q}((o;o^;#dRrDoj8WONibc<^Bdy7FUu%fObvbxHK}j z2o-CTAESq_fi{}Rt~BTipd%JT7-~dvhq=<=iCM8CuE#69YIG?fS_z&cXJ5sxCPjf0 ziS-#UOOW1-##^HCq|{DTv1GEc5$swqo(`Tbu@h2p);AC;(Q}l_B;kKcsn*qziF7<} zg8?lLj%QRYV2C|J*-i-~Ld6jX!eg*M5U_6?XAiS>_Sqcyg7&7NaL0pA+~aol+?jK_ z=5h!{X&fOU)C6RG>IlpeK-j=ZaD`0E8HQKI<^GESgP907)KJ7XYefp-=0N=2#qnQ2t+od z!G-**;uiDVK}a6>&s&<$8%W~jU%f(=7 zfjB+uJgl|x`YV6=G6u^)#X}-No>qNS{kt|pGfAo)SY5ztfiB_E?$YA9LQN*(U{J`U z6EWDBaqjH3(YjVEptU6$pE6Fz@^;^hmZdbtdVqsK$YVl&Qu zx{uI@aJm@e^iO;i91x%hcXfds)C@{NN!ZSyTg`}1x{)dtX^?tX*1VXCz9?N;p#qbc zp5A`31yz1@#R|h>5t5?M<$`^!8sGx@-ji?XD3%~+5U{8! zT4Y3#`kZQ2=2T(BbOf{}R(hgFs+D1YIT@ItWz`D+nn&zTI&yxTtWO7riY2hrkPrad z@`}woA|w;QsGzSdULKqXlR0ZLpo7wK1bRhnruu(<4mk(H^~E;!Z9*Z@YQ$uwz;clxV;Cu{{7zq`(GuAqllD-KpG!%D22f7OxpzY zHBL_OL{Oy^Yh4P&8TjP5{Ch$C0?g`3c3aJeIS6H}*=tN{%<|0=$<*+$R0|V<~kgF|4hg&|LFVqWk@~*O= z2guii^C`xSZnv7XJQz|MfCYI}$^`oIdUk&>ug9@b|A(AlD7k&I>>hJX`=ZH{W0u zI0q-8>zs$<3^|1PiilTZ))Cvl^@1!Yfyt&06{iacqE8iSIb7ra>14-;KXpGU|CfIU z$DIkY{>wSQsq=r(3P|PuT3Rswm&ts{|M9{6sQljoWBq;PzwkPczv-9)Ss-xQLAI$z z5sZ%$7R9K`^?gX%PZA~vrGW*ojK*CA!Q?Bfj)e~)L?c4|IT2-R^i8{N>;N>mCVb19 zl(mDH-Kfh1h9Jn2leP~g1hD%Aq~m`;GRw9T1bgypR0X1`mJrb=ymVi83deBAV)n8j zw`iH*A!NYf08p@Jg?QpIEXqTVg(~PY@h;QZS=y3~n6yLC+!GI$1k#39zgT|;Nyt~=7IKu-_5QT5mk=Dg(wQrMtDM`qJxu! zjY?uZ=0+r*B}+|=6ebGrxFuvBgJ=Y&-SS11&(o9QS_4!nqA+!k<1Q81$OSU)D#qZ5 zBAm)52J-cM*On%YjAAqAhBVg^88;3%j6}Vile23%L=p-8QtY+$OQC=GLS+y>i(!1M zmZQ|Gk zt!-cI25nzLkZ8cNDENOe&@)MUnE>uG@+Zj!#VL`FT>B&?Td*fo5-=oBi*EN4ZbZ2$ z34j9Bo0ie#g$rvZD=YKr>dN4WRB4!1$5=i^nV6!K1_FIvT>$VhBzxiB3+61CNj-8n zXjUuIVVS*nuV{4s;=P0?G=g!uQV9?V8m65{w5^#xi{Y?}iX4Ak9X9w8@JvB1;7vpa z&ZLa8SQARzxQXuIl;PyGR3M%tzSsJ10>*9LLg*v+6_{8EvB|{Kf(jcD?_x_OGX6R! zRH1cP>I$(`Q>?kD5{l^k5IH9N35vyHHgy+YMqNWyE}#xTcS>j3Ec8)g8AX5zayOf5 zflfgxEHy){LyCWluyt_7FaZa?SIkyuE>vF@ttTo!~tl&#Dz+a*N0afX85{1}%RS$%}^4hYBtOp?Mrq>(*Ki zov=C3e9bo`4VSWG$e=D)Xt4mlspW@sah!tOYu1_)4{dUgqp8tuSv`HOFo;$|^iLR4 z6IKuWM&Yf#fz`7b_Yy-Rm*gyWB7o^=@YJl(m`fn2lYQ3=iTxLer`sZJ$+)GrVVLVL zX(R1c8x&uo9h~R%o6X1qy3f$oYwKjyt4Rx@XR4Xw>7xH}-+#^Fm#*EZ(pNV^0XhS1a0CFCPtq5jc0nZtGNRyL= zQzd`ynW+zdYAez>BZWtoBE*)F9KwDqh4Bc-f*W)fGg=-@**XN-&CESH(_bt=^_Q|8 z$hZtW^@C+jXh=s!520qKPiPhgZQ?1;=meI>kmTX|JnRkUG8Zn_a@W=EaxJ$q3_5%@ ze)YZH098m@-+GT)Ul)e6?w5%ui_|`BY#AorZ8?f%=8;Il`k?d6X9y8kWSmRZRjEefY~>{7 z>Ri$j@r4plXIj22W<55oTrNb(r<_0(D*%sl9Xf@~RU1q2lvbfuG=KUd;sFG)0pKl8 z2>8qUR#D3*0}Ew^0H~Enw6ryGRl2<_Kb&96Jaj(5RzebE2~nMH0N9FHzRGiEhw{n= zgz@P@`bjJ3wU^v(Atn!289&I3U8zi_iNRxuM3%r~nKm3tCv=ZrSVAN+nNReMR7;b^ z-)LdDS-PzizJU%3Kz~A5v^8;|zPdwF(tx#Qmp4sT8C+lDiX@p_tV3o&u+ZI_x`m=q zysKb0R_YK7o0|G)|$Pxs&B4ehGMsSGo#hTb%CKI)chax3y&h@t# z^|30fW1G=>lx-iiFB-vhgXRh_tDsC2q|~>=g^ucX#-mVm3*EIUPLz+7Dp&yz5WAJ) zv9U~I>Gz^>7Y18eH0ok%wsvT=*#X7s-|LJvoUpqapntYyal7U3e(Gni5ae z0J@RssN%N72}B-~0Z=ZJf`u&!_QiOp&^qf3`fc2=E#_v;M4siy?ls~P4Wcw?olQWsSxuwgP|;HR1x&;GJAYeZ zj#lM~qB6f(h9Fzsq#;I-f6nc5fOcXspW7l9-T;FeO$$f3sYpOPD^T|f2e-cF!L8QH z6BdWN_*&e50rYnSfo?+NxUwCoa)%{wrQzg2=*3o42z=eT=$Eb$4+dz|Q+o}ZNGaw5 z+Kg-pG)xc-Ba9-IWRc19e=Yf))ZM6aZn?gA%YC7`svV=2wYvDN>2R+r{p8h zPK1Q0%OMWU_lNf~s7)f6h^HW&gi&D#oI*s67bdDcArC#bndWA(jY6@q+i>_pywC^m zqwqh*QvbIp0M^p~u_c>oQTQL>hXDR3m5zUi|M?((6#l2vU@^G8elP9G3)j)xI#B&mCtdP|4z zB?p&LwQe-^Zk&@u6#xaKDt}W-E~YPA%9EkX=a3w0t(;q?W|G`al{zv=7%AxSkgqLI z#P)UC2~zXPUgI2iSL;wkY!0LVptT+ls#WsEVFWYffdY?ULpBuUiGKh#e_*MoQmT?_ zBA7(=tJoqYWb1OJONac0Gr<}wS)qjVv%c!0dN$Jkp<<(sDXNUZvVUXaRFyP&B(V#Y z71hPVV)Aw4yAnMwQD8OQ_EZ6)DVI)xRJKqwUy;l_S=b259-IFh{DdE^TyxdEiVbBTM2G zRDU3X1afEqYUAX_%C^dQo6$Oh!sl!))@M1GP7$-c>OyTvaUqrccR5;5bzo)>tI!mT zoFvfj5re&baDo)2)0pcFL*MVx7<%?FAXUZCfEOB~bvj2!4u3V$UjU2>-}Ea2xkHRp z9)S=Q&j{R+x-j8LBa}(wsASa1t#tZbY<)sX0*ZJ5nFrDay3dG3;N2C?l@OnYH6PBP ztCH)?!x{+FN7j@_}m<7Z`@C|1NCT%X2_VGa+#CDvS34q z3mznNCM^|-T7NE3^11}>3nv(u;AV1}5ccgILI;$wAftfr6=xmF1!V>g=?~fIXL3~% z&lFW4)?3H}shFEl>m-K_eUs#w#$!OSJc@eBI}zuGEEyGAFDm{L<7=2sFnX>T3D^i9 zej8)4XhIBOAsa;U5~yEsD`>J;?2_SY)lw_!wbA}JrhhNUWb$}GM9x51wM4ANjiz8t z(y>`#v!-A5T$$Ih=Qt4za3;R+^YyF zA%ZlCViaOGXpHP(W+RZ8G$ECdVG;c^&o!eqglqv+ zzULwWejFpC6{lcu4{!ul5~@k;xG5b>K_<4m?uZmM`@zaKGzgO?DZ_nERiBwY-NI&) zCR|i4L8=fFD;U(5i5MhXnv~mRvJUEItW#eVcB_$qn17@cqrq zxZi{=w!=v8>gtTd#Qv)nPT0{NaWH$6cDZ+?XS<$)J5UX+>aE&6!X~p<^{(EzI?+=B zcO}djNqxUr+(47)DTo5ETH*~Ow42x!RSr$!;EYjIT2uW)8D??ZlM7%#f0~8BZVS-V zjB^NE!xnK&t?3k8BArs7uvBu}=k`PFC|@kHn@S7;&?N@*WO#;T7i+WowRqc7=vry9 z!UZ~r)L(^50;F$6-_G~gjKCrs3Q`c?V*?5hjI4;0BBQO$AZQ+9@KNN2fR`Wf_`Uu1 z4BOnAy{qM4;51E4g!;pF1Jr8#lg(f!e>*>*A~r&BSUy1mn~lVum_1C_<|}&6xoT21 zhqjus3+0td zW!y4qfwjC`Ij!O@WLxJwIaF@hhDC`|Q>xNQ-2IrefiP(Z(>e?oRG%3U9xP4|!51)m zr=eYzIZ-H=aJ`~hG!UyAFiJt=-E^uxqNuQ(b-hqH-2}G3oEM(nd{HJVFf;|pj5-w8 z5axuq&j;__#kzzLDOSXWorR7+f9zQ57l+n8mbwF+1V0HdnWmoA@kk<)?3oZJ1!81) zK`y|iq&XSwTbk)Qs$OE?LFlYyV|=t z9&2mKM0$#~i5~BON>v@{C3Cwbdi(ZI?&N!+x_=S|j-_tmTE$wE^kkN~e}(>`Xmkk{ zzEy|DoIEz>0|yS}E}-smr4bTnWKKC0@)?AFN~3BV!U!e}$@I=~~uPFDwe13nzCab$~YOMEU9-1N;dzt9YESJPN|A zc*zcn(-4#r%vaIYlr*6MPE*=UtW>J1fsm}2aA|j8P_Uh518jbbKUX==AX~*6A$yf< zz++zzf!D!)wh$ZXy>Vj2lOSA&>-Jrt+odLfOc?J7n29Ol{z$pie=Sk1;suJuSP&uI zX`i~#m@Y?~h{)fE@)M2Vy@njvJ7~f}p%RXY*gln2Bom5QVu;&NGCP?C!h8|gSs62M z#O4MI!1gkknPw8S-7@=tD^4Eqgw4k6;zc69P_OO&CknP@>kh_RG?gvAW-5aEH zjRXRk#w9p3%)DzHeN3tv`R%)VHL|0g{gW{Ta7O-kZ`-D*@zTUpB z_)8Odsko3WV_!=CV@ZD;{x9P)1AZs^n&5F_6(v0V-t5=TMHa&pPUzk;Vpw&CrQHIo z*pk0LUz~{W?@+s<{s87hqOl)!CkVA#-=bQE3-ItLE<=s8BW{2YK?nWBFaf@TfvWIZ z3!{^ok|Xq@?)?6H`bZrW|^kz2FsjRN^t7$jm(*{FJFJh=GM zu*Ye6mH-wK_ruB%Vj#fI=a`l-MWz+0AVM8X(5;vg%cQcF4mbV%89ktCUaAB6$& z08COK%e>C8?aKuRsEzqDic>csF9rwQUUMj$TD<8T~Kgqb~I8&!HRIcYIR#kBtJ zoT~Z-M3C8|(qf{b$!a|40Lx-sS%3mkZjz>(R6^T;UN&&TDT{|QZux406=6I&F)C9i zgj(awq_s|&$*HoDngoQ4kx3_i`;v}k#?cIJ0RRcJbl{Qb#G`SfJ8?yA87^1bwG;Vm zCxFM29ZGQ(fCzzfgd9T^2+f7Jg082+(C2jzmv&TdRiH+%!GT5&?wn=T2uJwLjjWM? zur-AEz+K5TW=0dEh^?cimXXCIej|1eCGYFpr8AXgbV!dGZ1+TXM6r5*(r3gxq4os< z=!g~$*?L@#*$crq4muP3eZ@{nNWCIhp_b7{1$8AQhnL*vXgED274!A;mgh;FIWs$flv!>Y-HU_u~V2d!EANplPUi3LVq zILr=b-t|rbV*UVs>;Xdk#dSj}vT(^-m1JLDm43DIQY=b>UG(yq3Yd%vOglh6 zm^<-NUbqTuaG=>i^^Ks?CddmXK76Yqdq`@26fQp)C_rEsg72_pggEA6BCAhaegF+7 zJgMah7385wF2$Hu4-qqqiOX6TgC;O8z*vLHsWKkLq&ItgYt+R zLqZseia`&|T*N|FG@7fh%ugfJ3pJz+oY5uDfGehQZgEuXm@*XNOa>B|nnh`+wUa2x zB)%$N+`D^bR|f{1qmV4(_01PYOT)Y}gaWh{E$WI6=b?c#+eJk(qfs?c@ggU=z*VLv zVt_Q!Ra7MFo<#|NH;~m3MbLy!64s?$#FeGKb~}_0v+*mQqu(XO77SGEJS-lwkY>Vo zc1(iBW33r`pyEglH#SvE3e=GwVfkvP!T~@~0|*0ohltY)I-VZ~_}YHVUh0yu(5lod1@A5F%8G%a-}q5m+O0PW0pE2Eq! zDe4wtCSfC&$P#e$`LN~h2gQ*b*VaIA65bjJZbngIO8SRtMJ)R(r;^e%1*5-$5kSQ_ zQ8U_WMf=91#svv{D!x<6ufiR`G9K+24NIvnC^SJTx|lN}i|$xRArB6KU@V0pS6CE3 z7Usipo&Sx0KN|nR1{45^K2ZEeGM-AO6#gUAl0y7PJo6#`V4O<4ca;c5qXX zMqv(^0yEL#m%V;LF35^(z>4_bA<_!BtS-}M;9$6lugeM;QgivDnf@V33rt4BX7XJE zS;k%JQQQyh3y>V0E=rhb&m{Km!f<;#m5t}5ER_*|NeSE}GCKKyB?zZ7hXmO{xf*pj z4#0%_g~mZmEt7IJ;N^gNhW_$M#4Z&^kn=Ff1vR8!z~w@f&!7jSofQU>GgvHP8H7PE z&q*ol$Oe7DxiJs1p|K6gteAtYA`>KEnG2{0s_z)lIGyq|f?wE+uP(wAo7BV7{TvKS z6RY8Wi$d?L#&D9?EdJ>=wnGrl0`5UHZazEn(W1*fFrd~+75z8bXK4LF2!-a2=8Tq0 zcKpz!0TjcPcjgFn6gFNSJIxql;YA%O4OPXlcSeA8B?UBX;<<+>$bB++1PpHl+ogjE zG>+?3^#Vv?Mo#;kXQF(SJYWim+|)DZ9v}=FoJ^DIF2>hlFp4n3psl4U0#XCwaIkQn zTkDfCYa<-1;;ErK0Mm~c2mG0#KeLH6e>@&bX5#QtYod*laBCcY*RY>S85=GDTX#XW zW(9VXLX8X^B7so+%@Vzi3G-rfq*N?bOGZy|r>`{1HQR13%DfO>8L%q+obXj4(5l#% z&f(TV5LKljX~@Y^alSZxlF5r4aH97W=glqFFaYS3`CA=qe)S`$M8g5{p1BVKOE<3g zvzXWdh0FluZNxBtJhBj7orpJ@tugsSkZA)-WPoLYB{*$&)w7W#3rVes2D=?iaRL$O zOG1>8Bo8-01=vx^bICtvf68}kQ3{d|N^rZpYE4nbSUe+e>-znObrIizL+hyqYskWe zgpZJ)tWX3_W)kh1r;vm2{Q(F3k1-eQZ;W@d-^2S>KBiQE0$$!700g^ zIbyFN-HgqzC95kuGS>u3ZO1?a4!W57D{yDZd`kcaeu6sXh=(4`7Xyy*OuwRI-~}o> zh_o_v&mc*XfyRcym(->NBT`t9BVD=ypIJs`3d+!XCl01sN$d*e*eg~fqJUArycz3a zyelI173Va6D(j}CEU(0Ry!1p9X4w3%LX}>~5rVE~N6KMG>~t0yo{ZGWga)DXCRDqo zWVNfU3Ny1%shLShVQe?eXcB&4Jl0ZI0_&@g)n$052_&O^RI)(m2R}zO9b|-B(Wb`f z_w>51rTsPqesrRv4$DuL#3Za^jf3v81tf*0ah?2sz!$YpC3A}zLxZrrLUd`V7m_0k zOK_LQCh2@e-stk%`Kx0$j!HnFcp1?k{n}XwWf` zY-I%ff=DL`T%AdSNkOv2Oj|tdTcQAaW+sz=O5&qulPxXFfe*Q8*UE_8#=WXZmN7Kt z2_;Z~KH|C~r%8S2CsmW@h6`>;Rw=2^2URlkX{v&(mRLM1mbIuTtIr}*l(BhRk+s*H zj8m-=F*!s6!=M^AE6Li;oyn-toZKt_X+_)QdnIg&_y!l7Kl zwj))us8Gr7C=P;whh@jB`d|cdxN_f2=X#JT8R#rR#34cf8aUE*-YWjKF1nEFz45s@s5hU>*>14Yqhx{bx= z;K>4Ynv1oD`F?M|F(KQAX7oUk6@OjMU-X*Azr7$qdZjVc1A-lFtvNxB8l()3N;O01 z>~_Mfnbn(8-I&T-3eS*2Rx9@-EWr$Q1s{AyfSjX|F(ETnvTdS16%!GVOsZ{vsE|-S zz2$LAPV3_tv+Exg%Uz}U(VtM!BZoe7rYfbyjWOSCcTv)MbJCj&EO7GiCFyAY-fziAYoHOoT$#)@a zK{lF9lWjNZMcF)#==ZV;JQ4MOF^ZAZCiXlz2SEi(wI_VU>O(+l>CW2F<#7AK-Ug%K zmq#_|nM5)gg(yW`nf5vqwc{`#4F&guYb(XWa3pyI*HXWZCJ8=!R0J+Zrw%K`_VcL9};fE_rST~GdH5`{5$PB6rO3-*MO>`iMn1xcJ5 z!gyksvZO^1j3>wn%TDuX4bfQ`|1axsaR__)5NZ`ZcQOeT5GEnN`t(VUiEz6E9FuSoE)NvX|ePP@1u_9gYTj1R=tF2g!}hrWR*6M8R2YBF#cG+l-F zACPnnL`i6}h)8{Zj38{v9hd;%=ul!%%Q+B9p(bMMprS`C{~#IF6rSwNvy5nT+#9H3 zv7$Y^J!CWKVr~*e9hzLjhNn=g<}K>9s-YsLP^#UfzM4k_QHos#shk|hh~FP+=nC6unhaV4OnOGBCo38%mz5pJIcnKT=Lv)EOgV7*|MLEbCJ!9o{n z)H>vq#2?}x0vTykqNE{`LUBO&H&NygLqrtQG+^Z`v4jC1MQ&6$P>8asnz1+tVoBy@ zimJoJP>B(L^xPpwDW|+CYOkqZYE(O`iS5$-C|=lCe%AmRuUvDQp75tgNf4i`H{ z;Z|Z63we-?@CWr?6CY2vwnd}xwM;yWw7aEZj^+b@fPF5vKS0@|YK6h!*UaNBSifdQ zk;G^QoKc9@%IA(EOv!$=yk|>QGmSy4~aQqfJMyM zOTqeoeH}W&a!_Rs@p6E#MX1Y|?aTdBXrQ7nVJJ0%WgHFi*Nik$^K0a;X_g&Mur|tO z>6G_}u%sz}6#mCZKYYOWpJXDb`Cqab?EjrgWIx3Jd=Nhh{{w|78viqoI$O`y?YDMg zgweN4;*y0BmA%4+QGuExgd z2+>3 zR$MP=ltOxvW{w1^oZ1Oy_~fPqh^W<#tFe)!zcQ#{e`m5S=*qIYn^t+pUO`BBURWgErNwijIErLjER&v+_;=h28MMFPkl{o$;QkFm(?TH) zDnxByhs@gn3uZhlo{mln7JoFsu1{y;&H7+4Qe2vo9PPqoosNiP2kU$Zy8xAc5=~KR ztkhn?W- z%&o*&zUu3FPL!c?-X)COD6V6ASWBfcG>W&r9VXVlWYNu|EQ-Wd#WWCPrxf?3NF>nEO#h>FuI1|F%A+reM2G$1m0WO z)@T%|lKq^^BWQ9fC_~OA+5<4i%@Sd(GqxvL$nW!ok*mvOi)Y5H4Pcn392Q`Hi!*4d${18w)33u=xmY1Aa*xV3 zlW;nUY(+z*)VSDRMoUS)YE`m28{H$r)+DM-mv1X^Wk@|@YGbxZ57>w?I#C*clLU;m z2#J%@HPC*@XuP0|E03Kjbi!~UgROHT25TEKT2oqoQTAB(+8N-CN=vOTif~SgM<^pB z;fPoh9yKvHRW(-3;;nLnEO?9bb>&6d)utL{?&OAMq8brH)eMqmXl@G8Bh>;kuUQ7Q z>pQb~mb%p>Z8YTo)TSdt#2!;o6ul^o=bVfp`Qyr^1k)z`(1LF&KsA$O;kQ&~R#H#L`!=*(lmT zPR26a&=z*-kycFn6z)SIeg$7zxZ!^_x)5oAFJ8V+xQ=f7R#PI^{ zP8)QKgbGna2E_pkB@t4At7NhSa>=Ay5{*GF$m-^&`Vo3z2;wsy_h%vE1!{r8@m#WE zxOXZheyl*xj|Rl%;3$lP(P2v!S%qcw5V-+HCAN5f zf%HfT_JVNe2C)x}3M5X=HIJF;0O7oN$C{LGe)a+nEB^xI^5eE-mnUKk=oKRVQKnpD z3y!v*u;J_?ly`$t?_23gz8J)H!EJ~nUbiNj*{}ny)u02eq=^uoV~gZ0awk?tQxi?m zXj3}Y8hCNC-o!R1zrtcRfJF?Ygb{&%ji*~M{OcC6NFM%)MlGrSn-Xz}q`*a23Jx0KAD4ZR0%hdoAzr{#8U~NF!CqnbfG-Zc*YQ^NOSP@hM81_Z*HOO;|8W^A` z@9d^KA3z-p6My6b5O)`jBX)9ij6^=La~_rK3;W5G=gaqQ2KKW zBVq9nB)1)4eD!sx6fBU5WprZ%8(?NB0r$t|L)rK(df z5G>R*MtlT0g50Q#z*&8=BarU_gqkW*yG>#O2#q+a?52Z=qL9?`YAkm4CJwS^0G6dG zZ22f$%oI=~RJdlWk?0ZyqCEwWFlAJ7fhB3j8e1}x6^WVg!s=XI0=LSq@s+Ov3C& zuvBXbkz+tpVHO#i8nYME?r$eSF7AfQE{q2uPEBcQqa3Xhqi}4s?5y|CEgHPK(JXnR zq`^zY0v#{L4~qT}~kLq*AXSM=qk3}ePFen6`M-KS!~Hyx<8321Ar z0=lCyu(!i@D-|#p9F9Y26ygvkGzKf~xO%^yA#{Osz!0S%`HVq6Nuh@p3HYXj@R4M{ zKpjEG_axF)%T|kNe8bd`S&~1(cp7+PG9ImAI@Y9!Sp(63sy-y)If>SuUd%&8xI-SbpfNaUkn}vszJ10r9D~ zhm!5&wp8Uw_UQ)>)itz(lFANB+BkTR&H`LyEMKHJOoc722nT!60QcB_L6gPO&72At zk`IK(LZfhheCrOB*eS(u-KnxdLxaTNRn@W9rYWovMudy21Pl`^)Mf*%5ZWd}D0CZf zdtVb~CAT7SMh&hxOUkN7N7)YPII1!nBSs04R2ZZOE$M|yXi}8p+u}YWb|d0GleT)u zsURmY1f>}g3^NWqND2vh_Tn{vRTGN()c^2ze2R|hYGPiDt=&h!72C_OW znWXR}^$9nRwPfc8T2YJ{p~%jAwr|4()IGX3Oh;c&N@pArFe^~ZPfGqB$TLK&z(nAv zQ<+TTI-*3D>WDIJX#iZ+YXts&s->y!yDel+31g{sc~gT;j^9jXvi09g@;6hhS^6fG zJJhj%-^qM|%uwj~Kb{y_6%8FasLFZ75z`Dz;FD7?5_1}zZ1yy8#SBwGwrwQ_UZ)Zz zg!ZW0&Ssfc8;MwEw4jZ`1rd$R^9+|NM*@a{aNMm-8Le2)D_yk+LkiXn-+E4zlE6=+F=z%T;o?3kv1U?gA*q7}0r(8m^v5qoHGdB8_qCc{<| z9|^o>w7G@n(@^#jgT}b(<6?B%9ionl<$}{s=iNkq8~M->X9iT7F>I;(||m{s+kH>D0uCJ?q=a-HHh!=;)E!CLkok|hUsLm$dDt45Eg>4p-R4o*)fBiHR<7h z_vtO(Xu(`d14$JzPnI|sB{32Z^uA5V$r0LKNqPp~4kXlE)i!$ShPgi%7qXgui^XXr zFknI3iD|xN~iBYE! zcCkg#QEsqJ=9tTEKF#dS6b6+QXix!v>l-ynNnti6FAX>G1^NP`O*T629|G?FH~lF5 zA7;yiRsCPa{=v^$`oE>)iLAu`wImXmkBFDji4Xg~eIP#y|JP0HZ!arVhE1Kt`(nb^ zesd9&1EFq-c6J%Wnu^0G=L6qC!__6It_u!yLLTI{t6&QDLvS9U=E8`{ms#_FLfLsi zQavKxp4nTduQ)@@#Ms1_K~0ub64he@O{fOT&N8#5lVH$fcW-$Ih|drk2JkxO`&Gu9 znd1BNWt7s1Sj#MaQXU03Mo{Qe)qSDVUB$yY1*BpYw=0Y!#f=wrc9qs7_1Ag)5r*gy z9RS^rS!B_^8F6!YOph3j2;W41vyOG(=exw*&0gFEzBmw}dc0&5U)99DP$caR0_@Ej z_o^&FX%~R<`E8-x3xE)~p9PY2f z8&^U@OGPZF9Dyh_tsB)~Lq9Jly)9z#g~UlrhFy%=i;6}$N*-38p`Yfy7 zC;14l3Jyze8dJq|;1;QWW0x^{K(eLlD6Sd`&dUmhhV`WeAeTb{lhMF0MWacsG9&Qh zF~+B$t%w$n0T)JdDrPTpZOL9s*cN;O1D(~b1f9$M*zq*yB%QHLb%@h61@PFVm;@Xu zntKhVRl5lAWtCBE^1g219K)+9QOjW2M}_JMiRM{M0|h^jwvI@D2dDD811+`{Du<-L z43k8wwIY{E0BAs$zln*)T=s+|4>iRW+*(C!W$ZOU?VMz?UWgU=7$62K)uGZLK;SUh zP9PP}cW#9ZobS(z!%7i!ZuN&z1c*wL7_3pV$|_aq&tfFOB=9plzw3tO!4vBaIFN_t z{yx=GC1%Sei#nS)mgio~f0dG0ks>%^h9NJWxIy+`2r@W3JZn(tPopxTLLo+-)N}c0 zBAI&JNrOJ$3TH!hzZqadRLvB;KQeq^G->f;7Nmso=lD(N0@d?z6E5zUmDiX;EHf1E z4oQCs+0^Qr9nwH#7M;9N6Dq*0E&mNfi;q@^;VabaR`zaIh!y{Nf0<471C(e6zO;VL ztg5F?b_A3QEe_9Yp_*V5790%rU_F_GWOf8#;V(r@iwC{#0HvJxGBC!`5s==pD*WaK zi#e(0uq@WbMuh~<<)N)ZN5T$R11b=}MZi{sM*BpB@GxuZRIL=5eusMv7sY#dhQ|lz zOA8xcPY2DP2c$xEf00!Xqeed(?4LxGiBt;bYZ|H4j&AH2QQ$o38hgmVC^;7T11pX> zEUY=^t~@GIku;B;TE_YVr#q;b@i>#52)9I(7s*frUPSf);%!%O#ytN9?eg_bQp~vY zlm%fxsX{~zXBXR(P0i_EBk-;co**B*Y=D5QX&Bu@GNe&gf2h6!<&zYPmY#o%o|lj< z*vUENs?Y54o2bgRmgOAKSp_+z4tdz%7EjfM_J7S$ZO^+{Kg^J5%q})f6 zFT)5d?z}LW;UBY?05y_SV5-?3(_lnVElWB-C~pwj0D2`dL{*!K9GFv7i;YO>NJjMI zb}!~}yu#M3e^q3)sIJUZ(z{W~s4l&%FVt`M72WnZNYylgC`sV)mAl;}g)=!P6{Hd& z1sIqcx0wx2hsW!A;2J2g$;buPiL@?;y`f=)Yl>(6V`j<+%@<4@u3B_W#ukj5 z8yo^ZFAH-=G;UcHWT2UfGbUnkWdP$`IWI&cUATL%M9#V0P$Hc#8ujR)uH0OJU6)KJ z!Z~UNULb3mgx=IIH?TXBm?0tM`hole)E~P1qy#6dZY;^!idwzi?9Y1z5-f($rnm?-C_izJmRqBoVNUYRs)b4HR}ODf%k@X)=bK z3Q`e|$pnGEhwAKt-J!ua^lPlLBQ;ise4M&oP7*2vT;GUD1*6gf;{{1-Xr82Rf{$%? z&0M-$G!u_YTe*mVlw>Ru?Gm?_j}Jxg7=M*7FduIXzH1bdu%M$S84w#Dglt5)jU>=K zv|g!$6}4Gv)W8Zo6SIRAPLg`)LLLNrMnZ6LQYKgk4UGGYHyb;(A^dQD_u@rFAy3T0 z1aX(G+~!nbYw#0<*wyEpf+lljinp}Wjq!U7i$uZCEf|e|gf4Z7wLm=PFwb$6EPpH9 z0n#9=kNDiw)6RHPZ z)lT_BT~t8NCtd8VJjg3BNhIenK9D-tt6)soQ8Ab*4Q2v8U!AGI-tay?Ly9EPvyvkT zk*Ib(xp;74E`_cg7atlF)_mEBBMX36D)t8k>F`|iVUr(-6%6y2bhYjwpQXRl*9Vh8 zh!uazc>k~m-=a(md@T~N|M;fRqZ`qorGD&8q%G*Wu7r)1j9-2)=Fkk(|4&#;bs=OP4l2kHzylzp zLc0|5D8%dqRZs612DSeOaf$&W@?jh_Y-f|yh#7x_R3s?Rk~X6a1d3t_=C+aTxHo|= z1UN*ZBaSz$WWVA#Q!Sy;Ld#bTdSXT57}z!5K$QbR!SjY^3LEM*?vp8t*Xr9Is7+-w zSHKCX99lY!zxQBS=^jrAx~R1s(jerxT(o?V6Aa7h$sHy!M$pIT zpj&h~c1TU*xbq$M0)#@9o?4}_dZ=10d+p85gF>9w`eMS>ZRWLVo&{6}KxML|zXB#b z6B9ySZ=4C25xiO)hx@B$jCs|N9Iap_J2ihFTtAasMA3ZdnbtGSE@Roagcb9W$4bT{77v?_eY^_a=NWLkqgTZUK2J-fa0-gNqB&m5@RhYR1Sp z-YDv&>R7pWdgYo8d~oy5-Fq$U>0K~qmu6!3Q~d8^C23+r!Dpjx+01faa0K>edU|DB zS3)4`KEb{+6-}`PFjZ>Z%8?8cPUU~;au(BW@OO7L@X0>#bA&NV#D7bbBsw#Rb{Ds5 zr!6IpE)LvM)?o)X+JLwgREwE>U*HXtpn6gES(VSpMJiN74*PPFzN#TPXdqUA0DCiC zi2s4(6K0^ibwEw{Xh6b8-z?${dRvX;g9AA zR6kqJ?Rs>%5r??P)#&86dV1y8Xm-O0Rq!mekTekHDa1f&9{at5bioQP!n8+-CW1bO zryo8GQKeGFqAXnA2qd_M_e{)w$cW@{-jlA3GZJgu;Z(J^HQStq=EHp1vkFfCFcdO{ zF(l(vH8DwDk5gfmH|~lWv1@-QQx=aeEmh>s?h`(nQ%z)jyR!=KgK#d3$7bfAL?#h| zCYeCc=kKK1s7odTs40Fg`-5VqQk+!4N6Y+>94xj4mEA~7HI?1BgzM#k0K;rum-aWhZZ-}Ydm&c?w*IG?kC;P1;a20VIrFc8kl-P2}WpH4)g9VjwY zkxDz{FEPhxhUS0}nnr9@3Pd3QxFYtdM~YV`0izXXOt_dfc6J&W$!pG-zIbako^az- z;s6S~4791$DD=!KMlsgY;A`nzUt2Plbiv;+d|2q4L&LGSaf$eNg_F#T7=IwZp`+#? z&n07MoKV)o!LdQ@X^Lxd=2R8^XtbB5FM?2w{^34VBs0eC-W=nb9x;^q34i8GmHZ$yakKnr zA&8=3BpJ;WWDwv5r5E+c|5AE{9#>d-5kRh!jY2R6v5`i~ zRXem|#Zs&ZFAK{Lqko(8D$LtEeL7f4z!`wtg?7_^iAZL76Y>xP4~N)u*iBn)UBfw% z-crTY>cFKN*cS+l8JHb^StacZ}x7%-LCM}O4jwWyS|iC#sp7oe zeg)kNM4w?(`+qVvSMXG9n;X9>I4mX*VPj|7vPgpjRG@9iI7c5Wc@F!XLVv9wk#T$& zsDdhA$N_7jkE0B|NwF_ojA4gp?X5)5j$zqoPJRz4CJ^))vHhHeLb8e7tX$XeI=QL5 zU#41Y$ao`l!c5t-*dK*wC`B}p%5<%n3@QOJoNyX7^#@ZuE*@gSmGdmZ+Nfuhph+2vN&)^a>R)BUZ zJNAxc^fzlr&54{7nOLeNVcdmPh#`IrK{aI*1*b@gYKeqJ8x$NmNG4A`Rq0quvaVky z5JMzlZ6anPO%QQ;Qj|Dj(L~xdCk?y+ANFy{oW(PR7hEa21@PO;lYWpMT(N&#_i%H- zu>(yH2&lda^T=`B3!Yn%BsQtXROu#|C?sbT-c~=;JR%Tl#R`>fnm(OwL!iJh7d5(t zvS>{8gH=^{Bvg$aJ-}trK#lwQs8W$(RW^-9ybG)K`jhyOC=mm8;|5&DYo-9LOYmoP ztYkHlF_9L3my#V|Nn~76AsQ7t54b+buqx>RHUv0QNhueELL&QitUPb^5DaWDuJAcun>f&)dK zfQP^c6Jal8A4vSHPUZiQ@FfnCcoNbVG5%f}e}MF5>LuiUI7DZz%PM4e zLy8rDO=+UK4QWDM$sr@`i#3xJx-=l5t(O$Os$x47kUVv*`anE1!+Q}^#Hc1TF?)%h z;FOySnmwdH*vsflxRSEgFI7{so5WCV%CQS z0Cpl@riMjUo&N_iCRcG}pKB@ymi(~H?+XflhP1s>1o)1H$CDuq<&sy`0C>bQhVV3S z2WJ_`gsvMHe@!9}Tq>5ZgquhdMpZilk|{EO7B$?acyI$w7Iag3dgZ=JR?Fp{Uc7ls zDBX!j@Pk0Z($gFKn4c@fIHOZ=rObEAL5G`_q;k9KQ$kUQZ%jnY`Ysg>7y5(G1w(Rw zJYk(;D?BE3)w?njd>>Of5nCJw>Hu#X z2+#mwg!fuzGfLB9Ag*|g`o-`BQpCj)X?Y!U#-@9wOe113H6L;2W07AhKu8`!mi1zl zGRK2XSz4n|hLJ~N3O|UyL4Am2!mW&d3Vp>vXB4z~k_(jBCuJmu(P`potv<;95P>~x z+|KTXI+n73IptFFM#8?M;(S1NGR(tR=`1v50pd zZ%Ctoor=YSFJsrssx9u`+LpvmrQ2f3mMr&$7e*7Bwv@f4?ud}cG-|-OSb>9ogoSV0S8LcBt%B9v?&1=kak8ZMEM{jR&*=3>BFWr1!#ElWCCTTrV+&Pk|mwf;f zRO6PUtbz~~^en?L+54bWargF$MGffG1|p_?Y6TlC$Ki5SO zx8exX;j<@`523)3G^lB&IFR)g14@s&eQrtHAfVto?BI!MpcdtJ>FFKydT4*es;hv! zaZ1JM42VsPAM|TPEncTkh)WuoA!dWr0KpY>+T}_{+@t*XwCn|>TIGX(>TmMUc;8yg zu1vwZNpf;BX^IvOR%%N!DDt*UtIAN8vf~9ln-LH)dPz7{BQa{~H0mWmt{}CyHjiyO zC7!m$Y!nZDmf_@O^(HSXg354&d@GI+H|z)j@-q=j#TDwt()8yk=x`VImvUmfHrX1F zWwIKmqrZ}EjV0+jmV5wz3`xKrQPW8?0w}cT;LI2kgf(OiWKykQ^NGfNLR5LR+*0Hz zMP%znOYRu7)gojon_Eo>uGpGJyaXkYWJ=u%rjuk*A>~tVNp7^S_1B%KJh7 z@{zUXf3xwl#(%cN0son5$$pss{Xl*+{u2sTgpdS-pxvHnLqE8GZjC=*C=6d;$`?@Q z9CD4%h)m~e3+WEx+4v6!#giVSXxt1vm~CBMbc zUAM?FJh?!qU&I{Yqa$^iEGd*hGI{yDX8{oeRLWIc%nqq0yH4YZFxfjCyFV1fO+peQ z2+hxsgS90ZPhj(5E9ZY)s~ zMp%udCfIkXw6&kF21E9DUUw9efn~HuAi@WZGME5?U_jc337xXyK4Rg5U^0Q?q;Ngn znhgpJBle0F6vFf1Yz#xPxLT>HzH>w@2Sq2*QYBi=7d2@GuH>9b49ilua}YxH&0mL6s~>slAdmr^i&VfO{=lvqeHb1uI0Ta-$THrI>cq?rH&O zD%tOFN+}J*d+kzUgaVD%iEa_!Y*l4(4XVP}^taT1+(qP)MkcGa! z+lf$P1F4Rj3kl&U0O(TU#nqeeVo&d&aJu*{EASR=i%Gs$8K?%|8FHOMb%-w6(+evS z$Bz_(J}jCP0r#~YS3rixOi4IS=~!@fazg~_W{5g7IIfSm^> zGLgnOjZcotGYn0B?Odf)h9^?hIa&xn#EAiE041--En(>;fbZcc#hH|B;e$-*(U0}v z@W|brP*x9Tc@bL^xXixUYhcV;9HmKr#1dkmc^9aX=YZh1oAwjdd&Ex05`c$;>$k{n zweoDl#7CQj6|y6UX0^>bVTX4}b4bZ9v|D)u;>zNia3i=*GM2;(rLoT@5C(wxA}pLq zrij2#fs)om)Nc;9fr@L0e@5BL7ZxqT{6Q@bAWq#ZU4nT6kqBR)gcRg-fGV4R_Je`x zfthCW7?^p0EQC&AB1vD*S@@lpt*D%F0dx7B*``ck$fQZr$7r>jMy82pqU(b-DZ@L) zGf`|ryot7>>nPTKm};6Od@!a`GZ;an*vFitY#>1;5z56N(+DmgKBX3y4S^6Y)~FZj zv0?qgI3vV{1)M!aRlp2YNnE>sjEFuU`lcz)GAxRqw2;#+G( z>7Y(uz?lOZMJwXcfVD0v*bb#+7+SrA*trNXS1a_55$H~?p*SAgs6(MoT3cIa9J&} zy#fGWn|8{?T8$mjGSQ^QX>>4#XD6?5k51XeQ*Su1MEOt$@{Q?#i6*0bCSvDfZj8wY zN`-v?xJmAmYmnlVqJFF-?SBs|(KM2lS|=l7iUTtZT@2Y{wLXf$#P~TA&Q_7pb}Swe zJ4Fa-F%z?ZCtmoj`A|HT2_td>m53Be$SHxhHI^z9kQBsIma#OROh;8l?&uQBW(U6CvbzTTwwFU>QBI<XASXF!xjUZu~;b8pc6pcz;zE~Q;C+@3_V76G=PFYwGn7<|F z#o}a9xg+55P-&SA=5Qp3IGanAaSNCf3!G3>sWe@b8nEQrc|ZZnAORCkV^THQNciY! z!$K=$@pWl`!r_hkE|<1>RPH0m|RyhuY?CXQ0?i(OOe0W@I!QI;q*eq zjUFqK4_8apfEb?3W-**A-XqwM86MUnicDeR39ZPJsGbmil=teo>4i)mNXCgGlIJuK zGs%0(L3IEs3dvX@E}$+!NH{DufV$zBL{!zc*CW%}NF`1HtOk88MQ-GaN`k}L-;eMG zT24~pusO7th)QG{CVYt*04gvlrV~K68H*Sy;*~@XMJx{2=}nBN^<$S5M|DNUMoa() z&-c09H$_T7Ky=Iz=f}2Jy3O*Swl-9@crH3%#U8H82;hz)P#lXssW4gxgeEKtPpA*+XI1v^cXrJp4*&NQHT(LhOa z?XclBn2rmCLyguC_gx{%UKv!C9i>;Dh`sQkYWUA6N20{~cC{ddCPe`-Pg zr*tCyVgCOE`BC}*xrz!$ockrP8O(G)?hbzsm@cg0oxn)E!J~tqIVN}X)&BYj{DdG6 z;18{a<*n_^JV~8DD9&Gr63vXz$R2+bi2$_^eRtIH#++gvDsDjNA-OnL+z8hmi_ysV zljUgn16>3!#*^uoHB){C^alnogmOQi2C`EYC<E1MyOv9T0}Ck%7fB zV>J-09fl6;5|do3!u$9rRGDm)SrH4`XsI8oSD<#I+f}4HNL9nOu(9(Zw)WnH$*YPM ze}gs5x=6R(5+@J0nMdWU&@>c*zisI>VkH4O4}&%yL%8{s{3?w0X0XVo6)I5D*|cQ2 z&HEP)2AP-E>9H2#Nq|mIz zv%*r$8Ir(HDaCc{do7(8ZKdxizXUxO%6TFmI*4#MNpkE_LnQlG0lF$7TP}HIf69gC z@%u4DlecKnfd)z*M-%Z`OR>d6Rrq8hm_p2{@#JL8WuP!lUXKLST3o*8HAdtIN)LjC zbxzfqtfaS&H(v$qxv}diOpb`n+qj1XK%RXbnQo6k+RX=fqSBSAp-xbN($57XNG>CL z?VzVGdn6YEoL;`dahY<&<|fr~e~=GDZXp+itVdnBh1e#kIMyp9)FXnM$>cInx4x1R zR+z}*aazDQ@1WB!Mj|BU5H;(j=Cu7nmKRB1%k;O5SOw`ZLl|LAXA)*1h-S9o95&bQ zN!T?zkC3UOYBi&146X)G)27zCuAn5WfNwYj1i+u7lNXZmn|-`!<{I|}f3r{-z=S5^ zEwDjb>o#aa!gcz_WoRF#P@k4upOb5KaxD{->^3LOlu#I(lrv1TZppxz8JVMKJXRan zV#4dE7g~3Sryb3ELdOmmZLZKEho*?#31rfbCX?}Dj6ZSW5xa^cl3a5Fhpa8hW$ko? zF>=XR1F!;pp~oo3y*cd(e@LaU>KkC;9=FBshVw@e`~g5HZ^$D9k{5DhN6*+ zoz591(fX zGP_zCmIgP0M;hQ!BCkOA8c2xe?wkUO&bssetVeCRo{TwfVaFf~d@`35y|7SzaP>C;pcf9L-+S(-UlTIZA0v{|zQg>y(T zgXsq@VB>-Ur!WS&gac|N!`c9ehE%--%`!1)j*07wz#p! zO>lL)e*%*-*3@du5lebwZ7p3)2)Xs1d%76KGpv02#;yMZH;dyiK$x078}p5nUzZsm zhk?lwOu`7zdc<(VGVsE7Jk}bi!w74LGp;!9Zw++@iI54JE4Vhs;`ms2C)(>9oJk!p zXzcAmY}s*e>^5uCmw6#1xUw9*U~d%Uoo07>6gi$s?SjLQS|1(tZ)B{Z{9(F*Jj}q))Mi!7k12ZYVux0WJdkv_6i~vU@ zNcv;>>cb<7bHT|8F$_i;6^YL_goVVmTY)Az)TmSP!INj?RVy`aMhvND1&ddZ5V*XR ze;@GmxKtOUx?ly0$X6P+ykr(h3mabI8u}VAlS2zOlxl)mRcOgz%gunX?4zNAR`HW@ zSHYu}aK0HCqwxWLE-UEnZ5ge@1Dqn5T}O@N2XA{k#S86TuOihbuvpG=oNA|si# zHb%0%*>Ma)bxxI+jIkH1`o$qe`3T{tDxNcdUUTS7e{p8>tf?p=N9TVB zs_#?z_uBHm>3B=R=>L;~`tM9j^27Y^2l3OR$w6df3uD&Nr5M?2M`L~&aFszwe`uMI zXbVfemO#+5j4)uHzJeJW3oV(qG%O2xOCGT_1(cfX2ilHcK}$APvt?toD6j>httO)Y z1u#9(2}Ck}{=77gC@f4p+pIFeywO-jyPK7z&Etb>%kpQ3dzLCBRFuH_Nbv}UsfCpu zMtfpk*x$rLf~mc$S<6gLWlo2zSf_gy!t2zSMj?t7l|5eC!2}mBuT#NHun);6{G8S( zI66=*3-NAW2RdOkB6V4<%G~L2AxYUQgtqkNs>Bqfz_yAUinX3+G@-#>5QTzM_Nb~@ z7DzbukdvRP7Jup>$X>!(NM>!PU8`CbpX(bbvH$llN6w>Nh!5GjE@Kz?_Fn_)z3Y8^|)25sem0 z(8b3ap3sLkDiP9X%}~!&&!DChZxBJ7i~vLo+Yz5Vq7!qHt}==wq$Hs5SStGpgq4(d zI*O)(@qZO9e*A?gVobEdIQvczd>k@j_E;3!Z7s1ZVf;BbBPqu;@({r8@p#XUlqb?t z%dN!wT%w1a`I9;;bwGvyjdU&Bw*i?6_rqVB|C@J z>3>$6iL7NScR6SNFNsa8vRjKyj~eBwCLz>%#+Fom^&ONX^A~&E3^12I7Wz93pry8L zDr3dn5UM4!v2MhYn-5`R5h&8jQK5cRDn_ZYWST%mjO+Gsy-wpTEJq0iHwcZFmk0R; zP0^86MXMms4xv)Q))0hV$IWZ0&Fiaemw$O`e+ybHM3cHiOlrHG&FYl4sk?M)6Q;XD z{4>cU#BDOkwt9O?87~0Qw>X`xa^I9=q+m3|WGL5~gS3Zgt7z6Ls06tfrY?zwZc|7% zUaF$Br76({MIs$bBwGMpUqJjb9AR-v$-;!77rfA#v_rfr!=`aZ&kIVr6^p5{91OZO z77tV~)0u}lu_2SXtsH-8Tiz70>hvD2Xv8S`Fm1a&q0EUiGFWf3_>*M5glEE*1p`Knhw2cdW!St@Mc6D=1_ks%!g?XI zD8P7yStnc*Yzlq&&d116J(N@dUp*8~z6!?;D* zi%w-|)3TxQD7G{7O07YJqlP3imW-Up-BBc_3y}1UdWLZA7eW-Ore&v43=y_^SukMv zSfeAuu7O*`J{01hhyywdEO^2<5hgk*l%+ZdqqHL4~}O_knO)_d_I!?L2>RN;n4WSJh37UGil3}@DF zK;xbP`J$a1axPn&(8^SM7R*i1eacA%S_ndUY4S~lF6)Mk)~-AYY)`ljJ_5oEg=f&v zA&6)jlb9%{(h{MLU?E1Kk2PoNm@mN1K+y8Xp z;=`Z+%8wz=K?}SuDZr`oe}6(GBwHx|7Z?9aWBq3`-tuAo?}PZ+(KeT!J7~GwC6cU8 z#(9_t6vjCG0gi zKKY%*hs$mJR$!l+q_{eRKw6bp_(mrObn z595uEp7dxQwVNd$YBY(gaqC{R&{pfUjL7jt_-QAa;Ump>!Cu+d9CVm8Cs8idS(6wQ zGd==1?}^o4Sv^sxz<-bUMQEvNsSNNg%bsR0@b5kGO3#S6RldJesErg?Cnr|-Ppme) zc_R)Go@o4O)g;3I*`6LT(2;x&?i?yq!mCr!w0Kp&6keT)W+$v!u!!QyAz!O9=B*8j zH)xpT)%*W~Tkz$qF5Pu4gl*Ttw(&E~PNrg+B>dYFYmJM46MwOI3%s64w#Cx$MnYJY z!T=K2RGqy;a?gkTm%?@*)3{H@G-_QT{^iaX(`0K3ajCg_zuTcQq@2YzfUO3kco}#) zar;afF4h{0x8lWG=wj&>xGa32HP#l#XQjvuJ}z1wa=rZy0f{TKp=F)qQVd&|B9ZjI zO|ty0Dai8MB!A1`n!qTun`SgBr)x60cA#hsH)wE)$EiV|MmwaljF@7yWMYwVXH=yw zEqEq55+?tO6@e{H-9}YN9eS`>llNdDz9a~68xTdRq{Mw0;*AL2kX6^bslE>$+?Trk z5e$uo0t9y4IgjOug_9j8qA)q}oZ+pYzl^kBbkY>i+LN8MArg%mwkVK1{@K&(^m!BE z4#*IUlhCvpe>ECyOLFGcAFZ`Re8;lzZ`zWvL^A-}n_Gnc99Ds1Z1e&VXW@T&1~;k9 zH3plxK-K_en-CLq2Q?w&elBmS0@(1Gc(W=O4Ix3HD}`z%9&1U28JNluE~to-J&%GN zjT(3Ot7`7uY*e;gbGRgDXKy~n4YCdft8Zj!1T zP(rHkzsWl??SJI4@j0O@J-0luI#sStG&G3|l|sMx|2m^MVAiGrMsS_ooATMc z4Nculz|_q~TPLg~SeHM|o(T;crwQ{4+sZdVf0@%XMOi)#Vt=u%_4tFCcp|Gmn90WJ zgXxxp`k*-o;KRM|wVRy@? zOe6po#nYA$vYf}BNvmn&Pp0B&{mEpewG}^^$|RoLE6wN!Pe_D(?%K1&VwFd7P&&IRHC)?;@6Y)f9?ROj+ zH13$pnAb`29aF8@x;v(m#xurS6ZFZJtnZHN;8V@i!e1JsvafeBC*(gcT1=)BETt2G zd03z4rd@=S1uWOnQ<1nshJ1L)=n_Q(447FkR_{aa(wUYfVF%_`WCpo`W1ygbe~qy> zWbdPlS>_gmtZu3{At7qa*3zX?8f!PiibX?U{~{rLxa1tkzD4yzoCykPUa02893FX| z9Rj3_x2TXCvovX}jZY)FRa$mF*Pf-2aAAool@)RkJDXkFT@aqOPhT;gL6|VwvW2f2+K;mUw-Ip&tDYD9K_0m=oee6<5Iab+$7=bi-y z>YkR2*kQ5Oo>}23r{@qLz(Q)!n9Y*ASqd1j%)&)U#_OB?m{V#&5FnKxe+n{g9BXEb zsB%$(G8j0>^Q7o`Lh7b2QFub(J+mn{sDJh%Ms6<;%(p;GD;!TV!%lp!CYT7eHS*$y zFsbc9VVgL$WX^%`ZXO!Mv}3#l(KCd%ju1TnQST%`L!=?cC>0%0uuA+CTw_9K!%UPH zxmR|G{uPrsqsiV6KEFJofAVxIR!9TW%}t*k?%;bQsX?qq`b9i`iGm8kp!D*&8h~s^ zfsPz${J>(2!n$IGNufyG3KoDGqJz{`BQ!K9yRia&>?mF%fVB(w>1x~-j(ZeJ2Zd=l zG`KvhTlh9k&PiEGuB2fiT6k*YcM8In$Ai|ROxKBkFNt&Wfq9dLfBy%8sQ;1tPa>Yx z`Ja{q@_%L$ANqeE#1Hv@a_Nz$E6DJKhpJkN!oTgB-HBRdHD}`XNPZRA-Db#9LVvVx z*lzorpq-dxTO9-7ECKIIM{=7RI$CMd(ZaFfh=l9N=)z`kip47k)mx`4X6vpf9>|=5 zW+;SNMVWm=a?86Yf9wZOfTDR`i&rNtuEhpVKN2*UldpRH$+CSp{}W7wC|{?gkni?H zcIt_k3nCt1f)E={wz_!j-6b8hV3hST8@xX_p+GPU^RS^Ya&0yk`8GjcnSv_ z#do+r3)8>BUez1D(TE?K>MR=3;47)W2okNUS-zCD-;ON#QZI$|r$ysGcN~NhRrC z*uz*W+ zRO=+PwEIe>VX=%JeCofmEqc)UgDl$!^tM^uGgR)LSdA79(uKfEmaD6~Aw^aH(FF}Y zx}fn#XE*rh>O|B}hrPGh)=P=zL3c)rCOVKFxEUIEHgh-CaKwTM61NO;NFgbl@AeTG z-E8sE;GVIj+%pjKqBv%)SO9-jx|Y$ z-=+|MQ_t!|tR<3+r6Z|WCX$Y|_DlrZ&K74RtkS?hR6zn`#852**wDvdM#AXh7Lw6j zuD=KBKkeHEJPbty7H~2qNpQz9nBuOu!lO|~-l~LiiQ>1>QT8%dH0a^zv$9p7)PTz`sfUH_QK~jMwWD?lMH9ke11%ix*EAZ!9tw z(tKl%rUSAUCz=;0eQ#o{Ncf+g`D&MZO!3e~*C}AW9ss8Z84mMriLCmq)o!*C!607| zP9(jeK9h=veXiMpV!rB6e_B$z!>NCPr?&ft%T>4*4&HYOBMt)Ravfiheq4u+MF!r9 z$Zz56;?^@EY~;H&?3rnNOh!GK^ou6vh*x)7wq#Dk-s!+7B-+IOdDP)L*R>I9vm>*Dyeex>gKCFS{HF4gpk2gW>RF)qSMye|K?Zan_L|NQk{s zlPw%I^erJ|`~HekIrYgj_Fq z)peHR>nPXLEXOnW@1*h@EHX;zPLQB%2^;U4`Vro5iN=#ti%s$60aGOqD?$h;7K9=# zlpGWx)HgwB^Ee655I5uA$>x6Ydvrp-#&9F4D!$&hD+O=FM$IMKnM8OeY9PVAjFasN zC0iTFP;o7ze{y^j7`~%Se&sI)7awE`7Cz-~z$;#WZrMJ`*bH`y5{(g?MTL|st!}iH zk24*2{mFzI%~CcX#+GtOT)iD$hbt^F4}P;fmXpr`A<0l8!Q^Iq@n(5CXJcqg+y`Hg z(=iLpOy6{5k`Opfwzir!X`e7fSJ{J(EezcmNUdRZdmC=Roe|p%njDzJ@ppl5Ag2tmgm!PlOiG0RUyp$r@joG^2s{7Z z4gjtV|DDOERQ^AkfcRe~p82r<=LhmbaXWMAA#msOmB^0}dZ*%!B2kFgz-nK~03(K%?E;HGGkA|y4^WPSVo_WS|Q>)w0LJ?Gq=*0BS}+}@ZkUxO-n zl)ijeD=c~z;I!O4kaEQ!T>8Uegk#|;u&~I4?r2G%l#sa2pJ$Y7>g5d*eY=g4^|F>I z2RrZ)#X56yj|&!_VGrN=@R9s;nzlFUs4)9cM&x>201llJuBUD8Cailg!qEH7QkkzI z7@>9?wfULV^7*jUjAhD!Oxwk7tH11wxHF8~q|d|h0l=+@cMJ*nl!L?zF4w>4o`qZ? z2gUB{KdUSI`tRYBN;k$;>rVeUAe|PTQZWCdCLg*>I0w?r>8B6wJPKEyDLR(#>TyKp z`K9P5WV1Vl{;YQOBkIkpD?gpn4Ymwl9QLgcSKXCtx<6D|F7($`L43oh`vo@>qEv>@ zjqPsMc@`9J2*@8;{_O4=wp((a_Ijr-8u{{@nf6EAK<$i{SowI5(99cT?L?P) zM1P@M$Rd!J?ZN`uR7jF4wL(Y=`}ZN2QoN-lrqm^E>Q8`1dWFXsR(T0tmYm*j;ew6ZV|CLYK75xceoM zFw2V8ldU4)fcxj>keheaEDd(< zni&}e=jQVvHADNGFC6Oe-I>fO{np*vFxhnH4dv@uAHr}8!Wgll@qOVx(*8?#Dt*Aa z*LGKexNu`Yb2$Ex-@so}<C1H zqHc#A-#k?Y+&$>=<<9zyoDX5o!#D_2$o3GFG#I$wcD(6v(_RIsPba;Ob+@?nJcvE{ z)1il-@AHKMXWt$BT%)13DC?H()K>k$ zd%(!BVyy90YS+DzRu$SY-_)Ni?EwS*Ln?Cbh~&Km-T5oya{2GuE1#_=1!jfSJfZLh z{yZ#qHWRCk_7@6&Y@Iu|xf=K^;Qa2k^xJWwu+_7LaxTa}B!g6N;%%aOw8($;pLehF zzlO`}@4q4W;~{Y9%WJ2rX<(U}@`v+Gt!oc{8)(|xJ-6Jl%gORpvH9DM%>N`gF)w## z86aUAqGgA|#eO`%mBf#VeVC#CGWf$M7PI!5#=T*AUJS7L&^(@?fVq+y85be zLgR&BqHQjIQ>%N{Ov*)9!t#WSMPLRWK}x$Oqp#_X_Ls^&w3slS30t`le$NKGku*$_b=K%yv}&+rQZ}E$a=O5 z)0d)G|6X!y=}71M;@`(T#owB`y{&(73B4B)d#+0Uv7hn4>Be();#)4Z+E18C=aM9A zH6AA`tvMdqz3}7lH8;iI1>&P>XTQ%k4_Esg&9W$e`f*D6xci>{7~sdNJstvmSKF_? zPvpTjr?R3pk0;T~8nh_emsJY>vjaDo+rkTU;+j}5_g_&vml4}Hbh60d$Ee#$l74gi zJ6tPJnpe@YNuTM`JC*6+t1{ph?ZcQ|ekc;(ZW00-9$ zuh%b&mb@RFlxkQQu>eY0ZRWq986GBjiiH2PR0(W9^xRo4p5d7Lb>iT!wcGc6uzmP#sSjXIQvU9@}}Yv(0M28qu0pfoS=W= zH6gu|Gq5YPkcBxZEr8{zObMEsiT|hV>i*GowOdpCOf#{vs3)($ds}XSt=)6nW^fLg zPnmn?T7EQK|LxW$n?gLR{p`z;O91g=DN^*`ao5y@kyGfL;LPG49QC>dfbs9pZkUT7LYDtr#p-|NO6d&)N zM!~elURO$sv&RTYG%HK;#?Ha^;ahPph@L}qEvb?dONyfaJau}(P;T3!gElT|>vA35 zXrC=Gf1oC4Zb`Iq_}+VhaXC?5c;Q`$xvJ-^en<5a6>>y*z^3D{+ECBS-J*eO*Pa8< zepJV)D^7#F6)p+)oZM(UAQsyI{^S44?9yb#OKS_FQGZ3ps@ZdLWq`D%iQ)KgV#E0H zr|=P_{0Ueh@aVjM>F-;S&tByR9~O44v%QhYap*Rp>UJAFU(3PIzxtP+{UY^w_N?d4 z-#e-A9o@9T#S&(fjgFMvxL0;Yy7Z#1o5F(37tx=WACJE4E{bw)mFQP5y{<4CScdZk zJ+ays?d_^t7ezaFP?~aso{6b(>Kj2mtzf&qR{V5hg~0}FyzLTh&ME(AEBbS8MAUh~ zZGIkcrDPX5Wb^Xk8J(f#Ro&J9?kE&Iv2!_CJSd#{^ZM*ft7UXGx^zFV)oBV+h|DAWyQ@f<=9^;{!hYw2y z{EVAgD*o`!_wmbU(kYUB&i|#jH9~3>5ab%+D zHBo@;u?Yi}GMuWcmZ+HLy4K=ny!cGo^>RV-qa4d9zSqM7L3`*&!Oja*fR(nh%`^8< zw^k6u6Yi6qSUel>%PXgRvnOD)fBlb}TH|`yfsjqRtL28D#Qlght9B95hd}N(@z-E! zo?77`7m_pZg#4&&PuX9zuj%te@GkygEkp3tfLoKtsfW0(H^HZUiWPm5$c0~ak`LUj zM6>*Rw67muJCZ7KOKZcNOLylZ-ry!q8D;Tzy_i^ouNFLN^r8jm1YERS=39Fz(Yvk` z9z{e&3iNaT%zVu|T6j3lrTm!Uch^W0lV?2H-1kq z-+oJnfZ_lBe6U&j_UmVjuZP0GyHmqe>-jS0WGEK$?i9j2k>h}2fE zvw*P;kY(7LM+DqP){0~c5iqIdy@%r&sly`>PKV5miyum0}O?(Gtg28dlYtkJLm_=I0K*%Qwifm0@UMSE(#1wbk=}79M5m%^4{Dv;!tK_&uLV?$4t^>?C77x45~9gCs{)n2n=a! zs8LrlR*R+*;hPKz7?xKf^z*IkX4S-ksFR6U?j=O0Fj$SiiPsSyvv*udvRsfEVscYM zWOgQqNIjDGs)~Xq%pDnt+_((?r#F8?w^pHp+4w?4?PBx4y9v~h{*r?EIFXE0-uPW- z+&H$;QGnX|yI2IA0#??OHi;-CvouymFd-phL$7p1S8s9Jopb?CO`PC60JHM#*DJCqUUid%&t;&9WFU2?(+%(nqHfd> zSY{iN;^0YJ0-^#a*ojjsjWq@rGatC0hpgS1-Udmn0S&`& z&84sQV%%9%l+M;#YXync7X}^KdbYUhLuden1IKCOdVM7ViU@Ay92gWjt(B@P)Z>o# zB~E%m1n5-jVGc%8Ompjd7~Xp#FXGDRFRKb?x^ z>Xrm{duyeS4I?N|$Tb0Z2pevXsKn%U>Hr?c&ht722k5QDtXS=`p`!9d8{R^fG3=*K zP%BMfAs7RLSktL`YOF|JE_;RcF02wRJNYq zn!%XO9Y`(;rBG!F8#9^UbTwZ-RvchShy-)Iv^VfP*e7aDljWh{3F0 zC}gS~+(X{67fH(SreuiMjJTKZC3~$Zfz||=U`?V|N-bntSD*6@L;SAD3!JH}foap) zHfb_`Sk}UFHj^L9o(=KBva9lo2wU+&sP!oYL9pUej*&;tIs_))zF za8-Aar^pF$2E-mo=1sz859GUXr4=ik&YN5fil@6*Rfifij{AnU~M9 zBYR~hc{bq%3n&Nn_#=I3i>KqYmvO`pfUPu)p=tEAr z7#8u{-3;=^=OGaQ!s!q4BJK2Yo6J_(dqBADd|7$X5l`||gB#?`@O@DD;fRsvL2#^f7N(D@RnRf@-B{`)-y(8#)>tO!#(odTCYg8z3G7+xJFw_x(0fT zOZs4f=Z>U=2@!%fI~9OX6!}SVVfEbGLPFqjzX(va7&!vRb6fIbA+sxcfSV|f%^vnV zU{47B!Sod;jaKaB=*R4A$}|(iqG4J=c^nXA?Qab~Xc7t_8Zkxq5O$^>c?T2HD=dM< z@qb`}SJZlfbWkB3KaUX5Fyu;lM`4T+ey2xG&{sXbMuZ11(PqIG^d-0qcDDJhxB#Cc zcwbWyWWU%QFt)fsfurmk>bn5xfTfR7pdf(`Ws;Ft!gu!M{#(+bLnn}n>*;#KUt-2K z22}{0r4h&<#aF;rgopcaG|Asv5Py!}O{3?I?n7zFK z@@m910K#KeEFw^RFSTvdwy(2vMPFWqd+^SGnlf z0KFhk$l$MaU({MN9^_G7pOMOEq>rUS5R3V()@6&bA4I9^vs!sx6-!-B7yq%{(ACIf z1QmGHJ45TqQ44je*UwQQ^9M#GLa{+DI_RL)76uUe;Hsu0x>-U7x3e=4TSG}Vm5&jH zu158yjZT%w0-?GSIK~#7z`;(?RRI>g#}ZEb7mqwcz=tds0pv_qjO4f8iA@a03N=4r zj9*W74IkPAXHl7W=C%sJ?y?h|PiZC_#}DMM%cD^=+2;T9`xTg28XWH<@2`| zw1CT?5{6=p6>@|?w^8vKdp?4inb?!e+(tQW&BBlfikiTGEDU-jW7~6i#LG)_YyE>B z2*27QuP0!YdP((f^=x9qsZ5qNh_EiHg8>Jv`O1?Rb*?$~0EVBxMJKZAml5c(;FkXT({$-R z0JUh++EWQMz5z=y8@Qqb+~z=QnP9e1ad@)_N((opp=*{%P!uxKSDee4%_}pZX)?bY zgHJlr#08VwoqmxFh;#c^Hwem!H?%z%&ziO{~25gps*&H<_Xe!bp8cYh9&OkThj zf_J#wxqNW%#OwkH#MyF6<#ExHe8pgnn8zMxOoa6iLXTY-R@8L}UrLS>;#VUWt|Drz zNd*Ca!WJsnjzr)X3o2OZCLL*mv+XrSg}QB}^J5f*&^xv(9UM^gTr1!l%u~n{c&}QP zthE?+8j+v$DIpGM^E=o7)+iSS^j z>|QdSbCJyAO6PH@^5P%{(^{}rPW+a^=|F21hUHz%2*EQ`aj*r1@O`rONDrx!ScE~VR5VU03_uYR z_ZLwVtd_f3$eh=9Vy7BpZRjS4i<-&KyDFu=(H++<0WG7nS?bHA0@Er4)K+_FD>54{ z6||n4?a@*R$M?(n(Mqz@$m$wDXQB+3BJh9NC3TwF$aIHC2n|_5m-o zfjIHFk1Y8rhj5@6}q zm$og9p0sb>${o|{Vb8WfOu4ocx)j@BV46_FEcX>Bu1uM(8e+AXlgIEOJ8H>XR#R9E z9>p{6EoN3z%V~|M@~O7T3ShJyIZ&mw-rEcHS^E?1HSX`(iDO~}O?ntD-j*~S!b*b4 zkQu>q%i|+ti0nr9EWlbu#uAX@0bYV?ZuZ4OI43YLmss7}_kriItY0edt8>Kg-QN38 zdhI6%_6Z9%qS*5H5YXA>oL;G7c2}pKjd-*lw6eXz1I?c32I2rI|7C9&nwiX5r8$FH zZ6Fxer+4Kp0b#T_Ch2)2pbkOc)yWEl69hg3#v*)n!|x1i1;k-C8Kc9F@IsZiae7xb zCyd4{m3AgJVo?ZQqzTfM))480PKEGB)0MRU1Pn+xp{Bny0^`VLSFi_`ThN%yw`Fdp z2Eey~UQ-L4uMK&9Iiso*wNZL3d7_jWG=J>3|3Ht(29<&-oE7#s8AR^FZ6 zWY%&*{vrmop`Qxb9?(+K@=qPGsdQ;wM`x6gQbw%(heVkH=$cJ-NG#+AWfg}QUmwQs zHvjNOv^NH{XjRiB6%cYQn;y!I?EdH@7>v$J)*sZD@AE;)~T2jLUzXKlc_bFqSP1^D<8-sA^O0H=0KI;)}MtCcjNd2{7 zN${g#$>!A|i_VRl)mMg?&n{q;&)V8>M))Fm$4SFfasm??qUbx|gh;C$xIWOcMVee| z$Kr#=&{{?@lRyY#mw1=vS{L1hzlx1Pk8Z~izuADT=MAA#QG+){fXadAt=+hn-Rbg) zs;D*0FG;>~;o?vElj&c^eWtG7OO0y;ev?K{w%aP(dmC}vM5&jXI(st=-<5)A*fE6@)Z2kQ}GS5GO zVie7L(FnzGmm=lGaekCBVvkM;W5W#qW|$|bz`7h69W<8_3*^unhgkAr!m|YWO<{|os?c@8}2IbE6*K4T_)oLv$sxc5`tL) zmOoTOgSyMTK!Bj6FWp!$P7g2SS7$MtM}m6EjNX?c3YN93Up3$wUdsqy{ZF&88qT*j z4EBM<3ncfd@)tKSQ0)pHCgUpUkNLTNpH74cfIJ^CHwh(iUSk7fPz#xVFY0tJN(Xdj zN|y0~jfj23wC$XMW9SKQnl5qPN1BfEpLi9CdbOzjvR@iuWK?I1O%3D0&fdN zkruF_sKgiZqXL?_kqCV4cCt=wrInzTLCUs2Y%yMq^jl|pVnH5o zwitku%*s0xpzHxBXB;D%m=XOxDYV)I|8t%Q=dlZ7?-lXxEjCB?2-fL%pT*idrV*Ln z$bpIX@;p@q(x6A{N zJMDBZY67f2lUD0O_-i|=m~M5~-W56L)e<(iAW?-$^0^cr;@3;4-C7}oVy=4XI4{Z) zp!N(=FLjS(D5qq=l*DIQA6o`KT)CTSqBP-BV~-U~tT^{r3f}M{^-vs)gd&$8OXD=( zu8N$X+hFZE^lBi6&npYc^H5d}9Uc~}hIlt)giE)NIWG@uIZ1Cs!)V39LmFf{>ppFo zLDQaz*>``jjZEI*>ca4Ab{o|?Ppt_cZfjPrM~LnQ9Y7+-jNu&Sp8OE3Fd&E^61>b| zXl^Os#i9?Os5;0Y$}#xMprF`Er$VcWNlFUE3AkF^Tjd>=TG&bO9Y50TYl92eejY2c zHQ`m%URyhIq_XYv<>7n#P1 zR10=&!G#xAWC*$zVMdc1Rp~FweQk8624X!!n^GwuXsE|5+=|f{#$Su3(|j{Gi`@pY)>WJpq41kKuPvLHJGii$?e1tC}E%(8mx*i zTF>MVL421B$2mJ)z8Tf?n+QNEXzsr6g6C%23c-tQK>hh9=R*Oxf~;y7DJFQyyeRQL z3ybvbyw5WgY>127RJ>|W$a_y9F!rcjwXPh?2S)v{dAX(ne>XwTF^RGZ5M-Y#l-oKs zB5=!}E={JgO6l$@#cMEdG%c_@6D?D{PHKcO@`Dzk+e3=5nj5JAZ_`#*oa#S|h$Zp* z?&1+m$C%oqUnJGtdD1;q4y^AP<{^Zt?rwn1nwe`%g%p#ALcj_-+o}2XoNmln<5=I3$yWEJR!g@ zd{T=M(7tPFPwYJPo(`hjwxvi2h4S^FJQHG_jn^b3^ci`Y=vCafmODayV(nLN2{eMa zTN7H_UY33%?a5s4%`~Dh1Zb%d1X2>U_QNcFnno&myn>d+C24E zG5hqG*42|$sa!{9HysAqX&L6kqIh#%Ui+E)0((4S-Gm7s!*Fmdjc1&lH_7g+$OZXD zDBFQsU9+H5+5C_i9)#BLenOmO>Xfam!$_zm=!62Ia&5$tPME7q_e4%*s^J-S3v=0j zuwK5sfH!t^^MhrN^$zQFG9=`ozAakIl+0Pk(MN5L=3sE_)kJX=jU({^U$If*HG%6S zfQ$V7Fy>pM!5$P`kiUSEnvq_~|46jqPG3a#Sl=LW`d3O%5@GweSW^;@y{g*>K+~$08LcG0SxfXNgNEj1GRsHs4BD)gb@5N; z=Vkch!t$cvIhmPspUU z@tGbT;pmPBB?+Urp)~0`B?L9&;%jgS=gBR4kIKz`CG2w`{H^k975XxPXb$D5Gu=O8P{mZ+u(KzxL7NJVKTt~ zo406UR5hK1M6su+At;R_f}RYK7{NbwY1FMX`-2i>F4Ayug2--+{^u6$HN^xuOhr#B z5$yQ+5}wD98^N_&9z&tAs5kq3kpIB5Bg`Xr>1^&xMNY1Dg#Co0iYF2Bx&b^DrQIc{% zwL3jooD2g3gW=W_xKooSSWSlMFtK|71|~G-K)5G`wx5w(?~ zhp(J#NJW5jRy+Rz0p30>C}NFKGvPO42iCSInC{5Ni}}#BY_z@G;2++HDrw7}U6m{?*=@KQ=Pxsowf)!T#Q zn*ve-OR^hGKz%0*vr=`&Xmom-SFdy;J3YtTb}Z$shHnT~?#q$|lS)MDIG$3LXEYH& zC`|GypUYXh-lqS)0E5nO(!wWW5THG}O-z&ErElS0Ti75h@y*9IiFT`7s`9@yKJN42DB1bowBH)x<7Pk2z?u! zhOM_jzh3k`PMAqVQhV!S`30w!06iB^j;EwGJZjp z&M~DWe%LniYj|#&9!PTZvI!o0feLmH;+!J~+KoY=f_`WKkJ9mYY2BV?o>n^t22oPV zyQk$z=38@~lOoAf{)_pTNsgBlYJG75qQ-3K^{i|LVaMA}+uLQ~H4vY{ z89^+L@L#^$;&}LU#VzkkG+MKHIb7bE zn{>N*_Ez}RxAY6bw?x-C+TMRQJ(|~jQkHL=Xyy5|{8ks36xQwBMt9w-4 zIFw0Z9muc8J8|PvNwhM2t~7=KzSJ*5$f#CZp~lrJJ9tPR0L4Zqbeqx&FPq2z`5Pv> z_;2%GWo6_lWsE+$LDH|{4Exe%Uc3PYV80o4Ifu>|=G(Y^%kf(1y-Ql;y52FyN+<5O zMx3&dKuTUOkN(07D;B>5U>=r~e#`dqfm!*)z6RP)byTiuC-xjd{b9PV_PFvyJa11F zgceI$Eb(_Wy`hD;R@~fd%0OJqY3?NZj#!^o+#f1DyM$SZ!i=AI9f^4ueM#UR9g(a>&+YSLbL#o87jvu@iK1LVVnG~9<7fP8F~KV7=(flc^FhA3G%3;0f%tk0ti^?R*UbIuovxr*$vLnQ1kijfQ`dmYdXGi2o+q za{KaS?@K4MhJw3wVv<+DjZa6F-pIVwe*76P(iCe}mIiFIU$(V@u`@o42{S{xHHRbr z00GAr+~4G<$j)dLdIH})`@$*W;YXbl`HquC!N-emcI-WQ5f1v-_L`ef6%nxyJg?N5 zwv8yA`d15ay5kgewbb7IwjZnWflc^yadh}qc%1an`||X$kL}NGTIdPw?DxUFM8)`(mGiq8{R>J4&m=d_fn4Q}W~vYKcgHm# z_qU4|souM9m3^~gAE0dD(GzgN@ObN={C@fmePZ0o$D#Ap z7wId^<6j!{+Vy6HdjiFBod+*j^mtS&MG!FXd+12lard)g-@b$e7P`TOPFU~T+0`kB z?GHP(aKJ%o`(6^}aDdyJ+cS|(s{@zUG;)lePbtnf)ZCkk*BCPf;$!UCY@D&U;J1Ow zwacz<9)|zq+xhQ=DmK_%dF{@V&Y)9zlW#AX_dot3XN|zlY(g%J9ecn$`N*J4U%+zW zgfb&@?v!qJr1zf)iXO7yY>S2aQPy)!)Ut}Gw8t_7RTI;F3%;k1k-wTG4=A8=-5o5h z*@P zqvKAWlJgB?K5s)?lSypInzT6y7CQOS%UdhMx2`h4RXFs(D3IHFsV5q*o`|G=pxA>L z%{0Z|?Qbi;m|TAif&cfTl)Wb^2~AaoWw!>XcBgczoq;&)V3d##>!$F5Wu)lbB8H-3 zGU)Soa&m`LImvvUSg*Tm@Snn@PUvn9KRO^n`8LDP(%Yr&l(bmpDQfn0IjP-uJ{{Bn zFo%c6?A)Kt1N|X4X0$@Msj?B?yUa`6Bo03`83vQ1m80GJ zdN))4s-46=+{N>6cQi};YlULAbIZ_4>U+^poW;VRtk3zWJ`=f>9=FB?c>qj?_eDogdf#YWw8Uk!Fj5 zhrj8$8W)!K{0sV-`C<7p_*3wg7u#N=T`i>dhn)wuzhB=WNczLC*f9e#(=Z3&39mcRNm{QX32me;~{ za*O1I`8)4NUcS6f%W^i7Kfb4npUM1iQ90(wE?=dm`oKRMIRD4-pvQ90e=X0D@9}eO z?NXf0a4h3l>}GI%;?=Z+XN^xJ1Uz`z6)qOnZyB~fal zpm{L>_9wRR+ii_hzsi%R=P$XGhx(W-Ef|{YuDr&y4sj;wjhq%YHFuCa^A4TospkFR zDa&b2cc$Z%!8t&_X0Oe_4_VbSyEhLWJhp$qXP-xm>49^g+iHuWpC}uFZi#*Rw$556 z`jqo_yOteFQ|sl|=Kb4FKQG>7_Im1-JUjPkZVfTYH(%tG^&uh1Q$*0Rns~rR@{#3(S5G_l#%r2iee^!?__SnL z<~$=S&i`C(_$4yHS?YRz%o%m%ZuI&c96Gf8^=!CN;`h;;$ZwU&3Mu;oT;184=Yg+PE_8#uq5GU zn96STh*Cf4az#ZXPwIZqnfZ#t%`%pj&Ffca&e`VoS~N_>9p#6I-cE;?$PCU3ryl)1 zuY6|qUfa31m<0vV@I{Hk77ZekQCOwR&x_t9?L3sT$>@s!g2SakZi%$s4*LvikWRYU zLpA-P=0746^a%eN)KG>r57JaQMsv|p_t%zrexq~~vUGd*hPrS|MB>)MbbO)gOH3FD zmTq@TW|50)m9q?r9u-M#@U2Vk{T`zF{0&_D_L~;{)WhGb;h7$H^bG>@$G)O2JS{Tl zzb@G)JXf`s1z26R*bKj_X%}k!S{EO_OSGqTNb}?U6I*h!ITH_s-#jX}-L>07RKs}P z!YJ7MWsCwd(K%+ky>8xpO{X>T#7$N0q`DBdCk+?fmk^Qr60T1%*j384<+I}*;eZniK zH!W0Saoz)^i029i&Sczr9D9jweXle6R_UTOH(EtX*U))#Pow_Myw}T&D)1BQG+fZF z@|vh6!=ufGCwR#EA{r!&-v6b?_5R+9p-5%uP)@t>4SH||ROQih$7NsP1eh>uVkO_=uO1Az(F3&)m?boH<{*Q%^mHG?{(OF(+7+Xubx z$-K3Gg-WXPtbgWXZ*gJiE~o2F=i#6^qw?TGvTr6F9|3kfPF7!@HogQGShjxAX+KdQ ze$dmtYY91j*8W?x-fEGHsHOwaQD4Mq){b2MeXi4P@ambD`%fN!73bJlY#ic0jQHS3 zJz|N(J->g8*WvxkLs7M>94|eqrMr)MDbG!YXyg#=<91;s?7LV`<40dMA}@PMz3{XJ zGBKYE@XzR?p?Rj+CPRs+&j$~JWEMLfR0_BHr0 zNALDy7Z-^#)4GV6<_Lw8Nh#YIJ2!%Qy4O92`&#X5cjz!>^@VYV{%+`bVSBX0KFZzEsX-+lmApZ@IDX0DKEJ&MK<>OFn-v zbqQr>su(xY3xl^SC%=@2778N^C-ovfLf$5w5{`sK_I3Q_#U;qOww+t9_v_6f1pG8G zs?rV00lJ1B*S`PlzW?3Q6Fs!%Zu!Hg)H)Tv*>h>1%8q9~+Lcz(^Zuyh+p>;JTNb8S z)BH>4;U`Iv{j&R%j_-eYB@F9-CG5(#_leez&YiW`O+F_Km)qID7TQV4-+MW_Gb`hG-uYOl zQ9Q#zUT>wSEkmf=++8lD;8w+S&q9&8@ayM^&cl)ET0#|(q>{{GoLdb(u1rC|@r1*V|1R6hsHeWulYL5Ssm-ySev{Dw zdT48V_P^rmI-U_QF>fi`y3yNS{dkxk(dC;KdAH2KAP_PuC86(rh^qB-53C_`W$umk zOD&U>=U5{rcG1E)ZDGh~p$r4hP_lu&i6Z;8eZ0-<1Xo^a9`xNU{%ac5&Gk4c zaX&mu_ZPtZPwXG8-n)OpjOH?}frU+RZOyHS!*kyg#-=X~s4A}and<7OT{9gln=z$o zj;*!q3_swxnO_^Z3Jv)EqQOidG)~%^mtD-HEqpdA9N+&H;VM2^MCGrpYv z6BSL-e$cl*@BQwBPoLbC_ZgNW)JET|`aS3WyZWK~xWtQunPVEap6rr$UH)8wh|FtbC=I&wg zdsp3hceD3V+^j9A(%t8BPN8$*o3OejF~($L7m$t}Q$GJUlma_haKH6|JEH_Fn|17z zQ`o-sF1iufTkK_63Jw<8fL2x5+RBFfP7%KZYdv`>T`o&dIMh=ZZKC?)^|PZ|kU@#m z7$+NyqC`^nyWY`j>8&?;=~wlgNETmFl!vFkq8{Jm)!JAjUFVMGk|x^x>(f1!em=cL zHV2Y+B=<|53L?CsE1y%fC7F$QjB;vrWWUC~)iYOr^>Avhyw}&;!QSWTVzU?5lX4#o zD*V?YR-R>Trm}bT9l<5Y`0VI=PQ>jMrDeTcUz4|RJxkdM-O#_XqcW%TZuH~ZWZFb+b$Cfc8D*AG@?)9EF4EFX%2dXC6O3rQfOiHeO^% z`4Pb{Lz(g?wzm57_t)2Sf!m-`76Z`S>fg@AR?NTTyQ` zAze`b;w(~m9C(dH?7XJ2U){n9*6uw2D#|#me|EvS>xruizw|cDX!L3n_V{tHKvT~L z`%XLB_boTxrQhh^`C$BJwQ9qxS^S|!`}5m3;|ow$2R=Xie(l+4T6Q4>T^h7!x`{)6P7;iy~0T}(;SCK?7McI>ufjV zAk>X>CEY;YoheuspL!|#Zt-&3>#fBvmbXT|1O`o;+c&>47VVEo{2V=dZ`UpRi|!`^ zmy-OfnBSE-Bs7qRiCvDU1^wyol-It-8d4C-6FSEMwJH-JcK&MHdWRcqV9;UNqb-H zK3FT|m;1|A^mKWno>Sv+M@4LAa@U2dG+{=yiUDK%k+<+hRMp*sAJFD8=c0apqE-7|cFct?qNi z9o*gR;Kkops`d|j2ckl=yN;KfdVEv)_Q6lw4Uc2c6>TT94lA6SIb0%`cyv>Cz_a~| z$r-)uJ3A~Lmpd?^ar$~kJ|}#_`Cat7gzPFPcY5$oz<334Dp6!pCr5iSyD;i`115X@=nC>)|X{n9%z}e`sejIyvD;e)RJeB%mrQO1zu}0OD+Bska2O@ z%Y*B!xc&~arEnekKJJaE(rcRd@EX^aZ}d=q)9lDa9oheqlOn=pl9R;#`{#`0MBKBq zI;oiiT`q6V3EfT1T^Ravd}j|3yZ^1ldhe3Gh)20Wv|lCq?`5=-BZ%{ue|h8C*z5z; zYS1`Zb!)~VMp(BRru)9@D&Pj11PE78?DC8Ku}enlzG!mB!sU)To3h$A%G%b-{rsS0 z_I0&cS#!bVpVP_^o%S{Z!70bx+PceT8Hczjr$8bvUc6{{QL9CHM2{Nv%WW=_|u@8g@Y%T^`vga>bGv)lT_>;us;MCT!j{y ze_aA5?0+Dy#3d#1OQeD=EUez^ZSOH#LQRBh=%Q-%n1jLltK~QN%42)Elk0C!CMPX} z8g2`)oQbkZ!O=LN%I@)|50vpC=A;rec2|QT6YiAFQy`Lmvr<_+g}C*qCPJ*V_Zh2k zw9)yQGMKuo;PA^j@Eu+r7-%QF|Moh(d@TEc;EUP+0B%5$zs4t!lt2Fd>yH}5U+Wy; zHT$Plp{2OI&W5b9w}n^S{nkK#^>`_^ zGl78Xnc3$>k|K}0*PWX7Q1L$PjN|V;!zvIcZC^&QN=!d9mD2xrbA$dj7JYzJ-#VSt z$@&Dt6oBcyXGFcKP%4~~(JmVpVts7&@Z z!W^yj^dX*Hf*@%A`aI4Z43AQ!kNa2J(}exwj``0tTENr%^p7+;Ccra)X^eEgq%l8E zXuqT}|C*2Gac27Wv?pgfhF{VE^#7!fhK>gCYd$&}=3nzY3E@BM{iQrw0Nt-VprfPv zr5&`-wT*`9mpW+a=>N&9M_L&ENi*#eX3Q_o($oHvFSG!9hQF2ff@G(8#L>3iMK^dBdvxe@x7O&! z&iVe8#dU!FV<#ozqwT~xu!8gwB3AX^?!@2jF!E#J+e}@`nQnhikBd(pHj>mrxUVhG zj;J8g3$(3~P)jTD$R|I)^VQM8g=$U9OGDhtE`JK!M?t+(Mq_7{F(Dz$O39JK&kwzr z7YL$hB20+}z;Uon#AFgcPUhm|DeOav6EF@J92t1`OM2L<7&i&1(jhK3pJ6#|Bg$t)gDAYKfU3(6MJlA3 zxe0`$X02?g>KrMpqHUjOZ&Xv{GB`b$3efLIjAr{Hon$=XU*6Y@REh?Fk|>l)YfX}a z_E2dw(Zy7~kAYc@ax@dOZ%3|Die$GUe1COFSV8{qz6gM9Rv}M+*Mii3RGankU=TsD zGb}(rfV$^~^fbSXHEW#MgrDF@MOpy;4td)T;|;Vyu|6w&y{x9&xT9wg9JIkj@tZIU z0(5#@bU6D6t!?E#im}e}fC^$`X{SnZMYzB`1Qc7Y~ z(&~sGuh2@-4VudGdEC|$<#f87j>JX;5G#@P5oa^e38=KRIuqY6H*=+|D?i@EOGDi8 zvb}s|7i1z04X03KKeB@#R|jz(Xd)8|w_{7CcP=gB%i3L?-xKf+Jx7F&U$N@Ple}F@ z-uE^`pS&8i-A=J_F$l{HM;SH-a(_V%kmd+62*#oq$@&T3XGl0v7ILpkos-llur_Ho zpC;;h1M8c{h~XMdVDHDyrlS*#pqW9;lY+{1vu;)QzXq>j?cUUVi>z4MMm&%EAv+~6 zs<_Ii-5G?jzXuHX17!O^ab6oMNX=u;p+r*uriwrBoynu%2)WbYmq~mutAF5r)hw|R z8XVcs?2@mb?rj`W=NV7t^Ip9~F|Ud7 zu}TS*gQdc+0aSe0sSMEF07#e0=v zwU7s&Y}wFu1I)?w)XsF;h%+^;HgH;ab)Uw{2c!lBJu;e{&A z=2uC&I>5?jI3kfa?%+kTnn|W1TswAO=lp|)YyV_5D{y5Y2k?(l1#p$bIdkw+)XV?i zbU7?hkUVR%_ER}BpNpfnn^s||SWO^msMQxEq?0TpEl-_9O|I&QR-0MeIlgBz_JSHB z*-3Dll}J*RP?d~K=YKi0;*ci7GUpG^Pxk_y2eaPa7mR9BDcD=j%)~hl8(iK+3L;ue z^4S12d?TdTtC$U-YG!oxYK_@E;x*{5-SJ15)oUpW?wDCjdbf>o#U6dhmoavqewgOy zWiJ;|)XCUV7t#{P1zyCYNZx>6#;^gzy>Uzm(E5heNsuY<5PvTX8Tqb*3QRs!xw|2Y zc#m&NJ_MGZl6i<>L)@N`!;!iZHhclJr7%CW96h)_kf&_O@1&SfB`R=u>+%@XSOVIx zilx5E(d~3^eL2?I!~XmI>7ggMfpC(ChU-nZsxQl&KFg(thHd<&)$F+-pS_#e>Cur% zwBfga#kf!Mw|`8l!x!DWu55S2-kyXA)~~PGdn)l9*$D02T3W*k9LttwA7^z?MPjG- zfQ|<$t#ADXt;IEd+m!}LuN8T8Z$DgzBts7EMM#@k@j|`yUW=ICivg=z`344>Op3A# zgsMH@+nxy&!sU*l*-XxgKtU6fqH6{wQkJPPRc`nes(-B$Gr;xpC3VfZxa;L85)mJf zIt9?7%xTH~t5gY^h&NR+L{Ma$O40H~C;S(#OU`C5Jm$6gN04q;gn13rP!`!?pr_mNB^6GUT0S;4G9BhF6opq=nGaRp%CnWNRdo&5nq{|DH)RJ~n zl8i={$YYb5q`u2avAHu%)06qHMLc7Oq+QI%zOOto5oP4sPkP*v;y*(_rv5)hUZ<{KNHoM;09G@S(`CN}j}xPP-8i;GpM8kJyt!T<7bi$Mj9d65P4{95~Id8wR>qei8hiF!x729w#A z0Dp|E_5I+Bnvb4&x-}kHZF~?ldrGrM^DgB04`y{u9J#G@i{F^o<5CO>KncH1Utg;W zFcN;>dZ0iQ%iz2P8iCSf-*Iu=Na7cq1nc-8B{B6?)pH-jE~=>t)k=WXcAETU0V=CQ8> z-U#WK)cF*l&$0EiQOze^!^WFlvV7VL^mV;Nw%r>l?4v$Ff7r4h!5K}(BF$bByMM*0 zcFMSSy7(G><7DG>bTXD)~ovn=Eg@FPvPm zXt&?7_UnnON!2KeG8J_ea`mIwbT$bJK+nt_9X3&$mU5=4jRWeworKNdWrD3X9t3|2 z{uDiE2dp9f0P;`>jQ&4x$EIVTW&9tyV>A7#NV3rWXC;y$q!GG}28_=ow|@@qirMor zFcb0O-YkF;Wr<@i@SCZoZi+ua)dq4!uBEJ)j6}WzlyMFd7TYe-@OS!tXDakFSgUiS znk1koX~IZnUILhn;6Xk(<769fRlB%W{=ytPa?eo=U-_v|_5k_OY>9XcA0t*Ml=Xw;{4xy}~r zDtNZIeKqex@7$z`eZXYNp=ZulzoMyZ2@kCERsxES>=hw?P(0+;$$vqt3W{vZsl|GC zpf$B4FGa~>_QN+uEAq~wNJHC0z+z}-Y(<{cN;(~K;3vq#cvH#&Lny4p0oR*L^NASgi=eMfKJ%6ty50$|LTW>_2L25n)+rwD@-kzAyOG6z=A4=6NTXYFowD90Y zeYNyKExJjIro{R-IjzX)!^|7hri^bz`u?Tw0kl3$3GnEeZyl^*&(HIij~1gMOck`o zQqZ$2X@(;2INr$GjM)T4-Vtb6n`n(CY@?u1enKnO6B|F&Dt|PkgxU|~b{IIHaxVEs z19$ulXqB5&_nFs)G|$Ko!*9H-Wsnv%=c5H%+(A^;%;vfx+(|H=;&*v4Gn=RdN5;{K zULcai-^!`Ui4&>sj}Nm`r4x7Wr8Ch*iz4qb6*i5&94m%LolIW;tf{(J^g=12jCXVo z?VQh-qIWZDV1L=GDw@4rzv#7)xyy*DSCuhOPor0W=sY)Ea(#smPe%OYX89@hDgiPd zWu0~=$3w`;IalwTXl!3oq{-@QD|?@*y-;M`<~Dw30Y|olK4Q$yI*WPGYiL;MrV19{ zd$t!RtX{@bM&7p~^P z@q^s^SA8yM*n8hVVLt0j{NC@SjK2Za007X-(MXLkyn@&FBZk-(h=U;WQ|00eh#dq$ z7<^edBsSzpp(86x822)Qk&eN8@(o)0OUu`M97*tWB=8v-!?gkN0A#4 z?5t&CQ5gC8#e9k5V)ps5OC04{TbyTjPk;Z+=%)XT6@ouf+!_QA%;z#Sllr7^Q-NbN zTSiIjODptn6Ct^>?>5lnD+Iw4c;3UnSOgq8SL$6l*IGKZ6V~|zOUXKO)lR)mfR@T< zwzq?9>bwp$GVf9{3XjoQR;XkcZ1r^0oHG(3hav(>;FN61eI4dW>f7bBntm6H|59+w3b&j~Al%YOkMR9)H6#%me(63Lwc61&jwZ=;X^u^B#4Pt2HmLhH>uN zqai9mBH{$Ee-Z&J!7+h=piIAaQCSyP|Ff)Eck4wJEJESsXKehKU`Q>$NXwa`4fE76 zln49RQAN;W#;YJD^*3`JVlxWJ22_h6I!4J|EdAyoY6#__cEx$Jih?v5P=7wQHJvfI zHgef#{b6+5yUdX!slmq=ixl=O?l1?CjG=f)J6CU=y%@bS51o!*p<8o5s4ZV%^Ed7i z;bMRz&q!;3N#63&XtuY-t|o=I;YsACLckl|dz&h!uar@4u4pDsZR(`2`sS%K2ho)ua*EMl1YiV*8m8|;64aK(Z-Z+ z5EZAFd}s~k`D(s?;N5}yiB|@9#2Gfg_Ar+*5`h&f8SqYvNhhq0pZ0a~o^37%C^5@z z2ZWCehTfsP5J_Mme%L-Bj4bu7OPV7x#Ze!l3P}{lChnSyOkjUbT7S&IJF|fp2v+@Z zU<&y9&$G9hR&boQGfGpt{;nW`ou}nsFTe5<)&K;6JD8uw@rbxJv43u5%pW5YVcOR z{k7^PD(tzgR4U2|P63U0 z;b;b)5j*Z?ndwC6Fg;G0$*2GT-r{*JH24`GaM@S3e1Eg7NqYy_%`|(aPoGoLe`i#P zOA7}9(LgosAl-1`sYf^E@h0%@Jj@Qj__6?lmqta$eN=>mrS-v~;JAv!vtjy<;hs6) zsN@{gKmYJ?2VYkDN+zF5n@24GV>AQqgd@WQUa7qKD+@(df=vr3j&B_>AmgQun=oB{ z{&p-Lf`0^*9VU!&c1N9wcIUeCID6=l`K^Gy9v+@QrO`;)#PV5_>5N0@;hr- zXggdRE*S8G95*-w#DXMowEx9J=o>m$%^jF8J|K`aP69X}vBc^~il8DoOH$!n&1=?_ z{YxOQmLwhvV{*xHlMY6a!tcqJvf-T`U<`;O@}0(YQZ6BNZVA|br4sOrA`PJbjn#r* z=6?@Gx>eTlgV0)h#J8%#CoM-MNo;!fGq|Ryr+<7GB^DpB$!_t$)H|m3WbRX_c8!c^~P~O+DQBBdZ>}U26AI zc_J6>w6w##$j?Kl0_j6r_`*nL_~Ly-uy9A#sk6C#rfJsuy=%-D02mtUwg4g(td{YW z7jbTCyZoU+-^sb8mALNurtRBTX#0gMk+)xmHCwWGCs>Ly_q^f?YFc z9b0)rm8pE)jG(+;Vf72nS~9ij#Q}=wp-E09Me^tkX95sfFr zc~S2-*n|60zq3hhXH*_2ZOM)@ReuF5E!)oUT1o)4halRQo*O!yrfEf7l3gIE$iPqI z**EsxXoY8+)7m9jnX1&R`0ztf8=trcTv;o6qibpa-)+@#4)Kl=9^Ss)G1q!YRrrPL zS_bFqRQSB~equ)WTXGX@;{^pDH8_z)PO3aQiRES><+bM6PaF%!@38GzPk*0j>@zz6 zhTmu_fD!Px7ojPaqoHGzreUOirH$~k2}lR~b2q|OXkC{+V59l?MOQcR0t6L??wQ)3 z(f5CA|3OzP^|=3_e+lo0{J0a+fBj=8)KU;w&qIYG1l2Y0vJ%R(iYVDDz`*k(uGyPz zTx7oTgg`VttmK!GgP%o!UVjiRBKJQ-&oeqOGvGJY)FBplPZ}_gu~gYR0-rwoODoJ^ z#|L?u@z>a&oYV1`y&Kw#*E9zjY!QxSH>P`%;lcAAMa1J8Af2bGTBceIKQ|t1i%ti= z8y-bTfo;S>TYiSLXEaRu-`E*F%c{6H@>D2d6re+|#^Wp@@Kf7kUw^--H-IL2rrKu| zHJ0Dt2t(+fueLOSX?i}oWs>?(M3jdOQD9MPJ^mLsl0Z}ScH+^A5xk}ryaIQ<{A-AoL5_=o4Rr4I z7|j{rH-YUu<+C>oV`SXwyNzOPzZx>ykV6(DVClm+$7G?UzhOmMIm+C$&iyiS&1-qC zao(!E%#sxNp??4iY6(!(v!S8?KD&3RXx5$l@V%Cil7HwRcx_6fJ{S0sEX%?t zj(0c^tQXI;^%)%jK=&JChC#Z~teXbvkO2gR2G^Zvcmr1`veqMsZJ|JDvI*zCI$e$` zm0kZ^C9TsOUb~=N*Hr!7JDBr`jgQ##cnz{MaW1IHae+=ESmlWnmzI!>EiV$`Z6Lp1 zf}N0iO1JT)n}2`3gl$lQ=R!xkL)=aL6sM^D3Od-gcphn8ewj?M?$eb$VLd3gHLcRF zEDV1TS#6su8pV91a`J(-62ybK+beV}U*^NgV7|{zZ9rm#_qv{@qns*-`}_Qi_I}KJ zW1hjDNyCIMxn-F@an@=BWPn==w_%?)Z8+`R=PdB)If82=6x988D)cxmic!!G4-Lqv`~X;zUYc= z&PN3X$bZ42xesST*BAKX$ow*I84{w%sO@5ilw?;}UE{i7ocy-Tbs34?@>55b5nYZ! ze02-lM!Z~}j;rLQk2hp-hXlMW#K4*qWlXeui_nn1)CstOHnRni_s1?*!5z!ioUOsN z?uA8#YEM82#)lFSfBjHm9iFBhjKSIYl@~EG3V&vs;X(2HsSR|gY&c1zxkbymdU62_ z5FBPyViSRyVmTE&oDl=J_-LClrDtQU`aMP!K?4-mMPQdzqD)vutBGaKDD@VJ?A)&` zqJKuy`JD-sKjdH2hdc2Fr*urmN-OjmBJ(ad*x7k?k6EXR|Lxk<7QdmMD_`jyQrG@m#|U?L>y zdP?^v*Xx zuhbS;H|RyY!d`T&l7^h$ye_;`XL^Qj&u9#^48QZ@TapE)lLl(>JA^x~r=4QWN;Lj% z#JxxHZaNk)T)JalMV1Z-4DAQuUd+JwDx5p|P%aAZL;`1?ZW;=@#@Ua7qSSas3;yMI8(GAiYEyDyL4yeCwvgwiQ5Zm{Ig0%quTM%q57 z(Fu$aOnWbS@Dc>!YU0}Ci}|s=+e^)E8ntJd_l!zH%kVpIa>56pADhQ}oahA!JT_Xv zE*X^WZ~I_Elm`O!k*1$y9@#rGgjY=S@T1Nek(^{PCm}&?gD}gR0x2e7Nq<8zC_r+y z=;zqI7CrV_N0#C#kd%nNWhFCuKD1$WzpT7NB#9epo9O&lG(DKUBH3k$z5y?!k0?{t ztD|+r))@%cHRPR{Y~2CYNoYn_-;cAI)HIeHX`ZoArhi7o`JLtg7^@!j2pW1C&?o^v zuDHlRo*+CFp%-04TVCMXo_|WOxzID2K}(4J$-T*;U#wL@KeJ5JKcm9XGX93-@JFz~ z1JQy3+cbNT2&sBq77bRn-#iq6&Io4iy`sl#Ko%cxMl|HyD;LOU7&F~DIQf95nsJo+eUU|BYFVr&hE`Mg zxOZt3?G3726PtI4@Rd6Mk}~blJs(Xu+6A};KQV|;V0poO%c!Q`&?UqybMH53F)yZr zXBzs9(gXnfhE`1}BWm4^3$8cGK;__W9{WRn|Du1-t#~Lsg5nLBV>FdHru}Ry--sN@t=_@w&bSlYowF%0#(TNj6%k$ zrWRq}6-CB;%~b`02qMx7@Y&O|`-g}j*29yrt=|G^?I)JlT4TJo%1a$$ND8h62nf1l%zJtDH+-(Av9(CjP^)KO0*~d4)9oz zzzejM721su6MwY97vVKzrDYSH(e7_gzkFqLzkf+#<7)Fvw>e$~lo7`BqpM+;VGSg3yd4Mx(hwMy)g<+A>A7nA5qbf^$h!&a)DJlsF{QeTiI& zI7(S6I0{!w2>~pPz@eBxjf6indbZ1&Zz+=YK>@555r3n}Tlj^y@f9d`4ggnbhpVSi zn@$GEi*U}y$O*q&Pd{j6*45a5Y*qXADbRV)teDK(}zZn(%dc!bcIl zsQ3f7a-K84wkUQ^LE3)LL&iX}Oc3H7>>Q$@E=1O{-d9cfiPP_QH9N5V$wCw$EAz2U zPgM|NvwyLQH_+y3Rt9NgwKLYrK&G~}^}N1pZ7`H}1RP06t94p_i5qqR8#lbiKpyz4 zgB0Z{m{$%DZN%Ib(3Yys+^zl%fd!^5Kqn=U0!qh_IL|pOux%{0GDoYY7hC!K8?4-( ztN$|{@tk+#Z|F@w<{-~bMBWnm7g*asGS8I!+<&jKfZtI#!bM?#@xb^llQYBGdU1BF z==OYV9*#Uy%n@`reC0mWdu4D?nE<6A;ozawe61~r3*%SflyjDmhwcCDU(_>*x0FtC z;wAjn4^JxiBYGsIo_Ve3A|rG~MXYR2SXpouP54EXDH66urV74Y)I0C4x714;4Oo3S zGk=XMnZYd{*)M6qe{BuJv+s$1OJG^dA61lt?+|Xdp5}|k^N+gzT`P3KJR*>ltxOD? zvYh>(zpq^Nq3v~5>;gQ~rh}eu7kK|C_#+uVq5LL!yy@cvy3$$(1K>WHoQ}=xO{;#d zcbz$TkiFM6<)vGFSmD@;NCX*tZFJpTyMO3$BVWq&Rovr@+QJ=ulF4T%dlf-sFqWB* zf-WHunSTTL8 zFwhwu8rWGO$&4$nwlm+EA%ViIS9Yq`e?SsJF^Rf-Ss=FiVeg^zYjw-)EDqL@XU{X8 z^PK+pJ1Z%rw6HZ0H(28+`I(P?-hZl2Jh}ho!z|bU=xgX;E?edG^pG@M3WcVv$WMly zs{mTBRSw&*O)fcHVI3)k4EwLnXV`OAAHj#BXUd@Ti(v0U$d-1y*J#lUYG?JP z-W;lxawKr~7pZm;A)ts&YY`a0yi1^?sL|K1f;_q5GTeXKA*b1$G;7YqIK95w6tHm{Qo6HqeFvnWsS@xE z51-K`nSVnCe=?l$q=Djdk$+r+TW_5Q#efM0er_vn*2FbN>++zvB8angBAnba&AdBxK0MKadWGXy@P4gMB^ z;`MNk1Riks3bus{HNo~u`+Qe9^r4TFNP`;yHQ9XWwH8W^ca=$v=6_QS3H$a|$aano z+5m|Vp|h0Yq%axD%!UYIUP&ygl6&y1_{hQ=yW!n(7$(Fs1U{qA(f-b>WRCFmM{OL8 z_xKv(fh&8CEW=eY&)EOVLn(o1Bc$^yoei(|Du0Zy_43^txW*`bx3M0c1|maGmm*oCrCck7E-VI+eX{-Jtfn{h zX$1yz^ij+7l~Fcz(5)_`va=6)pPB}*L7E+??w@JZb3doj{+6W)3k(n*)Zr!4Jr`jJ zwWDMv>F|B)aLBkkcDt%yRSaIjsZ4Y}-j*z*Nz0aiIfH7=VV8BPKg8GI?F8a#lcG^@4;Szw zAe)VnWHYor?x$DckCyvnb9&-XKfHb^&x|>{J9`q#AxS)Li$pi|iD3{G9DyI7{W>gn z8sR{heu55us(%Nr$QY=J?K_Rcix&u1gRgYcL}Ds8yDZAjDqkzy+`mi{l#6@D_~?00 z;{TyD^Aq!|ZESfBwXC1KNX4{%#_1Rse)YBf>Gkn%zS57Y=zrmw{96vmPg%sZY^{xc zIBh(gGx)=G<6ki~2nywo~}QWAb+U&jP&*Ote=iPeDYQV(BV?s zSZe9${qUK5IuY31R8QabXY|Lpzfa!B)P_!$hI;0Y$$HjC7P`2P$_TZyp0x!owS~DJ z1huWhW6aj@>8Ms}eG9usS<`I z9w~aP+49M=7?=97+Q$yFeY|*_Y#bn{9dW6h{ulv2j$O1gb$VpilMnC@KWv*nJ#_!( zSo5nR>i-vf&HwT&rv=c{J?CZo2*`Q!dm{y+WQ0RX1|wr^yI`&hs*F-`O~3yOpmK79SZ}*lYczpqsad09MAM9=}gQ_KfL3Ad_Ud);rvef z!+%Zwsmv!Ic}C_x(w@G5GzP!~@$~(J0GdA<_B#&rPvIHod5AyqFg)u#&-}=wr|%zu zj^QsxJbnLQ_EX(Y4Em!ZAfD=EczosuQyBghe#p;{TAnUwn11%(Q`7#|k57T&j{w2& zv(Nr2>W|1HfqxW12l(;W52|Qs{}O|Lc7Nd$w;29j0zDJ$56plcvcO3D@1#B@{t%|e z!0>0=o=Tzp!E1;g_kI?@!pQPhcoKv^WZ>8EgV!v~KV;%>;fFl@J^YZBpEB{!@I=>- z_~WyG4?pCBhT*Xff6Bl!frXjw54oUa__y$l6epYFO z3;5}QX@U#*S*0m1;HR7AUz0uHXMfF)5B{w6#{vZSS*ryu;AaD$PAsPXS*a!Nk9XXU z<@{?gr2kp(-`B*S1pK)k{-nYhm;PtNZE)#-uEtM`ApK7QelE2?DR?aACk213oeV!| zcv?;wem30!_lLeO^R%2X{H)Xom*Ho9Kh|IdhJUPM|Afu>=l%axuo;ip0Du2=X;k$8 zZTK30Lsa;GfyTh__aX5APc(+d#pXFQ1_q}8Ni+rq0NpPL0}M2etJ6QzXn#d4V4(XI zv4DZ@S2P9&!0)0l0RBk}%`=D!&!aK0(EWmT0AOJJ1yuq7VE6?AfsXl6L;Ob?J>xG( z36FI!{z40YmhPXtdIFXBHGdxy%`f@rX#oGMmxi8}=@-7x189GJmXYO`G=^XMmxkf# z#p^%Q80r7nM>LF2y752L=mEdxV_^A(_e?ajzvN?jvZwoJ8sOKqF){qwHYUbjP%mhh zm{@-44<;6dUua>bW%=bDPK$SZIHd ztEbWROWKp>_|G(!U-Qw@{DS=Pga-1H4SAfRnEua04f#9%hpwKE#UuH4=DPno&wqd1{L$0@4gZ7j z$NBH{{}=z`f6M3p;(vcUi~m8-#6-maV5DParl)75fAYhl0?_JzV#!heIoRlZ=$na zn^&8OmUM5fYky8zRqdp5J*`~-G|!-;4j#)h=0}d<6UUYKl8;0T>~-UtuTen^T~{yZ zMH|47fVx>++F`xCUxKc}`qSYH;6%NILE$@-ArkKv#g~Z4ePc$dDYd7N`2xnoyQ|$~ zgDwa$M_9SO&0>GjNGcZgv>24Ejno(NtpKDe1Qo^_%YO{qW9b}SaL&E2aARv;sKlbw z5!6xR*ECS}EzJxD3I$8CvB=BZg}HS|_K#XxF*l+6RkZiLfv#0nM62GYggtZ}Abv-d zdUa;zhLab6$ndT}m*^xe@6LhKZNxpf0;OoihYTc35zB5@F1^?L;j>>kNT}B39Kooy$6-<{>9> z>g9V=`Y{nAy@Gka9WkaakzO|AIUGY5dVQegOokDjdgU`pfXt%{DrX(ZBfYXj%-y6( z=YIgzWva=<nWBj%zm)MW!fWGOiG_&&58Z~1kA@3jQ@LO}NW$bSsS zkDmm5|A5PF_$A_vXKuhjZ*Vri{ew?(|Az>}`=&#inkJp9jf`dI4OHFv61%tYOL`E_ ziyjnpIC-9+dMjbFCe(%YI?3|044KwCC+a*Q<@pO_Lt zMPFxsOp6PzUMIVk-}Ly{b#+dQp9Jkl0A^Sg5$l`zGMLw~exlgjO`_6ArGGvv?_Zha z3awxOf1~nzGampCE{O)pCRB{WMKO;}Wa%lu8VE!vT@(#Xu7xE*@bQIY`Tfh@HBdz? zZ^*b;g`a)4uP}yy6=cEWe=&y`~U;FeIopC0Q!6)89H>Vdt^wkzR zY3(-L%#qx`F#KdJ57gYj3;9{oo7oVS-!cRsve1^2We&g4(lo)=R_AFoS_tC4S~C!v zJ&`3Ex)wK%>YL=VU=S?8*ZbBISsGeOG`!ULe#iaO3qD1(x5qkqVSmU{R`4|sJ}?l5 zT}l+jh*PZvO&iE?uQj!ynq4Qym1{gFX&HST<11K{nY(HG?k8DrlgmGP6=JwWk@4oo zvM|hO2<~QOgw2!y!{}L(4q(6DFZ1*1Jy)d7`{r&o1>IRv7v@CYx9>}Cust}7-P=d6&5Nsx^goG^@bqGn?qBEC1AhEA@}Jh(WqB7z zx#?<++q;X@-hRRaVPZVIeqDIf2zYhy&(Qp6G)-i{8T6jxVsr^Ug4o0_`9plri#6*x zrQX8flyGcG2e>H`W^MVsX zqzU`RTD2V|Q=SD9NJu>0h~lCYQbzC@*hY0o^gccl6YDlx4^2y~RT-y@g4*YVmA;SP z5E{xe>VI<{jEo8Bid}md8Q(LlpwEAQxv_!ytx!YQdZBFmJ>oVM6w!NC@YM{QI_AJ=T^_$c+~WcCPl!a zFDH{1;8#a$X=?TyAOd9SY{9A8@Su5NW2s%Lvv1RW(qbl4pY`r zg5?CfPEqjhW_fRlD4S3etWK-a6B^?4}k=7PB zmUb$t0G5D1DO^Hk*KP(1^UlYQ#H?NM)3kOW&IJlF6jbnCF@fI??NUdA zRRy9$gx7h$zBIUdlfsukSK$2xjx401!2g?j8fT_Hl}^YP>E<58X;J?@sOAyt%zx<) z=eIUA0_WE61SKboONP+2^40xU6G(J=K=u)8Fk-KEUO~F;0wmLp zV`--qS;`Mk>|5IQ(cO#{LZ{F>*MDK>*mHczZNx&@ZL%F>p?HIID*Qfvfz}G|;gxqJaw@{9JKa9vz3Bnvj5PT)wF-I+q-zR323YQKMSDdm+qBbI|()%YVU^<4-!r>iXnS~LbGf4nysJ;V8wgF<$vK}lrSVS z$oAfzgr`IT4c&g*f^0N|?4yPmhg*1O6j9y!+we zT`2kK*R{^IFi92DDja$t-*r#-I**|~Zd?w~Mm^Y(*sx;eV)|QeGh+@6StSf)EFoKL z3V~slUuwSP|Pq*JBrsZDyMUGyu~$ae04r|FX6@0}dc<$KC6eTFfGdd@ zNSANsqqTRe)~0vlEL^ML=t`v+-Zf;EG(_@Ncc=wO8z`!hTbADXIC6RfR zs_P&SXN%AASy=KL>whF5a;JGt!LjgIF3m!yoVxo)$#?(uqs!N;vZT}I>y1;e&hhSoY4@g;w7jZh zeqCkFwrdK=2l1RHB}GAeXRirtu7Vd=m~)&PC8v(6B>8l%M}J_$O6kmnQg1*9?B)e$ z`WK^CTIf9{Y^^k`!y6Y`K5@1t^(p1q1T;CN(Yq;sT>93Hkdovr+4s@y&MmxG8d(f? zGScY%fwhaz@!kj7`47JIiRRni7I3M3i-q=!B6}VG#=ch&pKPI@C8B*4VgS*zxz3JFDj zBbs@{Py#rtC~?c1BM&1GvBRacay?Zr1e^2cZA;3V^M|H znJe6|hoB9&*T*JzA{+7O$41HHSCXL`hIfLP$1TwrE1~NtP)*+*xtg)MRV@6bRIHlN zI)5-oc=tn@)!!W{l;&;NmMWxNWX%T4zLS*7^SW|+<=%t*E_j2m)Pv4v7W*WB)^Dxr zle?WZu=OPJ3VIo@4k z&!BYEW7*BwR?HaG#!cx~yVqpItWJ&oB1?bB_odv1K1VV8`m^Ougv-in#bL*0-7c z$gKO4p@r$yL*du#9tVaC!@J->68e@(HeokG9S3-O|E=13snxIzD|;t)&t6AoihnRc zr*Gl<8*rg&unq-k=&%kkpZpt7ERzoFNU~W6#^K`F7_is#Ht<3%Lc3w?m*BuIH@NNH zXaYdl=SDJ*zFEDKyfj(rwX37Xx^%XZ1bZkd7vD%icQ7%Xp3AhFW#ZYV3bpXD%IbKKq#M^Q}`7DqwfJ3bYJ z!GMkdA;z!V=n#izI8@3**f0)_Y! zgATcTPYQzQmo(R%L#>B3k~!^CdIpe`jB|P*1jsAORI@0;N{uLVFu!Qzet&%*n)8z4 z>&8NN!u|p?a;3Hvu?UJ@;RG`>YWoG@L9a7V}<9s@zxUeos3n~uq-_li0T)56BebR`l1Z$Oa0 z9q(lkrr+;%N`W;`pM=Ew`Gh{sHiLg2H#%}S zUck<$jN-n{^n`Db8MIygwqbGDg6M-2JST#Y`@BJZ;pLK@82r{z3PgGM*AZf;_yyv` z6~weKl6dfrCg6KQhSB=?LuIxmb@a*TcVvZ_=mS0wqG+I*cNs^re%q7e(%?9xaO<7{ zE9K%L_YdQ{I4NzbB}yOF{*bvLW-z`+M)IYS!Y`CG=o@H3a8FTdDQ~T%7RM zqKjB1yj4d-5_+-E>C1o1^PG=CpOP)OIBlzb>qvy(JY9xgUYM6#@^5Bolsp!uy8kPW znEClKT;VUvk$*d4hVdWo(*H{<%>Vj`ng6OuzkSXO{hu}f|2k*pr)|ss@JNpT+V<~x zivDW@KqeYoO8Q4u(6TT+{(1UU_ z;W`-CZ$CLKvFaOEnXLEvkbWv3U3LhQ5KpUjyl3Q@vbl0!ikZK*IgkB<&r_v-bw5r9 zOBF%{$gBVxH~E@?GLpjRqFxGW0I%I{P~o6_?^Vy&9jyp?GS7PSp?}*K#tt{Ec7&+w za_X$$8e}26Rs>vl^2F_UZ%%VD_e-0>J5Ca@wC0nwxLJbmwMx{7ZXkZy)7J0b!>Qq| z5jrH1M@BpxOPXX8{728PFxf~vE~Yk*EqZRm!KgVnY7N^v1V=fX-{GM#T-~oPUCZuV zC_DsBs|QdgNdOlY5r0|=Ilk&|t5MIUp6O=uJ|hGQpf~L6qwTuqy6<2iMBED!G$F^s zVi=DeNo4hafm8e_c_l%2j2?+Ac;@44>B%hYs98QUJ9LmfgVJvio_!&5VD-(pWaE12 zy!^o-Y)1CZFfZ?DcGiT0XvV_rac&!-Nl=_SX7%Kx-ZNtW1IujVlWe3>_n$&>0U}RN_lMp`}H=f zx5B0v%hxYtHxgLg7jYqp@eHXuz>F%1RQfsBY0dhh?yj$+`)}^9 zD|aH!qjz>jaS!q~N0mHh_hffA?7rTC_T3oC2A6S~-G5Ih>3%=EW0^C!nd+aTBlnQt zRC?fYmoBQy6nD0@zwg-7{hq9pnPaGYZ(8AgpUE<3qqUZ7Q;RucJccSi!{y(I>~M>Ofg_ANqdgv<*segJEKgx@r3Qj?}pAvpyz}j5Qd9nHiEW7#av&U+<7}OckWtejwi@9Oi+@90@ zdBaqEgvmh}XS$9d%BEWx8TlIFVhlPS-Yb{-V$cZsaY+rIS-HbievRN`7wyp9Sf;bV z-QD<=@#EYucd#JSyaQkQN~jueoGc)RE1Ya8YkvY;WDu@sBH^LBCcr+6uQy~sT33fa zcPEE!C2Nv8ZZ_|$u7wZq!oag&ECG^Ju5tacj^TpXgT#3V(!I|3uFb;8^`TI_U&8P< z`k`d_;@SjcTlmtbvO%?%A#ZgZ&iZ?WbX#^Ooiol|4-ASu%)eKiOBSb9-NR8%mCq0q zTYpsDXPj#ueApIG`Ft}g*k<7|b+Ji53yt_eP+KKZ90txO;iG&yXk4nI50VgIkKzm! z=oLltqW-WkISj?TqPaFTi6*jMKJq8G^W5c-b%`%jxcq`>Vrk!G3rCL-=a>2va?2bh^Mjs)+>&rkF_ENk$?hQnSW0t zTYJyk)G0oNFzG8r$0f<0yfZ`w8=pcHV3RB&CqtK;BgIsTjM4gRhKI|b2JUur=F zcy4kU;j(yS!ng3c1DVhBKBZ&gAz5Vg3ER~r2W?orwkznL#qT2ol*@}?2Im^?dPJ+ay^jOqigM5-a63|Y>7O>wZKqN`D%D@Ov`?W}tAOaM-5Lrs-)m!%csx32&O1RBBa;xOM4X z)CphdW?cXumZ^H9xlPx6!#sg$SQ15Z@-Tt%87+Y^h0Yf=X&oe#@VdF~ZpQ6YaJ$Xi z83H#O4D?b)R_cfE7370kBMpJf)VUg1mgA;Y>Jq#@awNCvF-wlQ*qd>GNl&L+eTgST?vW;Ge1UmbJjLuOx9M&aO2J?Z7ws|VNKug#ngSL$h`g_9CUaWP|C_0z=TcPe8a`}?NlA<}qoVwQaTGfWGTaRljw;k#oBVK}% zTC7=ey4m2bY;xtPB7xnMphm#67?$TZP26bh=B0u%Zq3UH&41ZP%zFHK=Y_H4NDc%z zmiN?6s}33ywbW{v7HdTyXH9bW#e6M|>O?ApXjvdT^{|$ylgyIIOeI`FhSxI6^=g`~ z$keysC~g`AuDCSZHHi+UUQzF0Ev*c{ru3N)vG}VcOTR3f_-MMThn8@wC(`fAR z$YmW|Be@1ul$d~GcNq^cZt;virEBpl$qC+rZ;)@hVQB~oM2*EA|Gr|HAD3fXqe?X0 zfSkqmDlh|iXOFQ(sy)8P?*;yz0tTE7A`4=*H;iT9M1MA?6;)iq$2A^fOV(tIOxe}E z&ZOGoNEE1ItOZ+rSP3bz3X3l0DARKie_uf^!|dSxuI`V?N3Ju~-fJaJ2n!}oCXYjvoyV7dW(%JxYmq z-i1<{pHMp%VvF{kzZ*4Am34s~zW-ySM=xZ~Wty{N{~ox8?C58gj;m=7mkV~jv|Xk~ z<2l~g`C2zi*_&C50P<3`3saw*@cCWaN|1vx->bv9i5ZwMt003_KNc-yZrJcdMxV4TO0< zxqm!ZVX{f5ueO-900hhlv3~S8y7Rq40pM+u^0^$_pqUJVlDSHjc{og9_G>=)&H_HR zqykMvw$|j7sztJFFsB{UItpMFp$mohl%{qkG$*6NDA9;SM-D;bjJAZJn`K8Qvrr9!IZCN}Fcm#!VSjjGYQmJW5 zjPRLK_a9S8bdKX9zC!Rq)F!Wv+SklsrE~PJTu{b-;M+{(Giky`L&ib`x1xm$%y%S} zMiHNGoO2y_Qc8Zs)e5SEA%9d9 zHm;_0rN)c_xiH+-dDA5i^E>R&O){PP7UW6?^&}=b?WxhO$8Nm=EROp{J@3T&f)C+6 z!GqQv;eDrr##amR?i4WxKFD`fnB{MIi+IF~T#dz0DT%bEdtmAlV?cZ5*wVn%qw@s> zdO-N2udalW0uiGcms4H4J&b$F>VNPKi%$sP1o6Z9nvs^@WPP!kZEUpy(w^?kZe{BW zyNVc6CnWF`c7-=HfWZHL-A76ACf*$s`wFsEAVoteo$8+M!Fm(fSrns%HUPX@`m)ZY zV570ru*@i^=BrC)PNBUi|6Pz%<5GZit`;MjsQI_s4k6zRg}V4jh>dR4cz^zAuMTb0 za>3V?WqqlvfwqHI0q>mw=mb-j6qBeQIqArcJPA_qRaC8B3bDNnw8|Zn%_^8BxLx0U z&ANMSkm{5=J86!Q=BQ0jD|DKZs~0YJ_*tQ(+LaQ^;=1VYE5oO39(U34J0}|lY^VA= z7RkfFyp>zPez(^QBn6t&8-D@k+fK95g-Z-1svC-VY6TzmHD!bnzj>j=UNkB$Qv#PZ zk31|}zAio?u+)ec#=YI;UXx=-F#Uetc=;-LcVAXAC9491R+rW9d44cCi@Wr!Ddvq>89Z+%poRExIW!2Ug%NGglz$YZ^Tyt3D_c#X zNVigkUMs;2>Kx25=88cKSYC&4=_QtV;2t zS$Z(veg#R}3&M92#?BzE-D{y=Va`^L-TDWCA?a92!;$5_2&i#3d?;ORa-uvIU2PN~ zvuN&FH7KCO;Z%~p*?$~?G*wNr^#D_q`T{s#ZhB;F=%InqPNBM}clRYJZUEwA^-7LT*76Bq8~Q zz1B^P84WlhF0~Tvz5B7htv)5J&vL>s_I(MIReUdk03KB zqc5&a)k8BBuY|?p-L>qeh83{tm#>&M9WK2-=kCFhZ(X)X!wsFZE@rvlF+$%Qj8;mG z%G)*qasD8!=@h^&6rAI~hUbK*QbZw-GqG(9t0G!NbbldJ03|=TWg2_`Q7@%gWJTYj z-yQZQCmkv!=-vKgfyEeT70z%2E)|Ir%Z|mjT4pwzVM-0;oB|9RMm26>N&%HQAOHrC z&8l+Ajqit&*6MsnSy6)p`GPzd}^jqJIsnF-`f0wrPm@ z6@Q~?llwfWH5^a-QTU9m-CE;=-{BS);~=&!m6k||a9Hh4fKyB?gIqVIJ^+p0+i zK_ks0s`6m_MSi#klMzuEypUNm&Ke<_^__yy*{_MrNNqP1qprb_C24h@zn;|7vcyk@ z8^oUxloKpt%vyj+F_PO@40)@Hel9jbet+EbO=JX+^<|E>z}JJvMN03`#(KNOe3>el z-6=z*mIE9-AR?u~=pBv>WJVMTzU#<~>W+heW!c^;h6`n75Inl^>_s*6!Ox@s8A-|G zboox`DQmqdS_B|RFR1LJ$X8zAqti1NF4vI=#o6aw1k|-n!M2OlkVc@TBA4GM>VH1r zrom*Qr+S~(*M_>9S;pd4vlDs^os+zwo))d19%@XgU?u0kRNg76E(r`d5lK`aARw&O zvCIb7=2ck_Xb(l3QibMwK?anyE%08@4bhG$2$~RRkIN|CK@<3(B-gmW*&!YYR(~`RR#CEr@!N90EN5bO>8XQ}$Tt+Z;#m8l6;*@y zFG*KDjm^$S#|tSJIj^BJYrl@77s7m5eS2u}HfZNv#0)p#EKYy&RN;}lAsVrzNHZQt z0pE0jzKx=Qvkw5^1E>TuAfC{bM&aJ}^xJ+34Q(!)L7k~!t=_}TLLS<(e}B2{_^BCn z!V#{I6y5Sw%MetC>cl(a+$C<=>s~tshFc8<_9W|ENu}9j2%J(3JIB&=1{@MeK4ldl zlrz2-8k3kWRL~G5He6~^rHu9Jr8Z62*s+KL^an>li&$*T-SbOhR4~veK(E?I!9)x) zGHE9r90?gXqW2?E`ZSC%;PcKW*Bc*q$UdUH6j7-)bJKAVO;|UYXRZDWs$~_nz8-9)#wED|&2f!! zI+(4lEf?MYW_Ug%Oxp635-?1M!cxRDS|hKNXYTjb4z{UH{r#%^J82I;@{>weewZ|RNq=V6cuQN*Cm64GktKrVIBx36@6LUA~=mjNoNzvWY&_)?__zbpZ zfL^OM855`SynnglHUeh>w?UX8k=wY|@3{7v0mdTgi~+GQxUQojUGYrMhT^PP&tcl% zrdF=4G{5IHahREgE>ddJrH*-b`dLx~tlY^y_zi~9fsJJAwX0}Ca;?Y0rXXuJGA&pK z>h0@-;)fj>mvS{@*lnkGXCkd}D#soKwRL507Cn8{bbpQAz>M7@WOj({W|Yg>Htd%` z%#S~79F1JL0d)omm&;_v84@`MN^6tV#LbBu5{1DV1b&)R!?GtDveQX*>?9uX3cbCM zH=V1s=tSDzjxW87ec)2Fyr1~G5#Qo|ylhc(a_LC$fIV!4tSk<}yzqSz2aMN)hQv@MB3-zRSDi!%3@RLL#%&PN&h9%* zF~mL^IOuW-f<+Z3mo=h<7L)=x&%+&6Nt_tfHRxPX)G{3_vGOF*G%6MXLW-GUrj9Q4 z&5`xI>^>zdk~=96kX9u7lCr~$_@_!bovX^Wv_df73rN{5e!Zab4jdpx;^16k@ ziH{*`N($!W3%)LdgH-qUK4hb(XG-p2q9$s0oj<&IFFK$iXaP6l^lso9ShkftyY<~^ zgI_VNVRPP0(9KqP2#+|%I?}5&q=@|BJuGASO%D7i3NSYwt}?dF6v76L{Q2PO>VMki zrrlkaGhSz*h54vfvqg->oOP4wTZlZ>4{EP&lcak^H`y~IOzt-s*(8XF&2h;-wrU3D z=!$Z!@acp*Aw>tiy&wt=3z0Wv2z28qU*;P+YL93e`TTCUuFHJ>*kAh#VMVTtQNo!| zmD)rX59=;zw>mn^nl-#|CRqf2uYYj@Vf2N!yLBQlU4|Bo7TX!=|G(j6Yg^_Ux zVZ4-=fK4OQhNwJBd`Kf&#KGV>cs(Q~{=p2)@7x74!f%aYtI$%T(zM?XP+fS*WMPeu zm=rHi9woq@FS~}U2p+Tb9GgXbOYCiHF_{+1uoChV%Z`Y3_pjfZ<1%MOwttx{$7jO2 zwb;tFxQ||v8?)pK6+J9&=q3_UrFcmP(IE5H#-|>d>;jfG{}v~&YHD>LTa)Z#H%3Aa zjS(t_*Zcv;1czx+q|Q*}b{UTj?Ws1cewp3`@g7;tcz*o;JWh_o8ZUA;Ar|6%2h?!p z#_)b@2d*NF0i3eWmvYDAK!3u>n{3^HXauDu_)=oy{*S{wO0F{$yOftR6?`z{(7t#T z8YBAd-K96x+#g$EDgy4_TUsT@5Uh58k^drZQ#F2rs#Bx~L6$C?lR)!6wO{`&V4B6b zMK7_suO)yYGdL&bq^t?CUL})4T6D_4y4yKW-IU!>$(S-)9x!AejDNIDYUW9KI(bO8 zMP?VhEm>S;UU^K&wr6j|b{KnEu{>aIS6Z`t6ZIW-V(_HF_2cxu#R5jQ&dG;6P8CxT zK42t|DT9m_t7;9m>pcaHnbXR&`}1B&X~wr&xumm&85K^%1415CncbuW$j(U~U^Zlh z_>Elh-J$(*nV+K~j(?~Qjp}T@gOa+rBFEK8Y3BXj$|~_fM=u!XznQUrX;t-g6%h&X zRgT&YA4eI+>SyElKo27L*6H>8Z35DPfhs$lFo2Hs!DJHwrv5YLH$Qt^LGM18cTpR$L=j z{*EdT8q9{8hw5|XZmQ%pp8Kxs4Mf@lnRo`7wff+7Tvk;aY^5W;9hidSO>yr;IsO2YYNfv zhTT8`FQWED(*xt<{Q$djZj}VGoPMLC((%QK#f8~8y;1AzdUGQ8{rJ1j@!!qRKNiC> zn%6>C!rEM^>GY|cQ5jKe_ObM(mG`B6y>XoCD4$VvDIw^6F9eyh*8_D6C7Ak2(Pi$w zEGwBkw|}&vs3LB0;AksM+}Ss$qhY>fzI?f2xnZ_tR(j)R%x-0RKQX9D*M1>XKhpwT zXteoqA4dNKE^$XwQ*b_;Rcv6#liA{*&^y1cBPrgcgozA-+jOH=FP^lF5ZAzB+%ze=??&o*# z`G0!3i>S%FFE2o}2MqCC%#M*2<7*}hhK(J1?-;i5dD9!Z(rMxEEin}zzb|a7ZRgSw zF;EUiWO0(UP-eA=8APy=dVjXHZ7<*FW&7mxtt%>S+ae>#9WO`P@EU{KtbXavO%qcs zJucWnk?*yv&EZGo`%91ytKl5XI`u4B3V&j|yz+H)dtOn{B!%ykHHzar7Mr}ocd$8Q zVRGYh2!$Tv?jWMXNLZVo93mFCVrSySHixI#VBbS>c@Z+9+pt*}T?YsBQ5|?ps6P#x@yZ99hL8&ywnd*Wn2je6qVJrAh#^|3H3S?c19jbl&~+V8n3XJ>lQo6-$dl22 zaSkT1Yg5?Ub8XwQ-Pt-jtHNNUE3wf3*q-)QJzy^8$Y^!`b76gbd8@vNHT?H2Kee!n zlrV9%2C+2wxvxSg^R49&bJ|hP=4Vfcb5o=1Ywi>@QP>wdRWl^?$Mla%L9^ zU!pQ)WD@VWH)HwCBu{;vEjb1YluJZLr z4T^hH+)EOA=#(M_y|=z#8bwZs-&4!h&Oay=&eYIyNcg_pn8!^~E?KB%a?+R5fiZVO z!*kGV>2R`960~5!X~hU37=Q1)OCV13_^NMHnmlsaK?oF>Wzi8o;~aY{l3p^=bo0B1=rK-}iFzcYOvm3wkn@R*J(K{T6|KE8esx1sLg7v^|5eeuH6 zLaXzzlie303ZQq$X)9>0_z$J_o3E@F5>U(>Z9kXQ0yJ1130g9ut$&#)AB^V1FWFw- z@oB%pX>z|~yY0TX9ZjPTuiDvPhg2=zXk^9A8`+ljnY(^WUDg}@K0NE_wiUfyYs9zU zy=8USnlR*zD&wu!X!`+Cb5RMIXQ0eAEC*inWn8i1jF!?!T+~KU6h*`rqTVf8MXD;s zg4GD_QzDrm_RR=Yp?~c)x6~z}ZDE{fIcWoPYoP&r!L6NF4(GrugbSP0R!3`&#{xlZ zf;XCr?)9llMeGtiYt60sgJ`<1RK$`E2aL@rCX}rV$gQ9Wq^6uVo%f-y%-?8~NRx-= zMDG!k$te@2_+J6x4q~*CUVVo_VtW>FvFspDU0LUb|T#%x_hZC~@6~(uHxhP5f=r41u zAw%u=9Od@pPA^6ktsQz*A&2q`9Mghyu)9oNUeMo%zISmbVtuXP{ALzjV}7`O_Bn2T6@%k{RY>@eIlu|PTVAZF7@H-Bfe)e71iHF7V?+Wz%+qsAy; zAzAMwkDSjAg(=b}Dw@`y6rJ@6guQsD&io5nGmV7W`;ohzt6IUhg6MS*S^94#-wrfd zKwGH72?Gt&8aD3Sgx;xY(V+2|lki63boPVMMTOuyL}6N^$mmd%uwif1tE%o{TT>?x zTI0Qy6@Ne~lnLG61Y;3_9tuDiPKb<)oEBZ01XNTkeyOcq27UZryX+7Ar8YP4HsY}A z!bM=QS)sUUXw^BCHyYGk*vBKgb2RqEf$gqq?X7<+QAfD_l?jomun!J)W9;{~HESYQ z4e(I$6Ey_a!L=m!6FWxL^^^Cku1;3;w-xYI+JC*lS!*3A1&2Hcu8|OFHdm1T-}X-4 zahxa-C2v^HKzpb|q+$ELv$_Iq^r%1_Np49=QG&>`@Rxl+YL~FODh0dx*eQs~xp^W8 zLqfjhjWB))1VfU!_T83wWG&@A)rACM6BZOSW|);@aHlkkAahgy8Z=C)P5zGM$4)$y zsef4F3RalvQmy-ewQ2>D@>!Tplfg}ehloD)luZj?h3}_0@N2mK3UAk5_{XzMSvm%G z-oJ7jhq=8Vu5be>a|lm568GUSp9|$WHuqA)RR3HS-3h~Fx^r@8V}Iew%5LCq0GW>7 zD5ucD2&0l#UQvvhTHX(uibt(2*?%pT ze~#!{;P{iO#Oz6 z`Jr(d@6NLv8fY8{7O^df&9A(8s zXc@|}O8*gO)(n4`<;e_{S-zR#7Y(^}O0#@IR=n|>d^20}LJ~gNe2}8nT$v#7%va1^ zY`8FALu`0WUv|-R(`UX>dm+sjzy~2<9bo8nVhYyjPXm(y^2RF$V(m?cw^wJ|V0$NC zfHwwP9ZTIC1qDXtc(1G%hkux+Gn)Dvn%yfinq`(W$>}y@Wl25U=bkbz=vrc=Up09a zx6y)H(G`G7k-UMaibF0^7H$0gU>Wf+`{K?bJpTa%>Co$$Qs-F(|>OoR^9U=3JW4v zooQ&N!|=o5pGP`HH{BbR*6H|*<2&h&-d@pS((6Bo>j zMtcP(^EL1@kj&2=T=)cz_n%FFWe1HrY2Gy5lG}>Gwg=+K0JTd^h{CvyRpmu^6BK)A z#D%V7o?cedlY<(U#eXL(&nqRoZ&+;oV#XG9HhF%-m1U@Gf~j_!G)XvOYh+e2;$w8) zN9t;)04vNI2zz-Tw7kOTRI}Qe0{2%hn=pp z+zSyZZAAfOehjWue1ltjNZUwncP?GLw~BIQ`CX_)X|0>z;%P3{0>G3%$cvGW!TU!n zvGG9R+5p?eFn?pGwS|74f^OlyAEB!8KX_E(c+x__EV|J%{g|KEWl z{g*ZJpKySGStCCoW&W*w41ZZ81L&TxXlb92M*vKZ(I>zpwI@tImA{}t)t;dBRQ^H* zReM6wQ~5U(Nwp_v?0+OZffuR#j50DlMSlW-s^KyLo;rf%$rhXWsiG%{&&MeJlW{aH z?NjszC4Y}3tKqWzDCIZsJJp_WGapNM2E+5|9xeT2IlqJ9`B)__;}a#!^nbU`{x@Vl zx<8TqoV6;|>wo_CvZ05MLyY^D?yM*hwbcqdvunzB$S9Z!!tat`Uq^#TqRS4<3h$ZtqFYFdt zEPvJLh@IBI1R5Wze(c!hVyMQ|e_a^)@@1tZSbkAn^8_cwXVy*Op!b%c)>QoXGZ^CL zLEXX4)H}nCh9TQn;q&${(N&VKv^vs%XzCV8 zcm_%QZ`~?H`zL1S|8983e>>Rq|3G*~I)9e`Nq9zPX4+pM7MW@3et~jhru_wSkd~S0 z7kI(PG{#?`+?bhIeu1-l9-i@6NIyEdU!nh=g=eH?`DZ(57$2$r1)A{J_R=uY{0gzi zK>rI59%*?7u8{s$SVIPeU(#rqo+icep(k8an#_^WgUWkNf-^{-2humWGzOxt^Bk ze=7js4>|d-@&D)njKARjJ&M);#sB-?@%g{_e?Qgmf8hTyGCwLh%v8)j4w3jD!TsU> zH^Kcan`6|%tEW5MBD(A$f(@tP`F}TsxQu|0eVXjyqF%@p!fX_O+bE9NwdZp7;Wl!e zs-9fhgve+uSRjG|y8hd{&IB!q2b=5r>F%vs$x*-3)bqTp*>1B^zgfoz_P#ZS6$fwh z)b;N+)L#!8NOrzDMBf|l`Sv=nxZUO)m)=a0%((}vNxEojEv2VKV2>JG^M8E0vjN=8 zwL=sTXf`(9NzAQ4F~B2n^}!4{!S%sfQ2QVIC9RzWh9RDxFffD6e%^vf10D%B<8`h3Dwp6he>c+IhLZgE29-wUre z2PQm{jRGwKX_j2m*=^@xzkiZ;FA1&1+$s{p=`ru|F4-1k*4Gz@$19iLB_3tH(u0FjP{IL&OUxC`t z*;|fN9g@_j9AYJ2ukNs}uF*p1#u%`nk_Ut+tdzY6b&N^b)eUY6b$`#Qnn1XI#Ui%i z)dbU1R&yE=gzq$sKZ>i5iNO#_9nraowu$6)5L2V&+hiUYE}g|d!7h{b!atAzQCMVG zO+-&VAPLzjhf9pEktVW~%O6c@0AiyUwx@Cf2`0b)rD~}6_zDUPgrP4`U78lZNaeK2 z7lS0sw#APex>T5mV}B5!CtL^{d@Ltr9Un}rU-;_?3N&_bzbv+VK|-Gm{u1{*5y5)6 zJ&9A>8Pm#`C3BLZC@18#B&43s#mg}8oqiM#PV^CtOR5q)aief%ab1aqN-JORw|ING zu@8B%sHGuIRBQAHqfkctyf_m{p9|Q(aq4)xBa-xd>oEI&*nhjPcQzRQ4dA7=QnR$R zqt;`Ts1bV$rD|4-wrVt1j7Dv0k5JSW;b~)4i9LfBG1`hzd&J&?+A*5Tn@O8`Q@s_#@yrTMy)tal(tBcR5u5-u)DA~=tIM8r$RWgZ z8^(E>@{%h>X~^UqI6~Om!)3?r!KuEQX=+sMPf%+r;=3y~`VI9KAwvyPNI~@Ln0gmJ zD1Kdu8pgHukX^To;ya0qAy&N_R^eIrxIofGMSl?ws6o1L_aFSlY8>_?y7Y;{f=F1Y zZWeVwEww{Kwz)rxi5*<<6~(l=oaH5ZmPw3%#5E3*D$F z^52Ax`SRjuteMIIkH#oVy@VzIsh!5Cq=L#GAEm_DU*R}$5NmCghtT@Zk9u@dzMwSn z4u5@P={N;}+VGyCS(o2 z{hF&d!M_3h1{9=3IDa@%Os%!&glM1Vupwr_HGnvE})*be74lo_lWe8r!g zt2p~gWNOh9woT;$Zz}F2F~DtIiqzFir2$o10Zlu0Fy7ZUiyvlGel0kAE@~ ziSKvjyz@!=7P3`5$UIr-E>#Z$nBAEJ*)gXgga72~UQXkrD2uzhwc&wp)3)ACuX>6|Z;?ZPX8l z-QsUeEahi>TWg{5uH$cH*<`z7#6zDs-z~REk6E?z2E#cq=^o=*KfF*l9e*5p-y$cs zi#V31AqqRjM_b!PRl%de4ju+6#gF`*v1XO`Y;<*OxK}OC^J{hK+R*hR>#b zg$NGV?Aki__sL^l(qb+}K6;iaK#DV=`4#tsBW5&tVqy}8MZGP3z87HV|8R;5^rc|p z&l9N65fZ`i*MInkK7UlN5L{Z-d^i~6N?YAKFPk(xQR%h8$yY-k(wI?#3d6Y% zdI#3nqD1Dw@i3j6MmkdGe+M=ycUXQN`F5M)wHPM~8NuRO1N7PThnU<7OwZNmY=ik^ zpXYy=ph33pys8qs$2c8k;OV6jX+PoJJbc?qb=i=UR&sI>NiP=Nn13qfMobzBSM+}8 z>elaE^sL_&wDf=t^8TW46zJ~O{=UKJ+tVkS>47B;#O641Z+AQu;Y$H3@QP}CtwG{ghzeY_d;Kj@jhdoG_QiR2J%q$C z9@+^Lwoy9~H;GApHP{_kK*(Z~uBPyCdBMa#pjp;KzZa|cf_5d&)b+-b)v(z?m0gd$ zX1EsbMZnzb;sKp2vv14^{YBDJS?-SQp}tAB%iC~+P{W#!A%89>XYJP{j`C{$AHeb7 za~^mFI9>scSAgRc;CKZ%UIC6*fa4Y5cm+6K0ghLI;}zg|1vp*-j#q%=72xH$jT4u z>e${$9hOgge1A+7%m9IE1}g1`_ZKyFyLwBmD=d|$*{ItH$S@`W1k>MshLUUaoa7R> zg7E}%E7)6!wV(mZ;$M+0nlpZ>al5PhT0Xt~4l?|;!uUjt;>nnFTP*&zp9-8wzs3LO z#CA>rw{z7&@ae^(g#0Q)!gR<9i{^y*Hjp$XUAh6hg?|f-XIe1}QuWy1?l8TiS_{w| z`E!aHSPoo0?>#IL*OVQ~=g2h9Q)w7b0n$kC!#X-@jjVt*4UsBgK3#$B{LH|_cLYL&PYI=vCp~7ZjdNBXWPYI+W$oN#HZXzs$zO6 zMP>T3CnG{3?gy${3&VX}9ktER^UH2W3Sp%X_eByRAINB}0znN3&=r6jp7x@q~p?`(E>?j|)zD?_>d#YZ#ErFiC3l|WFBSiM1 zaogG=kPCoHu z(UJKH3P%A{A1{NH^$Ol5@9|&(n25DUeO7{*H>z2VSgX=%*)pZn=Zz^dZV@WlaeuC$ z@=q$qYD~(D60-;GCjeIRV4dK?S;r*Tok#25D!VSvH0MJ6+3TroZ%;AY<90dgdY!7Z znsp`suqUT{bj?-ZP)9r4Mz+QtX4v7{^^Un#RZ;baGp^+ z-*S93O*8quRGLzjYDRR4zXU5Yw1b)QtFY>FDQ}-{_MB0C-y2hd)eljXxDh4Z;mNWk zYkt*S@M=n6SIx21AnBs~vh^51kgDf>JvY~eZmqZ^jvd*Kk&dL8a$Q<%h<_luEYoja zC*D~}iCN0*CPbN#;XPPuN?(~|_!7YlB=Q>r#y8%}86Jj!w=}K*yxp%o7 zd-{Zw4fcp$%~448O~xU|Wj22<2MFh+<|F>+{mc0dx0k`}y*|>f;xkK4jS3H=EN~ z$;_n{?*04KVqw?Vk}VWYgrs#vi|A9{dZi^y76z`6Hzx#1aFl%T+)<-T^6bApOz3?q z4e9bmFKQ#HhF44y(|_WNA1^n8G>x&p6(X8xnuJgkGJQAP(m%91sBKU_aA3zd?3W$B zB6X$Yt_&ovS-dGceB{j)oy=w`p0D>Mitg{iHk8;ys<>37s&ihN0;U8PI_)?2j9Aqg z@Mm#&%{&+;*nq1aw3@>smF$vU5Mkogj4X8^Md?2^a{;H;Xn!QYr{{cgDw=U43-^Ux$r|YT2oq-}MRSZ0g&rNKYzHZXeC@=EUJszWpxx;nG>Ebo|ILf? z#wMnG$)EdM*;sDEU5uB_7V~x?p<-q`|p|QHU0)QLK zko~N0_g+a*6@UDFduc~F-AdB79#ijh*~}1sZWz+prY{&o0)hZ}ps?}& zCD`?K^ER(*{dsx-S3s!0J}bYVT-hT(%ho$q!Fu?tnTH(vpPf)&Vv!kQD96~kExt+d zK2EPZ-u8$sgf@TUtJW$VQ|)PM$fVpts5Eau;6J*dzz@NHbVKb$Q0i3bm{jLo(oh^L zf?jPBDM9Edp+jZPX?2up7kcs~OQ`5%^l~Ej>iQ^t+Z)~yi`{gXi%aFz1HQ(f2rlR2 zlD?a3lbSa_Pqimo@q65wk;0xwq&<}{t*h`t(~JVRhnRmh5^1nUQUa7XAjdPr+OOA$ zW$$L-@T_EU8p7Ddn1xpuHw?(l4`Lx9gZ5=ciH|a+4)RNI-bAlIj2TOfo>T-apo{-) z^s)s8UV)tC&PQZm?@QFKjk zgX5WT&JBNoSJYhd$%S#3_O{|c>CUJ|ZxH#4?20?+BWzl-$?eHyrD3t}0>$g{eVS#3 zRJY&%2S=Q$!&z=w13rxl8Ebvib%&GQ%~y)pm&9c=^kC)&UIGREzE znrc8L8|3omNn<)7TYhW(T(#gY%1Mc4yP3PrC)$7JQKjp@I+s0VcY2A-JdPTE)?F99 zg(w*}$mwuwVTw|363w`_j5Kq0@3!IDtUB;q5Zfc_vdSO}ussf7v*72{w}Wy=-FITc z@7t`*3h%U#mQT?o&g*$EFBU6oTXqUvsea<`U*L1LbMZ}*mu2!HCvFfn(ygXB_`amp zPD?b3X()7&K}PH-%iy_5`M6~n+CsoJ*mY_AU+nz@lq_AFsEe*uTxHv~ZQHhO+qUgh zwr$&&Nr(X^f20&0(*-~xAaFNY1B^k?muo-+mO1;k_jNX% z9EGS1uKT%E6bA2qVwJSTSDjE3*J3zkIi=!(o9e@S=uXpbOj^8Puj!QgzOu5JrN)BsGR z7?>y$*Zy&9-*Q&wC>*q0maT5mC{;!zVM(!IHH#Ow;47-nnq+E1aY>G1f+BN?xa5y- z&zywtbV#A#p8)#b1e`X=+6VE&J{x_*(c|-h~ZXq-M zi<(JgBfGWKLAanr=08gvYw6&^PIP6%rc$YoB+t{z&-R~pqQkLMqWkSy)0ikmp1(}Z zfAF(J_Ytc}Sa~TPpNekn;2r7VuMw_DnYmQgo@NwexB`RD^0QGNm%xU5J{frOBXwRf zn{V$KQ-mYk0YI?V-X6wjcsd_r8)~@m!oGXDyAANazt(A9S&9PE1#_a1vCTov=NS{j z_=(>I(j)-DH0j?Ug71ESAZE4Yv1@Fbf6*JLoK*h-wS%u6B}9h@!mY+qy3 z$N~4MA!7^`@ne-LC@vr)LPizWa_lrVP0Pq`CHG24wKoJl!g^>;EFK~k2==v-^_FJt(G+Nm*;kx~Z3c^J|{L|jxRh&E$J{3veDD@SxTZ!1II9jl*vf8y99 z^*;@V`t>PJ05^-EJ=C!LqS}@W?RGRulCvAtFxw9hD(1{j)j6aUfpiYDHjh=@^QWS1JZ%anv zZtq54%o0`}&$VS^!%aT7vm`mqm$(g`K_a8rU7&Fe9iq?^jJ77*p>cRa@OkT_Q?P}-S9eN7nBkV8)h?C4YU5#le>I+_6M-;J z&kz!RWF=6Yp8o7*$!c8*U+E%4WF1c;a&(neS5$XYmrSRjVy3dTt>YQPxs17e|ET?H z*(ljK*+|vYv9`3DN7~SQM$NmSsZN~MTyVn2yuo!!<2>9f>qd|2&Iam zjv{+Z8lsNY#;jjyW@T27A^F(%Pn6T-F1_dnB+Fk*z96Ue^g5@gZl%CJ-Skkgw=eHGgVk@8 zy8WFb694r@Yg)tp5G;EQF5)#1OuOg2jcKmbaFa=jxAYKUT*K{9e=GJQ!MO(4-Oka9 z5jTsN?WcUHSyxLjuX@6)lyT4MF}BAAo7@^UR!#K>*4);Y^lR5oC@6XJ`&NfZ)q?d1 za5sW)6zr%VA&yvd@uMc6pU1MndWJPT{)%%Uh8~5Nc=^3VEP=mo`jncZAnzfeT$=X2 z8VzLnX|i3uT(wCxe*+ox?FP<5*M8;N7HORRuF6sIUD;LH**GC_&~d$=M6u7o^ct^6 zSbqU+Kv5d9PGWd|IJq6X-Pd_IsinaCiB_k;f(096il=l3R4|RkOVQ!C!UZL%p;uju zfh$Oy?pQx=g@$f))i5_-{d>_7O1s2{L(GXdv7`&j3^`APRT1cVEp!b{dJ^;7!o$?-`ZZ7gmx`i|$ z%q{1Y6!-)k@kfgzqNA6n=w|UNhF7jYu0Kt@=yoa`QxU;YsmgTIhU;rCGCzU1n+RxlE8(da1S*f^RwkVpbERF#^-NK5+6Qlmk8t~+ zHG7JtNW^I3`FOy&=l`tPa~rT8g5{gT_dgh6Z(l|3e|3;mIR9Nl+-x>uGDE>OZSXNX zIlvaE3qOo$a0nTPMG&pSNMbUXS#Qs^6%*`;ytFGrb7$AdmG(`>T+e8M=CT zUxvCxf3w`R3)vH!zq<|Xm|;ex(3HA}cCnCj0#YKrHDxgsXhxLz1U$dae~yF=Suchp zW5R3lcQONBR(tw2yFilZ@$jXKIvw@Pbuy6vL~?u~;*tOc36g|DBmJp%lVX{1jJZW2 zdJ~_Au>HBD3LyllIg4m=do8v&%%V*e2=J1*e=4;~ebN}Y6N06zORvMl?>CW0z(M9V9@N%@NPsFw=Jaop%oDFyJk8h_+Z<6Ix2a ze_kt{fQ23rOQi!XMx72n&EuUfU`msbB@iF)0kn|lfH;x3Lb_yde=%?#>=KOqto~G~ zleE%T<9u!8CLQK5F}Iq^>&v#k1Gv12Wuo2X@F_UB*9qc!8)^||<9`3xsym^Hv-9<) zsfpZde2GNctEf_(dWFMj4lrwunjhL}Q|<@^)eq7(n(K{;!^(Vnna)T|DHSHhd0D zzg4|}3`W&+A?Fk44G`<=86Crrua{oE!vf7xS&9bm&c2eGL=6f{q$hBZp!z8nSJ=LMi!SN8>MKo89ePjuHOy z=b0~j_*=U><^tK5C5#WatggnbF8?loD7h0A8)h!c6xBeCUFqkuWx%h;dDAp$$qoQ1 z5!^On-XtbQ%_sr+@2Qi9e5-2xe~UBtu#@I2Xl7g%RKJDRU)IJ*5NBz?Wfza``R1hP zv-TTPOGUni`6^tm(YE=Z_5s=8@bC}=IpurcesFaC9@gH_r>qT}B`$=Rw(aRK&6gs2 zNHWHIP>x{2-T5u*^lZmChV|mk)TpYv7v;>+o3(`@bL z_O)C4bcIpQf)g3TlbM$0vp%vRDAM;*2)$x}mR8i3k=e<|tEBfXawrJkh6 z7b!QKY29AK$KY82%Dnco+OOB(`+loO8N(o$#PxS5u2>qrbIVC*LnaN5+S3?Cag(xA zE!9Vql2zkkJ`2~QNvSSUCtrmpm%2}ortj*ok&k^yNHIJDk|?-bNdzpB2EZM8@XuJ% zgfQ~0)YRQ5puP+ef0>S6STKA#gy+~Ej);o#u<#|hiNL|{6I!!iMQSC{q&a=Fs0OKj zveT9D{j;6!sGTep|GhynUJSJ72b6)PBi^~M;eq)0&1d1_R_RlwQ@d6jDbsL>S;Va?pucHrkg>zRjSRXT$LNCf6ZU8&C!m;eWuR6A$6ZE2cHN; z;vsi$+Mvi|k{pgT*RCh6FKSvvW}8GdN)ih>hBE>djWDc08U?e9&@@OkCa<8x0z2k5 z{JKg&$#tYid|w2ATIoNOq~u8k2TvZkl}S_!Kh3>A>%vID1sNl$wKc74goXVr0Vi59dL*$-Uh_ z>*jplO))ut6~X!X41tkjG1m&!KOr6pvGW;=h1MR`mjD(X(*4&;%}ps_aV zDlsP|M=6KSJW3SMFql~v43y^nY&RKER6oq!C1RS{*O-rIN#wHJL~YKpA1?%IQAG8l zf5}s?T%LQQcZlrujMXYL+h(*eY2_*(sfpx--HBzhXlZq;F{`O@U49UQiVVBis%@oX zRLGNf^AV3{0LDvHXXl=~fQ(Uy+LUASiM{f;zW`32Hb+7wEgLew@x~DPE%cjMDZvrN z?pG?9-HI0UUQKHSGd_7c6x8w=fBj^Wf0`pnZ4J5WptYIfnw7GEh{YmiUH{$%qOWnE z^(jC*(T)3@;HDd~)`v>(#IVme`l<`wDZo7hxL$f*0M*K`S~jcOCdqZ%ePx3ND}V0Z zR6WI2&^hHeZ)1{VQql&WQC-PhOJ7Zt9?1lgP6W3K-3OrE7uPjCmR==x^;RujE&bPH z1kJPS1+}ph$9bMZd7f7@*Sr*%7L(1#9QIm+4XWwJlFc$Mh%IA{?-$Q(WuyIX_T%;z zjg@QKmn$XJ%@xhobvE{v^_KNle@)lU3$0sDS&#_fG zEEKhcDREG3o9DOMeCBJ$INco2*;9$Oey|Rz2pqjD1!-P48z(&8!M9;E0@FT(vQU;{ z+4O%}@4;g$8IlYT?R!HzDL6H;YTOjCY1}!XkIw`b3Ts2ArC=2NG%2Jsf9~!u1Lbfk z4SD%i)Sx&hiwUElHuWrG7{N;9vt?IzE_j-GQr}|F&8vLGSmEVKnImoZ3{YvxviYQ@ zjZQ9I-23jw$0^~k7p(j;L8JTLKkP*-pG51_?{ftsnJ6$qvdFO5*3c?f_L+a9><;jF zL1a(NPKn3;#JGbHr_Vwqf27~wvrbV_SEKW5Ub5y$R z2iqI&caSVbdh!ykf9c8Q{?k~x@d2%^dW($37fmtX$Pd6H2n(y2$ysfhLd-1>!Wo3W z_KB4i0P9BtOYILWTq!o^IZkA zCb5twX(DzJe=}_0Y0-%OUlzG9U<8cZ;eie8|FhTifaX7;{( zKuRsl2Mg~v;}3u@osJx^6EhVx#~Zt#kWH;adm-k%c0@9t{TLbV7akW5kWW#40G#N~ zZ;$u1skdSnb&=J3r`yML=fe!7|D}FK?f6;s!8J@fr3~hiICrG6lie;x< zpHb4J@!|`=*%F!aZg_;14?1L^WKv}|AnU6G9w(>dSi!fKCrDV0Zd)7lb;4pX-x4oSs$&t!VX^oLuy;uFWsen`L=eHhClF_QME`nA7%hnI5c?8dJB@naeoJ6faEFNBrF)6UBu^QE zsk2p{c&cSKD!cQ)i(?ngD*sZew{WrWXsmx@|MW75H|lDvrs_+!$S~3{)v#GN@mPF{ zsqInGHRgc!eCK(d7;eWd%nwbB+h(bq3x-A%8O=EekkPPW%h#)+aQv(Ka2 ze|VjwbZ=YeVM}#uG>&;ebwWBo_q`b|P{RZ`97O<8Gi-RFVLHV;h$&}kf6%XXw9|_C zF4^Z+Y`}vPP1CywF=^7BX%fjMfhMo$KP=fnp-k1^MRX*2+Tsi{+F`7CoElBTxb9=) zUHp8mQ)K3`hhcoHReYIl!w?|$3yi57e@$xOwchS4SI`rU8?ultJd(IMUeg|9k2W}R zkm(xtwPh&lzy+V@H@%T!F8N}Kv^GJt0$}|ButQO?k>Mkple@68vxt|12v6yp|NcVD2e~{qI z*Dl8DGv?HCj|oG-#9QNIesa9Y|+R29T~7&Ll58Fm9Qk zX;#m%T}Sqh7*Os~zZf>pgWzr|e;A$MZu^%X?LIZXq_aBGvebb{Y=RMdqIF4FKiB}a zj&HC-T-ZLs`ft%)i`mjU`*YdQf48JV3^2Deuk>w%`)~qs!}{T%ZERh$`YkZE4{cNX zDKOF5Iwp2wA<)SY)Br=?QfQ0!Z{~W#E(%k23kUCdQ4m&p!@itZPf~Ow0|7C$OZ+}D zpX^HZZ#Y9bg>FDOE>^Qfo)CMlw5*#r;GA3aNZ>o~;V*V>Fhedevz_{xf1#%>8|zKx z^vbcasr}V2{jjylB{-b3Xqt2;D(q{dJ!@*FT!Y74g#YSwI5CKHJIR(Uk^nP^oT-`Vjhes zSUKk~u!Wg%y5P!i>T1kke>e^O$UEeefRaKLjqoX5dIK<%shnz!Fq7MK!9oTx$ys04 zeY#EI-nl}AB6j@nNvH4Y?gU$M`uU-sS9i53HVk?Ipzmq;R*6zwz>c*2<)Mi%k8D$s zAQ;#tY>n#C40PE;%p!EjwGkas`*K734WScQ5uttkC2UiWi0xYvfBJ!-U-zMseZYn= zY(;EQ+X=9S>`)^pVkUz*X(Dr_nTZjtpL?tb{8y|n?Wt@jLS~y;S52XtS@jS@;$931 zL*V+eECpd9W9n*mrBRIoGT06ldS=j3`*w*-QQA-84Z9&(qJelqdNHkAwkEA_dL9EX zFWgxm(M=5kJ}#kAe@z2^V%D)fTp#Ow((rY6NiR(UdNEK{R3p$+&5)qeTgG~41r<89 zOaYjcA}1RMC_+4371nP_pA-sAJD$vP#XXYxUy)faaly%N%uun#Ikm&ZBjAzE_Qhky zn)(4j-(v1z!bW%wwe_k1(WT&`tA$eglVS+s`^RD8Q}ekpe+TXgp@z)h|Kcm)X9e2- zy+eCre`EV=Z)XVF8k+iGF$ro)XonPl12lsvb(K}MwwVc)5)+c-1e5Wi$WE5)H=l-+2A}$pyY}s3E{&1$~hU9!^U9;tEtaq^gSeRE- za3bsAIC`Omhe3NTlkehdOfxyxh=p8pqKgB7S#CZNVY zD2HSI2U@HpKW-gB3)^vz#9b5;3gjBbsZLebq*=G-nk%%dMw>f;_#^jDdkPNc>ti1re2W zQCvc*`eEjOpQqB~hO8hV17|?#z3l&hYh!uu)y3Pkk@S`uchKi7r$*1$N%p)}(riqK z+PCl?%-J&uDbVvP2iO9Ta5|pRCtS&-1&%1Z8oDgQZ(Gh?w*6X@)GB3 z;O!^V&DjCL!O(biwm*1+E#qV-7$Y8U4DMO62QPr=Pf%Wvo!oQ^Z{J0JC`bEX_p}_A z`+MUP<6IE-w0x@LSU@4OXfoEK9km*Kgl_X(e-M7_tjzbP>^o>*z#Y~fTcG9R6SIsw zxC}FlCA7Pc_S;M|n6{97=Tb7()}Tx`=GB}Vm7QMgWig9qSLzv+_U{)2qmN4wkq@0#8ISdHO7uZ^0C z>7Um8rw4zhf%|s>!T*mt!u~yB`~R*ZjP)-aVXXfb9bqi2fA)c~FtPl(jQKCNWnuoa z3ybAXUamj(LHlRV*Izoq{_NnQXJ-DBf79zP9bwFje{yv(Gc)|TjQ&q;re$Vi`BQyN ztSo=gmOnekm|2=$GEQyCo#?H?b%|4_#KH!}M_ma+ZWyT(lS=l(LYvi)gW7KT5q zN5@Rh`WNpxE5m=~17rOwAK1S-`Tm^`jFpyw`G0(1f4cp9|Cf=0rH-YZ;U9B}fB&ue zum8sX#lXnI$ndZJF9t^XzxlryS!n;q|MmaG?SK4V|GfYI;Qyj$WTa-rrDJ8GrvK-I z>;G{BSf#3j%?2~s9{=@py^xsN`UOhAI-RL_&LMb{tFm zV+}AM*ebRW`0m;_G{sfMjno9RbRMpgQx3|gQpo!L9Poc!RTJTT3OsyK0H=3YnFl- zS%ErHjRALYby{w^r@LY+e}4zRG4_3aQ=-DEwT-9?l~Z0C_yWRm zf(9&sc+jzys+v zF)K~=zvV!jJ_xN~efv7|0n`o1vIC zs23k1ux9}7b6!{nr{=s4CK;a{1$qNHeB9&~)2(BRN@JU+f4zHCnKAvlsMw~kPZ6cN zTsWgF@M*BVB#Bf!(LSz}oHUyiE@63SPFX&}T_LW#Zl_V6+XYWRpBSl7S&%u)7hj|8 z2gp+IvY~t=!mnNTFmCbVf!vDh6@4sr_!zB)_^jI055dvcWS> zbfgGOYBbg?f22v2dHQb79RmDeRPj;Fbg8O(5das685gYp8Z=nodC>@#5_c!lv4k8D z=j7M>kS!CFJ}gzG4qZ@XRKdsf69qR;*~}c5S_d%M^VEblPxSI z8o)u>Olf=RE<>o626aCXHB9Byhgk<{arX%_)`;BwtP%^9Kzq1Xgcs@K( zey_`*sB=;gN4V>kW+a^s1dt=!(f&X=BswXqN>f5Ni;yZwWQb9b8ZL~Gq4v~ zx{UoGe;pOtW1jmVH_Rl6ehxpAAQ{;)B8mC_>0C%06@Xze{qCQ^vA89xxK>A@p%cEC zBhJ~1jjBcv8zXs=!UcKcYd5D`59ve*}3~3$LPmMd?lXv)2TB<_F=O*R{N5 zx$Pw^*Y}K9a(j#KIjDA|%_<45pY>HZ*cy^bj09HZo><2wD+U|5`q{z=Bw6L+j6-5* zR(!-M8^@JQEJzEj!vl6jl_-;D<%Z;ArTg8Rbtr{Se~L*j8d2uDfmA7@)HZCs&?hFh ze>4fr)$lR3Y0(76FU}=MOnB7F&`$xJ!ygkC$-XLLeYlfV(T?Yqv8UJMD6&|>hg^HM zjE+|ib2YZH^I>55!({UJJ*%+&)OLnom@noW>aeMz-RW4>#7-oTO1XL}b--sKeFL- z{)xcg8PB={o1tpp4VA2Rd*o%Hsl5P)uLi{w1nX!V>AjHh#(SSizMJjDk$&`xiw=J{ zLLUP6Nn_+4Q+wHLe@FoO8K!oP(TT`Fx>cHL;Jr-?HJz)+>ub9Mmy=+2`lA(!E7ju_oRM&vAS0+Z{#mepLkxjElq1X)-hXbI6}Oux-9H+CJv5 zm%aia45lzhyZQpZ2>eI_AqO7be|E8HMf=GQ7lRn@nq=@?Ub?1S2nhhVl$T#xPwFKB z0`D%JK=>zq++s1+_Wx{Uq!(ZE?dLtQCDr!PKRr{rE#Y`yDdAHK;pbbKJ}m_pZqRL? zr`dg|hH-fkd*}I<)VujF^ip)Jbd3L%Uh4184gDMG$-?|^eknFaTIPR@fAmCDYP#uk zGCx1-oX=`ZaGX>7rJ*14Lobe*Uv;nsoKzYB&ywHQ7d#GZJQmdt?-!x}FD_JAfB-l& zQCfUa&A3@N-)&lU7$n4$&=5kB44%DXvV4NBt9fj!nr^1mZ zGMj6b_Z@!C)<{21QdF-Wdhfuk3(5 zS5keRph;yRczpMhQmqzoouVS8R^u!iE;axYJAV1J>&-)hc%$&?)gK}V-KXp`Gs5qc z=)oA_S~()%^yM-Hvo<}P7p1i(7wD3+#tWrcD#xdx(^$ZaEG@8 z$u1X!6~4{x!_-5eOR&AGu1<{RfA@I1SCBMW74uv=d(3~kr}UZkMw~D110t0~W#U@qYKz z>+X$g=J>=M*s)b=6FYR>i1P8y3v~Rst;wz>y#=+q>9x}Bm*Fj!h#M3LM-qlVOBhWU zD;O`>7Rxx7xnP?)F#l4wIP#xuBdz%k_1h07RG5Tk5| zh!a8{*$TFce>cEiF(7;~kSlP6PguagG4gY7!O29eH_zX!piFnR3b zr96`lwGCOe8f}asl{!6+2Q_s%4?<4;%f~mrj1xIGf3t^iFIF3Q{O>CH1 zxHhdC;Mk>GLWyYuF`5De&d~uAN_Fs`>!e$=c$qu|ItiTgJ%Wim_l1E$)gw*-aUw_OeWoP-zTP!8g=9B|DsZ;yE@q_@LP$KDl$-((!yR)h~^6-KNWH=2= zFASyIq|!uaHfKTk!9EkIFVAa)AKOHcb+N+~e}5z*!kl40qpTX@-g!}`McJgB$shBP zw&yoIKy{MgI*jH$Jv~X7hJ7t~=YGaq9`uB$A%&kOl8&WKPUXZ4t_h3bUYZ{=nR3 zf0fjvPSzLCQ!DVx9C{}#C0IYju0Yaz=eGp=6*7We2c#iAJKrFG;%DVC!eo!^p&UCe zO9Iah(0I)F)a}dJAw8VX&nm*~G-oL!tb|F0jXWFjNOXYe379rR2#rj2e}JFq$9_jt)pu%_Zi(AifT(9Tr!P;O2yRe& zh;<6N8tg*X#UtP{Q~zs|t}Q*2WZI7ul7QpHZQz{ol|v0`KZu^tp-49HraQnPtM_(Y z6fFGRtqhLXv7b2*dpW&`#~?SvQM`{Ys>^C8}{A|>s5F{rD%7<#xBx!vcFGt zpd^%0&z-*sPx|u|y+ly-bzRk77!9vz-!+p@Em{>p9#YK^I$3;_&p!0Q*Szkh$=q$y zyQt4_-1^`&+%c46=Z%X!2uOY)e^QuA+>$1zEJX}?9ihMP+ePgn#ArXn`KJM~+g4WQ zO#Zu;QjFB?LPCf8y}XQohsPEnop8D)P3t+HS|v(CXU=zdO3w2MH@!`KahKJ|&P;{C z9jL*2i)uCmyH0(t zPS+5lg5+#FV&5EE)71~rB>MOnxIkn!~ zUSivA_CZ}?^;f1YGYWyZhY@!fx}a)Q0iyY$1(66@@3^cwIg%)@{Z(0&T5QQ!x+Eny z!$@-x{Ao4lWO_XOO%V8R-0Q_1-)x#Zy-Bj90}Wne6;+4HHwd1Xe*;W4*K&30JM$5& z*Ia~JIqUDEtM!p~?Tgt2nEc-$1tHsFd8NETrjcm&fmw!yp~C2M8BU1R&4#$|h#3!D zG6@J~#E;lXUUF_8vnHg)+P3TC+kKw@ z^1HlyL|$KnOebBje~OmO>960?o<=N~Q_09t!i^Vua^%tyPnmthJfEITl;*RHz4TUr1`@=>tyC24`8_mljeEOb z%7GFOIYyLU$R-NE{g~{)?jO3MN1-aDXPyJpo|u$rCelXFf9`Kp$govRpl)$K5U&T- zZ=G4)4AekPm&YilMogW6 z9a9=AKeHzse>PjVt=O*STHlhDaE|1+v?yfXJ{;XZcGQ^!EDJ4u;=bpg*JXfJj zwk4=G+LmJcCYYk2Z0K+NWq3|XXkgq6NR#xvcabnYCH0oh9>R$^Shc*k$i>IKh>upL>_}Q1 zX(RuBe=TsEB>iQoD0&+V?&FdduA^%Bl>{=YV$^R+HUhnN)^yJo(OBh)LAXP!{aeh;b;MD`i!-~lTaibTX z-`Xn0#kk!1X(t}^5P?le8apvJe}Cn+mr%dcep><;Cdq>1P0acW1f$LykTYI4;@(rn zEZFF}kWwE0ZCTUO&q?l0w4e>RZaUG9URQrDkLtx;$QB!v<@~Es)aIvqkvS^ny1|+` zy?I5*`$E!tTn;-7L6Z2s^%*8e}HBKeHQcQ zIVsD}CaZHbowq9~akY1;r>?}Ur`>v_!|AKmhtZsF({&6@9sbL{#mxgGpJYBI-h@kB ze?t_ZkzUv(;5>_n-qyj6IZ|=@6{rC$>$rvz(pm12(pD+`3TRD+5a^(20ao9DlaV7A z6PnWX3xO|I@!~g!x-JGo+5XgGa(v7%9a@PaWi@$hv0n}Nro&LGWn zii!B-P>BMPzU<^2eG8czTemJDa8)0j(@qoW4m$nIgq$v! z?ZN^lCk-MG=ka8dprV^HojSFK3U_slLdUOto+=(<&K2iI{l_z=f6Vei(o!y3@kLR3 zGkzIln@C#Kk*;5@v-;18!#EBs?id&6_>zPG_>&-us4!&?qTuqo0S;8$IaRKwlPT5B zc{5#=7I|3+eLOq_%mmZ~-2Dl0v2m=Vi#X>o_rvd8UqC+k+Lqdy+Md}YG>vljhr`DncL>M>RG~UMYOyu=)&mj=(qc7>vRRMV0t7f*db`zSCKlv`^nGqfeAr+~%l*7CcB=^q69CYU*Q%@+K| za;FXP5Fzuzjg=J?R1Bmi(}Fae+t%f%Q(PM#O1kkbB$AL zunGu5n9;d$#A;Dfi$()3S}@eq4iJw|i;)Dcb`*O6cD}aGro{f@PW9n*uzl&_R}=_c zo~NrOI(wy;%b(h0pS}&vKC*A(-9y57pAShC#dKf>5D=;e@1R>!z(F2x=;FvtI{p+R z{iO6GI09p*fAe{T8y^izrYIN$v2L^B{gT0KA%g4bbm-7(sLr2dm&4_njrx{!6#8ZQ z0Cbv-jt)5fRWTCHlzbjAD#G2d&u8m4-RaMLC!o$URt7lp*oxUT5)A*Vae^l`%EC@Dca&xGo#|$rsjApe`H_6$RQg!A_epxM`n)Og({CY zN{pr<=@0;g{UHmoDwr?KO;4W36pGiy-cO6Q%{)L)n_Ot-pre1qd~MV<*Z4vgk#7Y+ zq*4fW!DsT=0!_clyn?ZYdkhzo;5pTjmo>(^uF z+U)^nf1N!3_L9axY{zAWeV4mvR5gP|69fed&VJiT&BP%r!eezum z2w-J}WR+$g&si}K@BAtb`uRCb7d1!o(PVm8E)&*QjoyRBJ6q+dpdI+!nrK@roVQIP zf7#LwDGY>71HVCdGhXB9jHwqFJ2O#uNl|WH|1Mg5k;mh2Rz@9Ya3GFpvaBfiStp@o z-sN^j(mvvi=qix8*pi(sd`^4{1n0}GI=APmNeyQfTIc+oo>V%Hp|303r`%%i?X|b} zVrV0lUx3@65A!bDuMhLrD<~HkD=jMRe=c61Q6a@L`JXd#;SJjjr%st&CqD-Hv${HE zuc3e_6{~7iFeI7Kkre6EHHI`d?M}^kd_jqwMFgFPay?NFr1$nD?6j^N+Fz;e>~4|fe+Wet zk;JH7+Nd(6Q#4v58qhw1IR(1({C+;NQYO$G++|z#NKAC2#b#|g?$lx(&(ACsmpY3j zBeqjY=%NtG%{)_=5&mF|Ll20|))KzNYZZ=OZZ+o8#+k8a%Og<#?&@E@rDx~J2;*qi z3H2`uhA8DvUy#ZoMFeA5gx>`-e@l`j`&8c~RFq&ki}3HqMKkREl|Bb7KS(Xl(wecy zGU@NBKvV%oQTTWVVuZ0k>U0zDDCj2fs`?66)hn~dMCX3S-#43|2686UhHZfu-)+G` zI?=iGB%Oa@nI%QjsAa=}>$?6G`Lb1{vnMA{U~6f88Te_OagcIX;A!UPe<&3nASmEJ zI+dfr-yp9q=Zr9l=o5ZHE2tiZS>7UvREupIY8GBcHeju!#5P@}e!mfC`qJrcI}=_T z(UpQhyL0E{mERBCi<2Cen9_h-N>1kcj3I`Su$BNxiq=Ni!ahE1#b*Z$Ok_0@rbL5d zquF5CpFJcbQ?Wh2!dl{ee_Hn``?|2tw74|3Q7N~H*G$B^nz#Qhdo!b%2WxVp-Cssox9US<-kSedB~|Q4*Sjl+h8rL|Ws7vN4SlP+pbLf9mx{%7&e(Y~5?m z7gYu{=89t!`^ut)j~lf~+ghP|VC;L5 z$0r?DS7~oubQiQ#?MX`T8FpC$%`lX9h{qj<|JP|fYn-%Hmr!~vL3e*|3M2E@(?0(; z^mSMBu5u4$Pe4it4Kcy_Cj2?zu^1qETvRsSTLPeWN>Ua0w>7Rsc-AQ^&_#jdKcwK zk-mS_i`AeG4-S`t@XZbl4i6ui4z;*Os*Rf!o>00d3E!3Af1~AxQ`@D}aSP)a7$`>M zu_J^y?#gCav`$z#IXjD_-V(qtU}7=%_wDNs!iPtTk+Y!QZ=|oEFh2^KtZuGsE^N=} zoZDR3(7T6*5QAwoP!CnUZDiqSk?YwbLM38w6mVI*G$}U#u(z}Ri z2x!cImM*QxY;u#mzsmVyJzOvlvO%d(XEEsZ`F6

^iAoJ<&~}-XDGsAyI9!uH5h* zIOobvVse`p=`ggIOIG559A@+&yQTiM6L>G zdw)FLY9=yen%T#4XFK6)nRW_(KE`Gxig?%N@bXWtV^>7f-_-+5t zl=Y2wVDtOge%lVV(Kf#7Q+jUux~ua0oN?{Ik~a5=BPxFu`k<|a-58YUtnNA)j?_78 zM;I59e@j{R(t1#z>;abrpxer{c3l;>~1q@we52@8$kLHvUps;zS6Fz$pI8f8TdOkh(K2+Ox1!MX`wKAZB?YllW>4 zI1dO63!3o~A^o;=D(J-Ti(!WNS&#{XoQWoTVRCJVdFsO~*d1}_;4yQ&5kd7q4KO@#EF?IW>J?~>du&=?_{7CO zIrQoZ{f_TxzSE7HZ`XqF3$i7X8=nh?f9NmNoUG=e7J!rG5nMryOe`Zin zd#6eygJptREk!F$666?63u+wdP{4MFY1Eb5!#b~Msm7l?Q`l$kDz6#6L6wj~W;B*? zCNGedtLU-OvZ08_No7^@H*{HT1!#Tnn5c_kyXLLNc z=k**oG0Hr(e}3|ZV)GOq+G&h>1ESG@==JllxU*WE5JxEZG+`V#;srq6W;7R5J6}F0 zgE=WZ2nV=ibWy0HqFjHVEErPX0v_%X?6;E~A6_hnV{}S@$^7IKblQsw{_QmTOMfPD zxO&6MoP2}d9@iSsEuK-lasiigN>~gf>&!K1u8MqL!z(QYgS6kR0k+hjs(6?nofo>?BJ zT3U%Tqa^cDTEwKx9R2;hV|`uD5`X#Q5zHjYqUOog?GT4|z&M2eVCJ=SZw} zULB(d>KVp#;-h9QOAaxN6Ws5YYhP(KZN3a#^Q9Xa5DELFBNuv+yCWqqb~ zDdRqd2sET+Ly^WwIt(ag{Y3H|r-<57W4a7RK=X2Qq)K58P(a13iy>*lWq%igNxri6 z8O-LuDaEfOX7O#27uLz@Q9*elz|{W6U-gq0OeSd|7BJ@dlM0Xu4Q%Q`Zb5DdtGRGM z08Epnbm{~p5;bAWWc&{3Sw*H24NOUdxzmrK*kn?d`l3tmkzz%P4k{8V<-=Q`Rz}IO zc*de$jax%oMO{ah5=~6~-hVi^3f()P-6t?+y3u-}%nO_w^c!c*F(d{jsCobgVQwQi zU?VvpkTAK((zH@dl1)de5vFmp5*b!$R+G&sWism74a)H*kQR{^l1WG65`|9XdJ%RJ zW*J5qjZ3N)Yc{JkmCZh;-KI9p)4bZ^$ZJu?xi;v6Y&;A>Qu!=Ajen=La{CMjs^7m` zvmUSNtop}>Bl0y^`VBG@9<3^4i#f-?V~V#ZHBt#E6MnQ6#w++4cmPy5iK}UxH_Hli z!_t#Bd`2ZTYgv7AvqmSACVg~!bSaeZIEj~jc!Y5iIP66xy+u-U=c@F8)tic~7R~#` zlmUnzHdnT<3QUX|&wt=Ol*Iyc*<1$`jt3WU%)=ekv(F>8avosRWq39Cf zNVj_S*bHJ56edI(wt#(uj3`d;s|NA2rza8CEL!2R2q?orv476HZ`=TJ{fD>oj3>

I&Vk`A4;2i=$>T+2G5BlXQh~&LMx@lr<;#a(2BY zM%+$7Us5}rF@SoXvei;e+@m3mhH*r*<{=KbX{EPIZrx5`B?|ZJo5v}09i(eO=uX(H^)a7jt=PM$e6I6dJpOEVr-j+tZ7%3bPA9rlgGGO*$fpmgYfXPRuXJ?8 ztZapzYE;bPk;7w;Nf8nrU3jnFoL4YFRQ5LkP*nY%9!$B41KO8JqCw@z!O#+la0aPF zZ;MWkynA}Gu||eoaS<6Ij|<*IXaquD0zKWJaR6OZ zD<<^uS`!->QR07EZ-+KE0)nIdG-@qNynD`~S4iWgnrhYK>0`R%+T*Ufmb)ikvJVsQ z2%d$@+o9)O`#gI+J3V_3He0r9HZmfzu{;WWa(xQVMBA~Qa(MtUWjQ_eBqz!fl^C3O zob=1FXo^s>P^#<1eyT84#P;E4QtFQS;_mo+TVeySTS&2fj?wel-F<@o7s#)=@NE`Oc)0K zx`@X{A+v8byz+V7*X?5XAF@iScXB*++TI(1SZ_NQks$Q57$0~cf^7z+a0S>t1hu26`)Y9k zJW$iCG_|0g0ttMqRXoN?sK&!&V6e8OG)uK+XVe4{4~_60z#wx6!m6tJ@^N97WF_ak zf7XAxu}j)xwJFji__aXzbGoCl#s|s>xD6P^Mw;Vw<(iztsueGcepTR#RzGl?ZJ<7r z3<}wawxdVUT6H_}Mg%zetB@G=D{OU?UVK(3#KAX&=<*(sICk1LLU!19+q*;)LtID$ zz(U_FUs*9{s)-n*Gb{2SJtayc_ptm31Y>^);GyFx{=-UaRgtfZna61w8+Ai=cAvYK z(rItkkY0)1Q*!E)43%0dH)}z_>32r~{Fjd>jqT>n3(2V}pzXol8x3BU7m~fZ#j$Q5 znJ!oWu3ptUI>>!aiHb9L3YLtbcnLzNNdEIy*!#$-t)he;MzL3s74^rKv7pc7JMvO{d{1g8zSBrzHLT zODrFoCZhJko_osA9Ux;^Lod!jymuP=J9*0t{&J1J#L=Du%{9F z@peSG-uK>4zs|o~f?#2QWrcrwc$W2)mV_NBC<;o;61{7f%MkVpsaPu@5tkcq(avE? z@ZB*?`W%E~$;yA^5?CTxhNVM8#GZIdBuNCxkO_(@wxJD*3eG6-m);fB36#lsS*j4s zlb%9Vi&N+8DG)KFHBhJEJIl6}t}}KUq93T2*`#U|RwyW&V%T&x0d9ZFJeXdZde3Ir$K_=csNlnOS#9*^*Y$9JJvmnNAy$Lp$$# zDH#L}29|2?$vI$!dcc3QW@Y;T;=(Y?^$_7Xgh92rF7skV4BLVcl^z$R6s?eg{CVHr zZy#zIDl`MBiNcwAzgv{&;ujXjzIPS3iD{O?U_HMO#3o&}w(^s<#;T_`Q|12dEqfB8 z(bY0v2E%mrAlduT&Ac$9+x?`V*xH0j78an>dVXclyQ&d|qrrd4y6X@I``D@T{*A^> z90Q^e%5BNDZ9X`5KDY#m+N5SKoJElO74YZ=vh`dtmPc1d04?EfSkK%w6)SRXF5M)L`zgPr8+x4_Tg43~?U>)qt8=dE{Z zmKzushELx$Q)YixNfv}pWA5)>rAf4iwa3)n3c?m;&6PTrI{J(Qh3@yD^JGc0xh({o z&yhV(cY=^aQ=%*?v)IGdU{{4uP%OJGbIk;8*5>P#X-*DdduOjvUyu?oqHxbLzJsXG zOSl;gim7+j$}*HDoIp8(k(8=)3l`4egwi&!knGLF;?;lZymgUP(PF}Jbi59-fJOt3 z@wvI#R*$*v$ed!sD@_{+zQGiy)CE79HY2voWtm2sN;eFxNi>j^m&C0jWtG{)DiY3b zjyK>$&@VCib9jlM_|6}z(_7xG^Mo??Ihq%($C~hm*`P9X5?3#qAB*aObm>ZHClSe63Dh8ITE(n^rA^EOXy;`hE9&&V9qTw zn_2OdRQLLr4ak>9#G+8C(;JogoJ8q<@|@Ok-MW9E*0i&K@7uM|)Gk|Tp~2ghQBv9f zB(RS=3q4J;SSjp=$pK8gQ#M3c#@+wUJfJglzGV;@Bo?3QBEf!Ovm5|*vdzLD-8I2m z%?oEeW$ic3->>0p;tl5;UoUXxSVy9Yrp;V;m+3rdQB|S9eC@2<2wWy)?Q+jk@TPc1 z4aa}s6ps|OSh)%H{=Sj^(B0dQzJ8Bf=hf`W`0SgDtsAw>8`+j(+Xsf|4h=y9%Lhkc z;>2QCYtW?5>dSI^dg*Gu5eu_~v-H)}+`Ol}8oMlUvx2v(NFLF-$|av%OU3sCpN{egn(6f* z9D(0`T07^-n@hq?gFJq;5ZpAjL9HwiQjdTDT{N7F5}sh+K*Xpth5Qv#HD=&Nmxpbg z7BQ%9n^gn53{w6*$d)#0UpFg5pVSYZ8LN0qKGD>yOBZ|G&`^bVN*=jvOb4yJmpp$= z3CRpE0eQ)Zyltn5rc6(JFyAv9M$E@fGiK~gF|G<4Kr1~P5hl-9&wB5m9N z&{o>a8m=}87jPiFGgx?mt$(7lUiyE0tJnl&Qi*`idc&gR*Z9SNx;#|L z1CcvYZ2bZ@r1(TaedYS8m1skKoWkUV)f7vqX6T{ZQP^G3`pq30m{!xEcCTR>~ zRO4}mLzVDK=~}zfF^kHk_<1wO1s;_t)zu2qo%7O!@20-8gdlc!**#7kqcB&1*>%m& z8Pclc;)%EFQ;pcV_Z}&XeBk@_%jBBkWayw&jrIEjGBpNkWHn11#XJDmx~L~+fd^rLJtJQ!y13fE6djp&-k5U zhziG!#kA$E?XAVN&6}uJvo1O?-o|O3r)gf1x=$A0f_MkMFT@yKwTw)?N?H5 zZo4^Yi5V}wY!+$!WsOqdw=c~)Ys=bp@3cO zx46jWCa-aQwIXlfT8qN2DzUdIZ3FG&l>GcWZ=z|mWC@tB13+ID)La>5-fg$~)9if5 zM;4}9!f^~}>WzPzuyCw+u*f+`A*2FyKVl5tX-z^Qng>xcl30V?72-m;(eOW5fkPN{ zX0f)M(JLp8~?B!6Rq8Cc#=z|o+Sp6jx^o*v?_ZlBL#oJzT?Ob;!DAsyGViphGZ(u z$G6@Y1CcR0a7H67VJElT4(8jet7PHJ?*(c~ZvFW5g=EKU6#Vw<76Dn1?JgO!TOgH| zIEr|fz%Qli*~$OM8x9oI&e@#r?#Cu*!E%=Ms_ROWK2A561!?B61hRh*Ub{s;-#0Jb zJ0K^Aiw%FPgk#zZMrJi4U)04~J@!*89=UjJKj5y^v|b9fG=?J@(tRh>hvW4_;Bt0@jbnd7AHFpN8o&S-oSUKuEwQ{V=@o*d zWomaqtGFe9IqXv?QMmCl9V-x1IS^~dlQ`W5o`x}*@*X(!3ue}coopLh*}s9Ul*t%% zU|xULg!`S01h>6gLuY_qeoI@%F4>&s?Mt$sZ$nA4O`Q2p=s#_C)p6B|C-$blFn`?Q zRCDQ!y$&Svd+x`-JhI@P=hS6Lfz`?g7bdNw7eL5}+#8FbD#$&@i9sbsDu9<-;|D0mj2X)U zVtx*U39Emj{IUkf>bJ6tDZMu?cV>;QrY`p?B3S7^kF=d$zEJfhtk%= zSQF#Pwnc=8woDKxN7Y3)qvQ8fCBPh)Gjo4gFObeldhcfm25Dj9j=>Q*mbn5RP(OWc ziL_P$df}@_>idlrq^M6}#k1-oaM^#L&H~5aJG*A;T$%ub2W`l~c%U4)y$(3y4~ZkG z{61aeiA&MS_HD2>kE<;xl|nab@OgQY36D0E8Hw5M8uLqx_JbLzTT_~AqG`%Bsw>hQ zrq7Ki|I&NbN(_)&lR<+t^&I~6ylT>)WH)dLstz={Z1JEM4rzz^_X7>?z8`;QPOSP4 zq=f;pbeaq%3TJ9!_Ti2qXbsN92J3ZY`jW5B)s|Byn_M3^$(5@GPi;ybk7kSAPMyzO zR;&X>A|U`Y*OZYDN0KC+J+weUy`8iX2J`?zy`-p6kNnV_-!ahu^-UjxbvXBW`mRET z&r-(`i|8ZKwrDYRg^XqnCenYhVpF+FI=hNHvDdUH{o()c!Ur`QC zX*_99Z54xDOJjbD%zi#WX0drz+Z~{BdeUUQM3e3#4t<97O0#;6U&CN}PhA^8c{$(< zJ%{XuqQ21rM%!~XjhABH?zrbJ^OGYc|2Nbl&NC1tUhPJ-qf_+8p4!eI^Lly91RV|r$`xZJz zmkpG%GLC|heT5d2Y$x1Tl_^Gp zSzHlf(jb2iw0e(_XA4ZC$4ViZ@|t3cgc6Rr$q9soW9un#0VASDM9Wd4MiP4|Kj=21 zYz2%nAUp$6-J$eD0#f)Qn(~rakM>a=p$I^9fgJH!gHZHPZOJ3_NvXgizrzT#48cZG zr)}*xd|c+Tg5>U^HK620=Fh{Isj#V;G!Y?AU`l^NGuko!WCRqRh&00+y^|q^y26db zFP;tNk9EVf$F(uIudrDbTR8s=#(C=&+WLG6|KNe!dW`=Q$0y_Oj!!z;eWwpWTGJ45xmnu?n0>e9+GM*y`lD?3|zfI?eat2@x=og3hQ3lP|!boNi2vkc5^ z|AakNp2OOo7Ip0z340a~AFrnXp08{sFc&7#DUANhM@URqEhO60W1}Z?Y#31SYnCzV zj{URRsF%@oG4^Vh(Zao)N!nn$RVb6gsbPPEbjs1{GQ*=qu$o}4X;qSGQEtLI>ONa| zw}f5VeJ?YOp2ZQFUc>;7TFk|Mq@~v+65qULsW!UEqy-AnH-x!VTkA%*Y7o6z^Npn5 zy;s-ev?k$!$Gi}ts7t(3tZ$NKR-%hHLb6>9$~`xe8iE3HZ)54ZElox0rwHmWEnk1C zsXh3fmV=eUEFi?VPfI~%l^uTU`QyMHVaI%d_94LiX7e4GHIUBg1WptJ1-CawXB)s_ z8&O=|yKOF;dSK=5sE@)?6*_bc0tPXR&5RAvp7a2AAU7qOk7tl6%1Fz<7}Jk!1C8W6 zikPNxP+}^Nd5%mBFK<#1dV`MhK_h?9(fkUYhw~|<`R=^axu*Fp>!G7DlX;!i%nL_} z?5|&VKEI;$&+q-o%>52Z`@fLh$w2padnyC-@Agy%#y@yK8GdI5{uk0a8R-ASmikA> zEW__~qW^EvI~kb%T{|iR^MA&U`geNguOikz&^rIU#~=KvxPSMnveJV8E5(2FKjBsV zjpPabpYy5yo#grJJ$l;z?os{sXr2sz=giUw(c{wp_k}F-nWm@?`chn1)V8OnuB?;41FR^iV$3j#Q%(Q z@&AJQ`EMTx2G;)!{d0yv@e4w$hK&()3YhLOe#(D?Y|{2?VnLm8QA`FR8)Fa=HE!4^el{Qe-^+iPSpL}Fuf4MTk-Yk=gRH-ejhT(X+w%_KQj+ue(H>xZX{cmHV|Ml75)(|r@!|!FRjK8fbW?H&G zt^+#e-$}FdEcA50%_TD(>u+m~^?#z-{zj1fwMc#^$TIvZUi^Q55M=3DSy>qV8G`Iz z`v0>2k^gIF;bix}ll%L3{ma+u-{k)?(=szM{tx~yBmEazI!1=S_`m-EKj;6l zFj4=yO<`lCX8fJ@i~C8~j3;ciJ-7c1;|>EszJA}1CtB4Dpl%Wodcq~cya_8H zoD#z)3*Yv>e3XBg|8C?YJyp@ao`YK8w2|jCE`T5VEK~Blr``RwinHPIyj;xF{x)`a zD7_$U-No@bhI4(nh~oTA@bUh!LzBI;Sv>{K)5%qw*6hQ>h2?$2WeLX9`dqx3J*Az= z^Gt^Q41Lbcb!azYTh5gbaOh8?Zsp1=jc_wzUG?<`!tH;Qu(j^xxw!~HX%lekPnKBw z%Of)9#eG@;2X$6p_D{$#Pwx-lPsikFyz9ck%E1(kL!RWN<}9_-WESfwp4XkXk*W7W z?XD*?laGPR$?mqdy}SG4_Ssc=j`u0y-r z0~3!OV>^G^1f#37*3#Zw?;i=$(4TLM$H#lpvSIfVf@vEl&FXA(R1N53`>QlGpRUid z`|;UA=qwsuU;BvBoha6ywKOcjZi_p22bMX#y*~E$(>C z`oR%)st(rU`k4oJ5^M*6^AehtLkEa4ReXY6!s=Amc7^{*B-X1_jK+ zOkIDJERj00<_CvRjP#k1%mfY^o^Cc2-lK|`I~nE+3<|VIsk$$+tQ8PIhd$7AzZKB@ zTP&Pd?*nR0Aqh%vxDYMfHbxOia`bX_wR(RWUvj4vcmG34{btt2R`w^CC&$N~ciot7 zacVf{3V|UrV0ecUpyzx6+oWO{bu+Raw)}sB##zsy+w=IOsy%F;668XY45}vH51`s( z#sY~Ckc2ID!q`J$fUc{%)*fLi!m4{<%&uDk1lJXL5lYK9XaMh@iq$48p* z=lfJZ{r1W=$r=wkn!ZC^z51hg0l3WSk|W-nZf_P8-XS&z1IEU!V3nHE`Gi(({tg3yAit92)CuVZT$7{>4NqkjPc2}H6@D;nm!`wYjo792yvw6PobNk^l4P}Y z*F0xMd?sC8h$AW1)YjQg9hT!7N`7M*eYPUa9Hp75*yndaKN8K|QcCE9ih8Q{kv48lf#yS(IdtIVG+7 z6-7IB8=~@yRAtdOZRPkZT~#c#jUC(GbKT zT3dHO9CG{@?q3HhQ(m^~K*N8kKqgtX!mMspz$*`+4=#Vq|UxT_6bvE9(;j@!z zG0PL(Vl$CTzCE6a`PAJbilJgt1XN=5Qiz3yE0jw$27nQVKy!A$Fr`?9C5-akP;&^5 zMyU45sD1h2J4HlB`9r-Cu|)!R%aD5dw4ogKyOb}iljPzI`5MD_72tm-Jf*dT$!IKx z3bwr`rs!+Wf+D7|CPP{7#jXQ#C=gnO$O?sERL)u!~GuDv9QrcW38%uSE zBhrtQ6X0wHp-zIP_CRleMt@lMRK?KBig!G6(5OhPZ2kC%A_ z*r(TWLe4z7E36aS`DTCEn;a~lHQcgbh&LBAu;{t=jd-?=UY;Z~zTpnYG6N;2k1zK? z!BU2h6SK(y2bPrHA}iKcB$11iG{5?Fg`+4b`-*HY-gxpB)F7qfMfun>cwSm0vmdm~ z8Mg*6Hags;=bVrT(h=sSO5d#e-#N*d#kbpcX0 zag~y?LOzowBqpY~_C71#)$4JsK z1K9R!?)HVFD+IEa9If7FfFKD#7d4&hmERq!PH^BfOTsfK-Dv{hT1MPk8VFJM$I&Ib zqg(ZJ1MBtHDK7WL9SbI2JNT5A2?0ugic}WJcBAItLt;ISjL>1HUKd=4o=B|tXwNxp z!PODr+q-`ffJX@^Xy6+6kMw#VxL$oIlxfX9WLVEQKoYAj*s&-u+WY>uf#5V3>)P~nw_j^v4{DZD(VeQ~0(?$s^-u6}OD_S6H3gh! zJ@j;RkI1$hxs#CfW~_`KHIw&V#l!bSgK*tBo+p2@h?dyEWavQ;Z2E5vScNK16!M%E zf5!TaWp@<*02(0sIT93xO=8I&oCI_lXKPz4pE$4_W{=K$iSKX#U!Pn+qh}#3B%gow zlc4Xrl)~Llo5Y3KFQRDcSek2^mFrlq71+sGhfAY7@?vyM@)&f1&Id1o)cXm9!|BW_}54|5^+c#k|l!06!*vg z!;G4s?es5&^J4FTUkwstYOC$NXDzvUVP^bt*aOw^D3Sa6jQV{Op4`F^7uMy+)>(Yp zjjQHQ>rUanOZLDkNi51ytTFU#YLayfg^+)tr4KU1ac?ByK~`6ZOdyY=2-jJQBo5jd z`fR+e5_aMZTuX4oO51jJq1}6qF8RNR$cXRnAU6mKwjJ@&Z1`B>oJ1C|l9Q`A^wS&O ztHrt}&Uq%of1#wCu{+ZdQ-@oLQ6gvn>--|sSp@a#{97YsN4A7%s2*_QgNa+Nh0uS0 z6Je$12ilXeA*Q>_)wtt{(C>cG=0twZKD200YnXV;)ELL$J_Hev+=WPjGqVcQN9q|< z2qtF6+I_3yWW!?;`!rMJcTG|Hc90Sl;W42b~~VH`&nB)>kyu?%Q`(#_RdsZdv$ ztqF&>7yo;#2p)_41o`KlgEwif9w^;OE}iQV76M;0EJDtV;Rk0USBxpl@&scVrW~q@ z2+JW~*>*&el_^JtS@qBZpLoLeVb;guzG=5Eq6LZf+vnp$oa@dyjUjhK9yWhZT`g#D z?jtUv2AtT;G1XW*o$3zTg>N*@TcyFxli;+C>c-zB2JP8TmtX_x@yEbb*(hQzD(m%% zN0!+sc7P!?b}QEA(;SrsGbxL#-zzc{8PhLtU(~Y*f|0eF`qwmVkLkU$RQ zBq7P5IttY7@#a+GjSe4}Cewdf2|Bkbva`53-iMAi6N9^c$nZR!66A5~l{gE^by7R+ z7eHuP#kc@iWHLfw9}SAiW4KQ{%m^5XvYMWl*ItSofVDcx8L6|tA-O8HM3#&c;O zD>FrzLf|)qE^AICVN`qs429$tI!n?yqiD39@rYZ(7_9I+)JG5pB>;bG%~ZC+npKF_ z+rOfdNPfG1`$qD9mM?y#!B~56jee0W)y?t#eD`p7eXRbjtew?0r0re4dwF@bpCKG^ z0g-Tt9f%|2FgB#$H3+)D3!$Y2M7as1=kqRw8)b;8D2WqI+YH(N)OzwA6Ar`*z*eB6 z6|DcuttIZket!D*j5UA7Nd1M(kdR(BrTSyRyzYG)JCp4U@$q&4Wa^$E z-1G^V;7D~)c0nSw$894uV%pdavOAG$g=_{m1^AE(*)R?Ui9&x>tu#rBkOoHyJ^pdX zHmZZK{;5{7IddzP2V!fj7mg<<`W^Y|9p-^9Lt1&#x~h1-X~9HQDU7+P?fQYrMfDNw z`!D_BWarwj3f+A?pB^_e6xnP&A<3t7-z;L-L3qjWH5x3}bEE-?b~}EW!@y$zQ$Vc0 z2_qiBm!K$rzdU(PvoJY-UT(N$Xx}~*;l!6 zeDX+@9B`v=w<=LwXkLGzCN&!zAFD(nF)##ylNLJLI?CB0|~DgQio&?nfrjG z7x1d5fccU><_kq!LibW% zEe4C*eP=~|SF&LhcIPP9d}jrRG!UXFU5ATeGl5X&s^mJ11)}H+hf&>h~PGfxRYuM{`?bAp4Jw6 zy){`fmb-<_j-)43AV%Y-eG%-Gg#2(Yb_#wzu_#?PzRbqjmgi;B5<+}4OX`>>frh#? z#P|WsTs0CjgIQ+s9L?0)x*%LL6R|_2uw&Joux95#llabm7Ck8-ed*#S+6peR_uG^l z$fenH2c>rKK8|g>8~%lLA0l;2no8Z!l6y5lYK8jgdT|+dUhghx1N25-rx2*EDW_!_ z-6d_fHAtv}KK+-{qP=?;N;(;K*Z1k9(M9H11Lt)HopC!dUKZ%L7O==d_rWAM{`hPw zJoFJSs24terolb?$uTIWsdeowrjU^O9nJA8qVQlrw(SZbLX-UYJjioUp2Ev^c z7GSi0C`4ztR+N4ch+?r3{EBv$a2v@BgcKg@Qqlytp)D~PkS$d==uSa#?%a~(Zj_JTau|!vxyYRZ4z?r!N;&`+fbiC@+Sui%IE`Al$E3_>Y+5zqNq$x0`KEE{=>G@` z)2MTr{k9{7fg+-~8-b+02DOAm%wj5kW$3xoyZ0rP z!5!VF5vi`mwxBr+XzpGK)OqevDFVJUbSWa} z5uDI2HKgr}igg7mMI^E#Q?ZcDemhM*xf{erXkww{udnf;&z}%&9_X#60;<1%lHWpO zp}8o#hKE_@ZxCm@zHi8CN!T~y?YKgmA42eG-8#dHAy2D3TSL>LOi{M;nbb4X|tfhTCbe8s^ z4$oD+s@*=*gF zR-2z?Bjg5#iv`2<3yT+Jo(b<0wH;s0;=z|N9F3nJii4db#yc>ZdOb&fBNoV<*xR0o zPJjf7A?qzOTiYBkOAh8|abNN^ZA%1YRS|Y6*>LFe%cy`wCv>eC!!&1<5nb9K>}p&)Xv=j4a1%y*?%-UA@Eq%DuBb-i?5D#j z>Y)uG6;f&#Zb)ZAcBBP=*{yb@_bDbqu&;p!ALjC*Hxf|}wmOsq31W@PpA;d;Z;I^WmwD`3j_mM z5IuWv%~$rH8uE;6z-8ZI+=b$czX;_fc4ny}`jrHUxE)_V*$UN(6T5v8*7aeWV-#kO z1fV?i`H4||&Q^ba=lfvrg!;y#O?DLoEgqHK`(xZpD}*+Qe#UqfeH#!gCUiKUHGmN> zNG&K^euUUu1hq&h;KfA8B9xn@GbOaig7}*|J55rHh9t~43eHpj%G{w>`0_-JNPN;Az8_h+82K9fzFLmC;thCz9mrUhaWEJ>Sr&#uaLF-W zm>9^symdQLM6Wxv0?ujCu4D8M!|d*XCo>!Tj)BsZ44@5y1fn$6m=ol8SPW~n$`nB; zI-CqniV!T+u~QS~y{=S)O)Yp;8iv~<=H8%XFr~{LL27!pwT)bhe4s{e zXMeQW?ukNw(hQsqs_ax6by6V+Ix0if19>mQij)%rgf489=_ctgzx_avUPrZ8a7DF7 zzXYj{&fQJfzBk}db?WY>y}t5?O)}KpqpHUHK{Xhx3xw0IBRFjK-~?Zn=93XX8Rrv_ zNjnfuY=FcGrfTsxE@Rd$!l%Td6+wvrra<|ThpM`N?j~2)b}Bk@B85ycrko)Id=S24 z;CMPPg6zfQk3;Y?qr#7bL_HNc>Ec$*h<*oJ$Gov2vi)OQL)_`yk zm8`o=9eUPYxrV);$9!e=-YIJv_;^ni;Vbz)@>zKh$@>fBblLVmLz&c5oaydzS5T}7 z{`uVkVR*9Q)t+JFMSRLWns(LPqbPk1coyP+najXDo$IFT*&VcFvK42kDg@441w2r@ z+fJz^{gr{mJSF9zQmO%6QyM~M#Q6>BP2(^)_-^o(ljttF%S?;jK6RH;vt~f0Z*@SW zioIWrVkKZxZU|nw+LT{y;}!36Sr>}2`L=)kJikzXm>Peq5qQTzTp5v7O({!2n1P0W z?oHP|=ykh;2UO4VyQis_7DqE6cZl}78+`Yhh0ugP1zblpwhes8s3mewxd#otM$Vi$ zZ?jf^br8w*SlY&U_7D7?aMa%LaJyUhj!RO8Qx`Vgt1dfngpR+}plBxGeG8)!y6PGS zs8M_8-r4bN{i(y#_gcn{d`ZznHu6e;?sC&*r>@aYv;WDG`wV^gJ!MVw z?Pg4(>&~+Y`MSIfcD&jT<(49{IYpc3PR}dx{#W6)=-Y`D@ol*}&B5XikXwpkT%?bK z&;=@&njW_fQ4n>PLw`k-cxWa2mM5Jwbaev^aqX*~qtxS^ZU5!aLv*$+f0%`TE5ZOp z3$=)v3wOF+{5*il4sv*=HWR)1+eBjIYqQ#Z`dL`Y+OU9dBBe+aaZ2_q%-R(@WYLP& zwGsF+YGUDv9)EnVAV{UvmMuRh51XF6!x5lz((V);C~0z)2Oo>>obhXLjeu&#eS8ll z&QQ&LitzGz+9s*9kgAg7Y{V*mD+p`nVgJXPln#u-5FIDQQ^9lcYvkr~PIKG#w);J& z#BNYK!ixS3L_R&Q?{rPQNroJqa_VzW4w|2n*X9 z^n+@9os}ZwQe36LY;ZkAN^jmAF=bewvLF)wvX>uSeod6*^FB)|G7{bnhH|Xi!iU66PNPI(JdqZ#r{KFc zW{vhq9HmB=_~}$+zy;k=7ZFE0pp7r3mH-yTW^2lZD1 z*r%7PMP5%wH}r0Q`|E5elxw%=`({`t&+7dPT3-@Vn=|8Jw$Bqz?vkhZW0+#=q5azE zAsUy2zN_Ny?MrI@40lWPkN10FoJ?=~E`kY{367*i@3rghi6N}>@CzD`egStxw4{td zti={?Fgg=pe{98jDDrAQ?yPmzV7K|iMH=1fc9e$;9Oe#xt=ao)*wSOZ!XkLjKtO3E zZTr>4WD1@xauuH-3w6%2ual?$WN3Cbn7ks&R?75W}I zhtJSgGT-D&4+*d_lleS}_j83w)BGS>RkPfE^Y;Axb>p?&_|j7ZHa0y}sw(t(mqf-n zj0FC8Trx|A*3;-x`Smm7T4r!Ml;IM4wb<}ZcaSQ7Vo@@ijkKZW6A%Z^lmK;#x-x9; zr^!x4 zrYD*{XEe5aJ&7xtxe;d1`b?oN-SmWu`4QTGP0Do_N@-+*jC?^8xtJtTW9;*s0($5I zT_t6;;fhRqNZLW7@~wxKo`}xZ+Yi@x3^ren4c+*}E!# zrZux~NV{Xh(w?RLJF?i*n94-luSw8tSq}F50?!cvp^i=TTBz$4Dm>(L_9x=8larIu z9Nr#dLy@1AmzCa^gB{w7-U!>p{Z|C!0#PM7@BP-9sG{j)z&PgS3OyS7Yt%-vAYC3l#nFH_&X0KQBhV=TqF&w2{<{CW<3O_Q)3KY-M4kkSSi&+tT}LUlc^7L?kA!XK@8Kha15*h#mU#8Qr3Wf)wUZv zJw^mCOiVi_iyb!MX|&zY{z+7}JEQbdzE1O$X7!jhGkz}GxD8h>u^GRkC4oYW;?ri#biF(Can96y@vh{_k zGzAbsmPZK-gZHGgr`N1pR%m?64iK9C3bGZ}4CF$*t|85exK?#Uu?wLiw$WE&pv}{A z8<7G-KqLGt?B)aN^d^}bAT+iYX-#KF>jHp{rmm#9DC1p9XSt0H!6zYqnz*Nptl@LH zV51N1YK>b3bqiM!e4o>7zF(KUzrI(odQ%`ZtYEykvvWXz-!rlmD;725qi&XgFigEO zZMS}1mInuacNf^ae%7=aLOEoxH^P&K>bE$Eo?PBd3MxKgaT z*#;bEiI>LT5FCUyS4_@-vDZA?gWxD7vI-}VI=W8STF``EygwD@3w3qnA9n22uK_hM zZRJO>hEQoBl*V^(bn8oNJQeI791NZ>EZ_N&%CPfu@o@g!c&coBaMDUPxxu{X@UA9P zjZz)+fwg><&!&Z&9sZ={xaqZ~e=S!n`@5Z9oN&0g=^pV*rFOf2k$;;5Z-?hYh0E#N zqS;+4gV~xQ!ENv4)LrwFXRTa}7NmD^dZ3&q*d1>)LQ-T@yw#_B*X}U}!=__(v#b$J z!BYD&g3yc3<6720ZIs3ZXyBTw&>nBLc9&Oc-~B0sGW8zOK`X*7aBwm73BwEzJ#E_* z<7dP7*ltqTdttzTvG*}RCeN%Z9ypijPd%MrZ{aPU&+oWodst@}hfuFZZV(=Y8+w~! z*7??zkEWYOm&BXY*5y-*gkb_S;&CKwB#agXplyJy(6P%D6wW4>-=!{oR*Y*Uu|AT< z?k+Y?Fu|-b? zW$2w6T^^5a!3Xdqb41KLdstO@U0ob+=v5Pk`Q_@e%~ zH&3P6=5TX=F+dWasL59A@@OzmN#3>AWIEfV%xvW$Xih^WS6#|u=UrK1z)-B8HMWug zW$vG~{Kz5fA=%|}tHRm1W_xhp()jQqWvt$8cWc$*ICLecQAi)6;alFg8IWPQ*(KUC za7(74i(SjJe~5ayZ#g$A9bS>c*P1Nd$J6*#VVGop;YU?FF}Aauw_C_1c%%j*bKdB! znvd{qR7ig7VAjv}+8do)Zu$eiIJEVY+etWtwgcSmmU{E5&t(m{un?ySsy(FW8P>8Q zbh8|_H0j%txRZ(WnuJSYo7VNT9$8d6?aK`* zOk%1mdTtafRQLyBZ)ZfK4MC1#OP3xCxAmS(!qF*p-FFPLcXC5$iuGR0+OaRzBgW~f z^R?Ax400j%W4UYA%j*s3He+FJU4td|Y~9~~KIPl5xr(oO(tlh^b4QY(&lWI+QOW_q z5wgr~;D7HE4ckD}PY@lEaCf|ROLZH>?6n^CY=|yR&gVBsnQ;&^OdHe=hUJo!Dif8L zSm`SmA(n*4wyRkrL_o6-xj5jQE|B^l*BZ1d4vk=YBCVZ97_U#~0gMT}tx0~WE zy|;VN=3y4mwFxr2XltCfno~YV+Y>&aJ8U~1*UOch)ElkuIGXFmfu7sSc_f<4#Cba( z24PRw%Ok}HMrGF@RjT$=o;Sl|kXYJir~eh(6(8cVnP~Kp%S622ozja2aEQx)<#CDo zxvIA|U+`v$8`FF)I*4NvY&2hx6@6h7Y{xRNdDLdlG?_Ech83$@hPA-vYGV6F2h#dN zB;Fc&2m{1zl!cGAW|Io$uK&YRs&zBc)7{c88^PVuCDXsNsz!Lbk?>6%->a%deLEY; z>HY^#VIC7&bzQcy{?bU_B&t(?%|{fY#i8yi{hP#gv8s!pZhEgYViWn>QW$a|RXUbk zc&`x%bMSp~s|?LhO{gTZYi(?BBwa^;xHlOV4SC1a8~3wc@9l)(&9kRVm#noHIca+M zPs{}t(u%`o{mx!{z}hINMc2Memn6yT`&~4Q1$y&y#ENa*$X>Imz7qp~&huV*-ZznL z(LfSJY-N?gEB-(nPUHk085hbBVI(DT#=xs#_P6|A@W4R@PKyD%Ktu3NWfk2K=bYZSN)?T4Vw547?4UMOv*hisfoUfes+o}{kcxDaVY<1!0D;foiG|FnXuX_) zVX&iP6?Gc9<6(6*T55WKzlsx7;K-!nVjW45tyD7tUMpLqb49L|tJF8jZKl>3U88$7 zgzRT&g>z|xyk$}*Sl8Ipx2*zIwZ`wZs?bVrs!)w;bcD#ki;746P>VK?lV;^@i!o+9 zQrHIwiivec4Vp7{diF%2*_IDV~$0y)YGy;mf zIHR7GkL?PcikNpx!{U-J9?lwz8ReBP#udyB#u&k*VhbWXAo{jV#_!AmkF*xC4qipi zibZ{V27So8zU`MU+g#s|?N1c|q@~7350CCxjzJ42p9Kyv|8AG0r~i*{fb?~8 z=yh~;3Y%hqzF+`yd;z-og7PP=|A`rsp7Ec7<;c`p_2I&#}o>Ny|3lS0Ds#z4EzZ+rf2=x{@Og8Y%NqXh6#rVq)?Yu#g^)Kzw_U&4Y_MqCV zdEzz(a(+Vm=UMGG=R!bQQ*1C#NF}1b)I|^baEf`t?-<}P0-ET zz&Ldj)z)9w+^+wrz_Wic8dy+Cl%H4nA4w{F))of;g(1Oz4=+^4rDOd)B>4B(!v7G* z^IJ%N@V_;re>N`ozehd(PjSKjJplCIjd}dHxu4P z!e;+Vf`jQ;{tz6@@@udEFi6&a{hP?YLg@oH|1K-*q;;c5j7=x!Hy!i-I0;1-ubfDV-Pg(4-w)?a=XrEaEle>u-(Wu(JC1`kG8{c5m&Q zON68K9a-?`M&yYN4@q}7$?yrmbmOXjA0HMQAE+je%!U3*d6CU$!LlTshr;Cnew` zJO=VxQ)5<{xZ|X<8QqCxT_h>+`L>(`L9a--;hlP&`-S*MGWB73Cz&3Jx_NfgC?<+) zG%b@v@JCv;6591O2kik~qBU8Ud@HF1|4 z94?u=2utnG=6{0EbgLFanm)m>o*nvRV|?zYVvtrk@)UjM0(pD)zc<4VG$P|4UZl-? zK-Y3F2Km`{2d;ROvAuRDBUIgLPx6v!-JHI7d&sQ0(%Zt4fD!Vx1oPW};0VF9gyLNn zV3|g@bH);GlY)Q^Q8WYM$Jzv4Farf1sha_xo6VgCwH8kCRR|2E(jVtceC7j!n);IO zl+IcNV&~e57zO;$g$>Z-)i_syXo1k4(Tiq|U+>Rd&i}StUr!QKcec!66jujUUtb(o z7rM-Fd(Cb*W6?30{mx2%md;5Aa~f|fohCI}WH8lFCr>Q-BkOpXpV3A?+mTL;$bb7)lTYSe8eTQcl#4!wb=IQx-`E2vd$a-dX=YaS>YQQ z3!We~d?e;&;F?Xm%8Y!w`%h@_rIRRS7&Yy$ucTQQ$Z6fUn=z1oaYp!UQ%`ce@EY6@ z=a1h$XF4_>Oe|-~sP1Jyef?BmFXBJFFwqWi#x9#rEp^uBd8#DcA2Qp1X2z%&sz|qU zHDZl&sob)`5oV6hIfR^oAM&MSF9w*)fa8>Bhc6mr%u02GXK(s5p&hbENv0QKq{-lo*H>e}T z@*|Y=(c_B_Q;{uBz3(T2T2KT84TlIL4)p0f)?I#0$9lJaBqrG#*4)z`9j#5zl?VP4 zOcV1aJeR$XFwowVGV3#SCg#r&CUW>8soCC{H4cOL`l7uv@vxGwom50$#MzTTYFTR_ zcOiS>LkxkEBq%utT`LN0L=Bn|C$mFUiuAySdM(!}Dd|*NuqXOvu*cq#;soU34rg7k zi8w}x+o05cTbDk89-aa`4bW1YkmYie&M4G|4{ea0;;L)jYW9By^so|&8ifYwYQF`CZYQ>PPW`KsGJM%K%;2`hvv*jj)*#;Gcp<~D68 zH4UK}x*i5s;%EqJR(wQfh#{06vUe7wzB&E*BQV=(cY)!(($md@WwknotG* z!8^&PJ=d#o&3={n^TalUJ0*%2ePN&8XRq#jfpW|x@(Ocx^`;EdeKuz+Gl=S}DBs;hCG%={KhPDcUo)z{6#4%C zVm9@Zrzl7H0c@byN+M;;Q_g*Q{9M6aYE06--cy3=OuzZAct(=@y(u5$?4WUUD*hgS zxjxEAllUuexSOer;`f6TbUhR98PBsE@^U4TjSKy1;>A_ByO!XR$a_%*WWG+t{6AgWR^;136G7{LI(nNej^&oMre?T5{qEmBC|)ia&F`=R()f8$ zzyk|KVl8==2}6!P=fsU+CtU*&_9I5°e#la%btpUAWxU4>|aMZoA-e!Q2uehMoVkuLWS{9X8gIN~`?%rS^LFcCQqW zPZ}k(K$xaMJqtG>Bcz`uvnV06DEw;)&FVs`XVg5t8e<+hkeSW5QrzJ3NdK~oRR1yn z1{`dK{b3JOy02AKj)$kwn6=C94LbJneL}@zJVymMCW+#W-|oM!b&TYH_fOB5tOMBk z-WjOahCrWdhMIMVYHxHs6+*3l8U50?u@XS$p9JkxE;2zLkbFHK}}I8Wx6yHBd{ z=x7OMgG{c)zfEvNTVx`CO?C;Nu;g~!#=bZiBsDYG0;7iA#R^`0$(}u%txnYBepH(> zb1l404EV16<*RTA#bjmM54h4~1Xct92KD1b^E(1SnN)v9%c^d-TeHrWdq+i%SPId( z@~P0_sBx!5flvd^$Q-F)R3t2vY;@lM2f;G-p93cARm^SZ!NK`NEvm82otGnpJbuk z>>;9UZb|uF+u-- zWN8Z}XR(Khru%+jOdT5hwp@Q_N&0Lkc(5L)`!mgdvly)e%)uO}S!Pu>y(mY*fPNZ- zav2Er2XS6!j0`sJS3!Bo)44$g$23i83w@B9oHm=Uk!%6SDJZ_jA<%{frwg7Hi`+sv z^Z{aA1s^j8CHRVU(2d`|@pCbnabTLZc|_F}As+W01r5l7VB=JM|ET);blPKa^VQmd z){aGg>p{@rW?5OdBEp-9Yzc98V?V-H_1>g&IzTn~(o8!ghc)+@=RmCmCXiBxt z02+BNWk5uhP7blcG(s2!jW1jzWU5B-E;jCmNOl{$jbX*)ei(QN?({TueGzSlSGn{{ z+WT4=jjto#WdJr9#|}IEhm?S3fmLzm!bc^RjDDMGn*Y1g5>=U2(RWJ=npd+cVmIw@YQelD8HUDauK=}Qvjl8*aYpoD6HwEo z7~gW=>MLe1ahKEK(7)d}EIwd)6fk`jLZ~8(F`Z`;+#%mG13Iset&i=(177CIZFt;uCBzSR%rm+x%SyE39OmVR>b7HREa(c#;FLj5?5 z-@;g1)MLgI$u@AlI|>pZ_{;Pu;DbR;acS0+S@gWg;gO#QjIcM8e9wi0#`` zG%MA8@B_cO-5B)c^oXag`DR~#W@@F^It;vTXgnNDCoehA7LB`@xM)4>6+)oQd<}k> z^@V^4e|#eL(EQ9?G3|4b?9nwz^F1Qu zK{YZVYB7S0CQF{S$}}lvTHzj>CH~!+*>RWlHoaQJV>2ioTZ<*O*Mu#9Ad%uB+0s2@ zKWTn+6x6rHLbE?%3pPKDxU=ipOg~e*#}>{>3qhIkC5jZIu6Ce5ly~i+k+CgoXvwRG zJK6JGUwuiV&5ye+=;X$TsY+*Y8HUkkrP#XryRx-8IVM|}RSv_EL|d-+i#E=f-erCC zlJ?STZHX)9Sh!_gj~xep*`u&O-U1@1Z5b!Jg#`xG&47FZEA2*fh*RK`ctvP$p?E^w zfp`OnXBd_7&~kf4%49Ndz1aiOYntNh2V+9e^he^EZd2+dzFH_UiY2vPuVD?iVUAAJ z>MVxDn4(;jwuLfOn2)PUNNaOs%Uz!TSY9j$Zrc2S$xrhJc7n`*&dg}G&$9#>SxYK6 zbc_Lhh+4o#*CVwiv)w_(3JxyU@LlKdP7)$!&zhnZlZhsZY%ix)({^u9uy@R=3w-y^ zQa3>m;ALQZ5oXx{?vy%qUWcSpto?%lA=$W|B9f}_;Rc{!aF(4|z{#SJi!tT9OSyNf zL8h33?l4|Wj<9Nf2`y6?zGi#p^|(G_0(o(`{!$L`B#xlIu!MtgD(Dpbyd9Xu;qp6o z7a>&*mq(Wo;ULt7z-q>Z(WC z#BSLs6~CO0Phh0BB-b?`9a{u#^fjESA#147DFIV$!71YI8ta)Xk!> z;5`o^nR1yOC+^Wh#K$KRjg5u#gd7YJnHCMbH5_W&vP)NyS{SHmseYOn=zT^LtsYo=3K(Y`E-m4VwU2@-Z&{=NLvyvN9u0HEiMh7znI?C!uU9xxk4+r!tTDXPg!fHlx07 zc-#O^dR4YnThw6QHo)U`Lu*!GICK+6XYjpboj*LU!s$=c94Gm4P=TRJ>7|PqxeH@( z8T`zCfbk4HQ7R*b*g@RRz%$S|b4q4s**#w6$l{eI`e6Y-+@%WTh}ay~mw{ZZEH>@z zj58R2rIV@+i~b1@jc_On4W>?p_}+N@IQ;lEQCZRH&fU)K4N&WX#q-D;A>#mJ0RsV} z7KS|fJcb?P0U8LJDAFiOClq0-aO&{Kfz6O;lgt(NQb#@|0VRVZ4hl1+{)%DM3su>0 zd}KjtJDMIDh1E4&8P>Od=ZgVX470Lw_eJVW&$eS^BrVPCMjrc_ zZCmedyn;PsiQ(=iq28^c0S^26Xe%~{eyx@~`@_uZ>qrgvY?Cg4#TaG0f@YQo#OeLK z!r_iJ8GQtEzZlV6_V&~}=Q{{R47y9}TAQ;2$NR(6wUU5R-SFPgUJu>1k~Jlsi^U9o zHHqzLw(XUlC=6OWUT^miD8bOn>|LRr5*)IwbI5d#&cV>ilUbcJxqXue{j?{aY9?WLNlvCtoGQiOY zmnz>i(8Cgn!s&n^SbN=c+VZ96!{XXb!KCYM&JEwy>* zM+&+)tZgG`ww;-7#yS-P-?_omOr0!r*BUb{`UtGie+p=U#MEjf7s$oe@fQskbP856 zfqq3zIwcygV1jnKl34tKKuREgodc|>+Q2SdF>``GdvekcLViu|hJDX|5QNHq`N+PDTDIx=~z@BU6dsIYG<|2vV&q_ zNvXhLZ1d7QnqBQhMN_Bt(by>D>DX}0-3`w+hVr#%CM%p|hr(9hqIY&~Z8fnioxekP zs;r5d(XA*6#CdV7bgc~g16p}A+^L%Q+v0!}EH!MsnlNGOEl_2DpFK4tMoEHB`7kr_ z){52@)Q{8^6ngkQWQoKA`KZ-t(TFP~*gnQM1(dP`U|poq^94aFbG{P!KsyR0e9-ni=zhgKTqxrD=+1AI5Uf znnfI@Py9h&E1*VylzhV6>$WcCqPSA+uKZYgfztA|JA|eG=e0Df6t?8FKC$U7W(AyH zxAh8vMEA5qo{$rxRH1U374tbVr(`x>r}Gnvr%we6D^v3(lG{YtULt(~rD>7_f84tS zf6z#j_Nlr{A%yUHB=K&CZJj2L1(99Lq$xSHGryyXWk?x+0dslYFdFkf+BUcn-z=A! z%3O$S`ojh#)%l6!xgmryko9*t38C?dlsZ(LEbAc;>{-lw;fbFPJnim#6Y8SLD=T}(jGH_S}H%}^WH;<^O$LANUbq~$jxj~&heKhn*n2OA> z%~vb?dM?f$(^;yuq%UIt^WSN%4~a>7(v*7w9+F{<$C=(VG7fb20i*o#1-OH z3QxD!bsI&?IRkHE!WjGke0`d~b9C&Xh^tQ5ScQG2-_4z1qGJD(VC^Et)8K9;A?c+77#ZO0P&nGHfOOO+q z8sg~8kL9dsHaPtux5kb{>-qBu~HrkxkssRRK=j zljH#lu1LipjG#@MBWnJ+T~E7Vo#&Re&&&CLY!&?_&c%w>s0ETbe{2pd(@ZZC+0}Rp$WzUWty74CleM4LEC_o9yv8^{LR~o9JG#?*$-@7 z=PN}CEI=+SFg2Y9PIk&qj_ulj-T*-`L$Iy@46vRQKF;e^T=j=H?%@MAo(j}9PYuz3 zkpRce)25G`krw^vL+mN7I=dO_%c{d}7oJC`+Q%%Xb9#nOz&y%xmIRBH%WF(2Nh}Tz z8KDZMB9D=S^_5p9iP*sGf{7LoI^a5yFeE2fj7r5YD4{*NlN7V&E3T5(O@oQoh1W`8 zw+>J52X~kYdo1717LX|M{jNo!!)btjCJUL5Tz-@$L=^G#WsXA0uDDzb?U(QvMX}4> zE`6AbUb>YF=NK)m#mx8aneXOU^TccK(7Q4XfX7l@Snf|uCuTR61Tlx4y^-I)xKVyK zwPS@3OmnS)b|`7S|JZ01mY#t*;?-4`e-Nj^szL;}dUiUL>iD|KcK?|7jU2ImK19OG zbyvLu5{b|Yke4{66%PLq&Zi@?3KWPuRAA^6Zki8q2Sye%^$iF710cnO=D`2$Q?y}( zAu;LKt@j=GqIL=~B4{9AY>kKtl>#`pwg`+H)gfe{3|&{1Fx)`}=Ay;m3NEYdz|tK9)3A)faUy$@uZTKIZQr$olBVHFd94)* zm1Dr$w3a}p!l%Sf_s?g_?Go(-o;6IBE_SWCrj-k2raFg?^6#eg&tie;%IOfCHgE8P zg$MxUV)=<*I-PwO#gguTJ6T6_f@Oj8c~J$w<-^t6XXf>tX~-dB8E1fh2@~u}ctRdk z7#d{B(xE}Q4SbRiuRzpi2m6;;jG!g|x1SmA&FVpp*{VvAh0tEvBBc2b57L?j6LAFSXV@Rh>o)8uY)q6X` z&R;1*(Mglx15;ol`qQ6(5}n9jZ*F&D50MX71~RcXfwr^ni1FGP5E~CWuU~~sEoxza z{Q3ozKVe$`j4S;6u1({)2 zmyEWK%S_8p1xm_fF{;xMw>Bvii1dCG!Vf7u{~gH++D?OIp{8lETzIF@_5!RM=ymGC znLFdL^jYT~tm%1eZ^!G?>ayixl+TA*!MRBA zl)*7790f+H-FDD_PNYpirMBKzA1SyyW<92*2w=VnIXv82M{mV_hqMm#uI^fW($f=+yj6yM3c^5ws4!dp&wZ!)Ao3fvj<}Of_J|qK@!`q~k}Yik^v7!k*y?K&KsL zT%cb_(;X{+Z0$wtQy5b)PpbUxI(n1P7{FPsZKPpDPM54AR)KN1KWH~AG$v$TSUb}& zRyU?*`1<3awQ`vm%#c*I`=_}I4lO;OM5uO{Ska1sFamElS=${0*KNQlmTOJxN`{=W)_Sy$4&+ra( z;!SwmG(-INF1^A<%fgHZ2C#LS*#Z>bA0=M`gk=nxp0UPt`f{q7YGKaY zo~zkThVZUxiT|0cd7zCF6a%NFuEGu`2T%zuD~tOR7Y4f1pGLfIc-ma~=W+Q!(pI0Z zJJsoZfl?mk>Z!yl4=q}J^8u~4it17xt)e^Ft$PBT{$7bN=F!kyo6nPAI>;Z(bJp-%6xfg@C=fXAo z2`kO^btA>{@zsu!m>Bmr!*3{;{8dl6;<9u60iGF|RCOSG#9`|+`?6I&E|cR*2TgIr z4L3^p)KQT4z)j*T6lOX#{Id6N(vKyyo(0A>9vGvHy^KttPOZ)Y%K>`wYi`AV)MX3k z=-VX6Mk|-c$7weh51$8>;o8!xqj`Z*tBhx66v?Y~*WS{k5$iELJL9866{bA#B7(0f zm-)t&%IcTh9&4;i^h(~P`{|sYzPAnxDy(?O4`V}6eN_71k4!o=*A)-zm~c*ds=lfc zgr&W5WEyyEOCJj0_ya0k{c^s4h6qWRlh4a?ewe^zlw zLf>#-Q&^tyqK4c<6T)y7U_4yHnU8edfYYaKPS+@|YX&I3QBw&n0eer+=Rp&0e+are z)5ngJln;X_6pbBK_nK@=+Vl~+XrEDY6MhqY6hAEpw+nd-qn3M&xv&+K2_nUyCB-W8 z*8I%J8m;!BCI@Zr?KLSV?J$fw_rh{un=W?GO*wth;YsDe96PV4u}*4jsk#22$?Sf7xk9Ty;lS zHP8*75t`42z07*ptzJvBP_*4KZ9pBB5~5$vWD?(iwu~-l6+n+rU2iiE6u^Nr)N1R6 zU6obI^-7fyi=N=oPYN@2yfmgfF;we{USOcX5m>OnRPHD>mV7-iWwOcl`5`iU=jZJ` z)g7iBdd{NV6C6jogTLMHf3IO4O>ynmrWdsWqAX4V6Zb)y>T3XtCJ_{;OXv?N+!aBX z4;vgjj7pEJ>=^*Qk}eB&uOWe1oz{e#d; z`Z9jbP1>~88LvQ1WzVu}AgAYR_h2ti{B|<GcR2FnEkniBr88;#-|=h8ChUF&*6#l^qp6sab$EXg5e}AP@iNVzGXy+!Sh zF}AHfye@E;(4GSn$r&SHnlznR2)vmhOc&jJ@=jO)48M~6tl%CW|ADg!$EVUsgrz!dC# z^&$uLkNhKcfB6bKKKnwXgHbz}qmtO zS-Legc$}Gl-%_PYagmW}Q>M1aAjTPb;(=4fbff_HSbm%VyX1W}q8rwyjqll&rOoVN zRL3!05lI=<8Rbifh|~a%N8O`M8P);tKg)`l{}P?1f5Yh?j259!>lDF@zWPNU@=8W0 zI){E4$w3FBgZex*RKx|m>p!84R?nh zj_C*35(Fdzf+9ToS4CgioOwp$W%8Bn&tU(5p)R=veB$suGt~S^f_#QQ$>U1^N)lkc^Z+*ukX4wt311r;SVMO9^a z9eBkYhw55c8ep=~F}*%}VrvB?D=jd~skd)+`sc^ji~pRTSnr#qIC}qjNCK4fC%yd> z8#Mzf(?5e<8Z+yoh3$ByaIk>ssjUWT|~yhdvJ_F5<|iTn}ou4AneZNv$AFJy{{EyV`u1NZFtCNV7(` z7tRm1=r(>1MzD_0T<(iUr@l*}8@ypDA9gwz#qNS8NsxN9eap=n?_W^WWDs15e{?q; zTr46<0=YdY$=Hy*9bNul=RS;8L38mHInOy_W~N#m)F4}}UYD=$bdt`DmHoW)>vNhG z_db68YP{u4YAw4laKo2NE)#ab=dY$~F#lb$;Y)%K_>Zp`K3_z>Y^a{kG|*<~qer{C zj*sc?(?Ef~%&H%~0BE06_aD zMbR|k`!)Hyp#)~B$=4QZ-Y=PI{_^-;5iGH!@FIjUBDjKPfe0*A`XJBuTi zA9dyD>uSwK+kjxN$1L>>4Iccvv~J0t?*NtS+h%_Z z`afc9=>J2u)c;Nl6FuGUc5C`S+Q8{){}XoW{~Iw(^o)OSTGRb4-~T@hkO}vn;4m@$ z?dAVF1``9r-+bH5e`6v2%dY={$;9$EgE=@0+pm`_tSo;AGX1R=%xp~m2xa=)?wS7O zv;WFx`nAdSH~i9nglNM3e-98%;D3&2V*Qta()}t`|925h{{v7YM2AcN$BgIyhqz3P z|5%anKWOj2eQ|l2{+;%I9jpIcpr?OWc>fD%8Tx+kh`U95jYvM|#AhDQ0-9_=6TCcnylN16ONT880|_z+qaf2QAXDSwKVVPX6Y zZ$d}M%=E|h=;-KKe=lRA{SCiD%gXY{WALjl48NgU=vY~QL$c7avHW&i80cC5I9_^s zHu~Sn80Z*&!=?PH_}f0{*cg7FHzvB@_Dj#m{2K~}k)D>}-@&iYv;GNwh5lbLf&YMC zVPs%p{tLhIm;TaUNBT>D=`a1Izx0>>(qH;Zf9Ws%rN8u-{?cFiOMmGv{iVP3m;TaU j`b&T5Fa4#z^q2n9U;0ab=`a1IKhggKRB&ZM000F5U|+4> diff --git a/research/papers/qwen3-0.6b-study/qwen3-0.6b-study.pdf b/research/papers/qwen3-0.6b-study/qwen3-0.6b-study.pdf index 064db1ee5e16a971808f654f5ccd4b1ed763958e..e02f311d428627094542ac272325169cdb3f1e89 100644 GIT binary patch delta 15124 zcmajFQ*hu9&@DKb*yhBxZQHgn$;9?Awr$(?#I}uzZF~Rs*0;5HYajNZPjy$Fhkod; z>N9Jog0VCrf3kPoYOUpwfp7c#?95mF%-TY!>(vQCZ-x)4DR{Qg zW5F&L|6HFH+3_W5A?L-;+6L%ksz>YUOSomeZa$G9CCT$HtlNp!{< zXT7IjK*HxaEY7aEG#{I3TN-W=e@5;l%{1Y;fZgu`m4+R3HU|#r*dsa}T5&uCu3~zf zpB-h@a3`u13sMx8mr7SARhLQTa=i)oerC3x*)EV9hym!xq!pm5lrQokNC?mU<{S|3H`lEgHO6_k!ZaZA^pg7H--c$Y~5SM3Nintw?? zX-y}5QQE_dGawop|@v#&gG>^0vL(o_})GUH$yZ%0oLt_fK18Kv>okI{RtUF9PsvgwCsKv(d2YB5sQYZTnK~Pj0tUCiZM@9Dc+P( z=-)-foww}2he942-JjYm@4Anq<4P!owy11r&RHSHjAig{Z=6vTAT`zCars~h&bYV$ zm~yzTUrS{e#py=TKiWpcRQ*xs&a&InLfWD1evBNfOyV1*NUY+*+% zb<#qmYR+m3PP1+sjNJvUGu?1Yn((DZE%nN;T~T8{RU3jCIpn2um|M=5esR}x#mwZiXdo7RPU&Y*IAH{q_jKH)(KtMghEE@}57uo#V-9CN^EEC6NUJ83+&r zj?cwNnng;+f4;Y2lSgA=-%ATChqEWoeKCgSabxVTjwB}UkLw_FWRi;GmBCMOL!{Vz z_=662ri`#YEO%h%38Coab&48lq_?|y%I~^c5SIwmQ^f3He;H^HPcX?}fU&Jo`S+d; zHQx;>v$`ZL9arEQ6%Z`>m$AbG=l30HNPaF=DrC5Xr6#3T%+R(1i2RRb&#EVvAmV2d z&!XwG%U7XR8b0ZR#>VBCo;}r?6mu%|Zeh{_Trws$6ko;8K7EhR((1gL)p)erp^VH$ zmQrjk@H{$``xXMOU@*0mK~T;aaeUy^KejuqHroSMH|JPW9*KF|+QK`4w!cj^OUpXg zKk$+xD6gxKh%{vuSSMAZzL@Nki2n2rU%c)?#YrB4F~bqXFp4HdPzoAe?tF8OEEVa4 zAuHYObT{yVX?oxsevjNGP^KLQansvpTrc64LeUhI-PQ8f@qWA{qk@(d)z9mN(;ySY z$uM`cI;Au3C80z#dJr)HeoQ&N>!R&WD46vc=?!1ASXdlxzXif2DCH=&D`q1YW^Au3 z>^jvL2v7nGnb#aK>kIg9+Wvr4U2GlQ*kTiug${VVu-?PyLm`;r;F*PP>C73js<**6iM$1HWyA7%e7L-N3N6$43TV`t385r?v83)jq= z_~;QHuW%wWp~xHLKaaBT1Ffr;=I$OSDR zyjPqc7tdl~xj0c0nt$&_t`6t?aN!Cot=UG{PTJqil5Y_t1_QD2lMTB7_3sIdqmvsX z(>69SI`~+njaI-BMRAQcjhsf+;kTHO#kCSY4@m$UaVC%;fW#tN3>Z4Zjg%7Zu7Q5M z;UWtOI!nU?=8ZP?pz`Mn=6-pT(BM*r5N>=?eiq}wxZ{w4Fh5+z6S>Bj34zh*g#ql`$v-NlSN7)6wb5y~sQQqi6h$r2!9gdxy#nJvJUWP(q-U zpshxA!-4Y!QnasiID>L}aK*peH8O4m_ghxav01BBl?L%E=b!iGOW;bpv5=wm2vRWQ z^0*oy!JvB!(Mq3ltKo9@o<>(>;>AyR3MHBS5!YE`hnqqNv036iN$#+mS#1(qP4B2~ z#8`9y+JZolN)Qqi-0a6AEQdhZJIP}2S!Q!r=RNAgU5O;vet9Qt<14a}at}J~HuDnF zAXZLB)24SSh2^tcv(OvcR%MLeD0t6`4i}s^Ax7D2%TykduE~@72_ZwyrS}GG3 zNXEPbbME5TnC+O@(h_s47hEPR4C8Us?YQp%vT!^h{6hwYhHyP?uF`Oh3`uiEpXCh~ zp^TJpj73O{9|_4$`3i#X<*3?1xns%FBH^AO?pl5!QzCJqSYlt6`g$zt0;)mW&%c$R zwu%hr>^zyFTiH3M-O7taH;U$heoNLj!Jmw01wtBP9tl*nG6cCe^)=EKs*YNrrz$Og z%$(_lPWimn32o)A(jOFYe#a&;i)hY0^GZ!#X~`2bGcyA+jlFv-Bu9!2T(158x^`v*+Y?Z${8{tvK75u>dsU+x}dWz3in&t z1|Hr_I)<4zI-&hoDugF({`7C$U5QY;K==A+uT0a~@l!F1Sya(#88L>qvUL+KYEF-* zB53JS%^=^J(u^kJQVzlQ?8I$?GyZF2_!76RP!^VHs#7B2Dz8p5>7__w4%sDes)Ntb zp|F|FnN`n{SStZWTe|3M?*s2297ItT;n$p{3A)r$29E8*ZZmC7RZI&>>bDU7vWRDAbkvWJrE=e;zBkWq- zA^2b2yFop7oK*9I3m~@bf)4;bLzM-Y|H9FQ9Ow}oe&?{^BD-c`yN_ZH2%zBqRFX%X zvu3NW+jsI4&~n@+pEMLGepkTC%_+Ee6dj-xno7y>R`&R>qsG?Mdk$TJrvxypn}gQBJq7@r{RB<^Q= z4ipp)hqJ7={t!fXaAGRB@QQfHfp5aU7utaGJn*^nYc{K7`~#mdJ&Fr z;#fyvYS$T@?6|UG4|}dRSC7Z6TcYjBI1LG$HMtwyR5MFkR`(99&xxHc$kylsSQs-~ zQzvIfGb5Y-mDw3v1)9Q#n!Nia!fDre@wq*T$8jHyFNpYK^YP z1}=}QLsTPZHM*8e0K1=u#ERKjNZ1U7PHBPd#dg*bU<~;qYV2Z_s1|&7cfR z4ON(6ydd9E*?;NKzP!NbgRELv!(&AGgajL_*aYoaGXWPeAE+Pkav@WIno`?|#D?q! zu==@OBP1L44sB6D;KG?F?wgJg%sU+KRt;4>k(jgMYNu5ECjn(DL%jp$DLFX>4ILZA z56q+_$6rmVl7#*65EF|qb}{utX1rPhS%~K&NPyA}^0G|>XZw$sR~0cmmJeMp)2Cc9 zo7H3|KN>$cL2|#yWi2f513ys9Oh21{D~E&%)c%(|T%CyNG28w#qp0UqYCpxlbiB0b zjhE7qUiBN6ViMkWTFqS=tl-Ke3oH6;Yo4Eaj7e6m6f^;b&{d>6KiUHHc?DgoioW|e zOl}B0qt=IB3(1tDIRPEpI@|Y}EJDd235ZA>99f;KrbOkKzY!9EYrmdZR zO05}dIffZijxMVL#T;zQs`a3EiT57R*P}A8qTS$Yklr#XBoj!!-$WVHwA4xco=^&@ z6n3h%kr03+XSPb}LeY6>?dW;7!quF?kIJ%$YV}@-RoXfeC(ZmH7}1Lb165#i>P-VM zZXe%b@%t!E(l8u zjQv?p|8B%$<%qTxb)WOPBTx0jM(rV(tSRA04Jz<#iKL19%64$tzshFhlE*M6xiE%_ zRfbDYV9=Z4a{joSve3nd%=Hd1H^6i_AO5B^3pWkqC?iBT5-EsRZANyBY)G1%fa<5S zmV>GeX7d72LbM;N$xle>Hh!6?Br>D(4}i4VZ3#qL|E5nkp{H*b<%yyI<*^;vI~jJF zYr9j^#S|&%6>gZ56039JQNdcZ99^+F{vL1Tyz~EU|Ew>9oW#AvhVC*?8*ilvXSMqx z81W;Z?I-%{qjb>$0r%Z6-y^|ILtJomPT@j$(oAu{9s3F2R1wR6Ie!ALH0Dy^r+jUx z@XXa>0kPBmNi^%2KMkeTsOC1P#V565^Q zEO$I>&9J<@{(;&Ea>4$ktsHXH#NyyS?RUUV(seshO2T3T*$!E5VNXyu{=Gwj6&1oC z6?_n&!M5xUVt@fRZu%i&0~cq>BMagnEamE&H331Fp)SjbF2YQW4!g^VJwpG`5ZN4tYJeiJPKX~~3-DSQz~#A!%KHPXD`(L4 zIkMzaqi)ZIGQWwQa1|6WVy)z;)K~*Dw2xc4if3xj7Go_PCeOcvM9L$Xx*1#HBK$TX z$;gi+`oYM!xiRP2Ivw|Z?aQ>nQ1ghL{+bkIA9NmSszkitPPrFz74tR8u#^afSUfm7 zh$fipuB(^1XPr?cw8j!~z&`*@hkt_$aU*9I>`A5f@9)p(NML4ES91%77lsGoW*|dq zU{U5P&yPm&KU~~IYcm>|TdQ(~aQkLnn!#zp!FTy5Fir&o#0qun^sC~ohi@98pnjPB z$s#M~I4?qK!$YXE7I8_fJ3WHVur={oPB$pNY;ergR;wEHhOfzQ_>&a0;6wnGQNU~p zQ)yX0rzD2UoKz|dae(#U(eVttqDY+a=?CGsX$WKcp`vd(BUyp#Nihe(R51V98$i04 z(-jiQ`pqae1Ce z6Wx!%ffFecd09GZ>wr&Fgiqs}@kF2*EZUmt{|bs0DuW8vDuW7M2`~Wg z5{|w*21<4q1TAS22WC)UrqJh**F;rjhBovB;#oQ9AaoLG*fod1H3KtoWkUsxm!~Pr zQC%xPHn_*2gDZW_0@3#~Z`ek)6KxHC-95+Y!irm;FhJf#AvWKb~Z^%LZ7&DATrzoK;&2lg4Tv)Z;g= zPTwAUNT%$}h)G#JKLpV&_H$<@V9~!Ar@X6JwTnB)=&k-HI=RsH+^LT2*Tcm+ghRUZ z2102THZzN2Vs-G;)jms?;RwOdsD#2{Urs!>X5W&krCblU&a7F_yld^6qs%rAKCkT4^rT(jtCO-JLNp8Gve*i{Ex4BE~92W`BPjImEA@ z*Io;i_|xOWS&XB9*=H~SU45inFX=dd&+ zxt`hCE~0IRTaaBD*Fs}{*A&*yn`o@oxF}^Sn_+Ijmq-(p?J}YsRH}j{Y zqBhg3FIn~5vwO4-m#xjAmx{VLlFo5U)`5TTHo0U(ye`g%PTJc1^CaU5nMs%Kd$1<& zkvh&%KGG->@hT*D#HBjLr{-d}In!j+_=Fyb1lj5a2PfKQt7~NECAYHbW|jN|^JHI} zAi$Y{PX`L*DU6FDYNx3aNUVL1u z{ut<>+)8X%ys3v$MgYtD@Kdg*K~RrEv9|2pSFA*YVVlKPP109wbdRljB5>15xQCPx z2bT8f=Pxpua@5`*DXB=NJOu~|u{q@_FKU!D*=Fk#&=Q;E8#*Bn(gHgAE;8l8)ACxd zw1f1{TekTrIZQcn{l3B8Z(X4^0l?ChHpj#Hkx*KsUYg3s9Aj#&)Cb4dpZ0%-u5yv= z4IO$5iPC@C9pA~|QNQFyB{j$wW!6hhp*Mm!XdKl%tB4mYi#c4p;C~66c#|Mt*uYEu z${5~S7P5vdS?+%SOTM>4_kPb_Rl+|c7l@Xsedcmz{rD;La&3u!a9V!fN(S(TPKT~@ zMe};~XbHDnDi6?Stt`G9<~x$ExX2tObZz=~4fZPc6v(a|xf-dtZXR~h;a-u-aW?iG9Uz-W%YQD>hrG2z6WTB0(xY$3St&&%`=hS zmVD(<-P##ns~c^e_y@o#>Rczib5TpZiAJc-pXASGuU}=P_9MB=cO)ba`%}Dou8Q}Y zFBzemd}Rf5cMmOUtfSdowOiH~TWG__JEl5`xG!3RC4yp~eN+%-U;C$*gzNYeILe%< zta@~@rJ3QN$N8j9f#;9I!V?~np7EGH_dEF?5s!;@R)@YS6XLAiI&_H7E2Bi?`8ic9 zn%~}@UEdEkUk_pDW6jrY-}ej9iQkt^igAGmZ@17r1Uuu0Pj0n6Z!fnwwKGQ$$#!~V zh?psd{|*HmkzlfBNm9#SLv;AvNgz+@TWMt-HK6DzCWJb?0HfXsN`3KLW|%QbQ^Yh9 zfXwN@b@GvM%~PNvXp1`vYz=s~e}7S5t6A(asUHQm)*Vj`oTc~77WGNg^-MpY9wAM& zC2x=87(1+$qUV%zB(q4!9+OhHO}wy{Z#d)|gSaN;$x5UxBG*EfZ%AbxL+B$tKa;`; zPb39N9PE<<62hdIN2Ttnr4quWh{xoIVH_cur**Jl(cGAA{a)0&d{7872iGZ>tTqR! zA@c`uef!79hNGd6A`P%l>jTIFN)V~YR@@PH7o5kiJ{T z>{?=Em35}*skG(1l7~XvF;)xIvYLJ2d^S0*b@8Ro!b&$y5jpC+3)H^a;EI_DASBU03NJuNEEm{yE3;tB!JMb6@9JoP!AW*JOvgnX%>^4x?#%$hgT>;J^B^-^m|+BZ{s)VXp?PB!?` zlHEnttc(zq1$!$TdAWf}USFB|P`9JV7n0pj|7OMfoza5}pzNq@Wk*6{?& zUi0}JDeX?R6RLLN(G2MC<_8;p2Ct-@Ro@vUBUXBIUhZq>REF1p4fFjcmBqeD@%WPF zxqdiTZrJuJfpHNil1Q3Ki-fGi?XsD=yT}n}`__!Bt~~6HGI&$2%p*PdD$!Gp5m1l) zX(;5tTamD$4+?X{CD!IHd(F*IDbxglim+Q(k~U~kJmB&8Tb3eG3i1d3oZR4WcZDG2 zB4pD?XH8!T4wYsvtbvbsD=RDONKsKqNl8f{oi+Y-J?jco^g_=;|NE}_n`36?vRMQY68+86@Aqv!kW^TMFwj#nh{mlkYq{^!#S-dWz^q;_PNF z>oY(x^(o~96cmE9DRrcnM(`v|M5>j~oXEa$#*`_yQb-p5utXz_{#m5t#|jfC4l++E zn2{Gx{<{bkQW=o~Ts}gAD0`2x=v+jW=Gy^tK3h+kI;ys*)E=(^`kDZMa+koL1>m!P@~O5~A3C*d@?mOf;WX0j3p4Is{}&z5swvzz8tp5EvX+iR@0zwQy)mhODuqi3A|iFn~@ zyqDy<>13?=upSw)5k1MJ>v4dXNdWF2&(aOb(ae1l{c6ekyaqJc)dDRV2luBwQZV@K z(XZYQ=(TYEQAKl6>U_u-=>8Qd=_@WBzLba7G?m2cx-@U8zL=l9KC2(k&q9w8g2Zk6 zL|L<}EiP{At?P$tR@_(#&sP0Wye04d*f6X!OHZB;1DLoQ-8sw(9eTdVVvug~4 z_8=0g;W_YFN2>8w;#B{=6(-p>pY`O^ct1N?Vif(!&ELAel84nQOTg)PKN~$8{BO31 zM#Ru#YW-?_)yW*0flgJ|aT%zdjLcIblKEq~TUqaUn?wE4tv>o+yAzW~EZ?8ph5Fx^ zlVlMY;vLI30fTmFeH2S-jmW5-xB*T4HlEEr(TJbGHyeTSQ=jeA7fbY(_}-S>+U`ZL zL$zCOBrmp-^5i?V8H;^2v=QP{P0W7E+%mSQhRl|^WuL^ZiOtwX2+$uZs*s``#++y! zh_h8T7e8?vUv>@`Uw#UguoGaBiG5$0+R#cjs;*@^m7#38x}|NA8mqU|*~pvUGUCXe zVM9)yibAS#*o!Q3VEABN5)^41mekeBK=x2uf1m2p%9Mm+_h+&1uQ~20?D1rK%fFW> zNk61fyP|#jQREB*ZeZj!*ZRiM5#>nPU=3n2r2?)>P%*4sJ}~v?^rMX%QrkWpv9is; zf>wL>zsZd<2(9UmsQAKaDmV`a{jp!V6w&?1hnV_<#m#?srY}9XD?Tl4l-{9Rm6xv} zezPS;yV&Bx&*rwyYOvXCa9QuIJPOO`+21AxN^xG3j7@m7g8~x!sbG@EcBx1mMKlCm z&qyqT&Phla16&?%DFWvV`Hp-PJ;lk1-o=h(D0L{$!RFTIPCMgF`bEQ!-O(5%p>P#k zU{5G(=52aD`zhHqudFkXxv)bXx{nuZMm#r`hF}{Uu2bjI!#?=B{p938%|69K^Y793 z`-WrRkbB=uVZek)aR*J5VTO(q$7#_io)2_dKGR{VZ=Kc1AL-D>zS*U$T#e^x2eU{K zw{)Ad(p-14_MY^Gsi&dAItA*Ze?&XGk(SDlP6HBJ)_$0JC?*zj&%*{kzkReraN7~$SGB(w{Bew=*Bplb=jTP~KCalW)MGoCpm^|B96r#)Up-g_X zc89h4lJa7dfigMoXP82sdio+-k#B=;Gu&6(`?TU93IVPS$U{G`in#!TI?Z#_xeRy} z+RKA}j^`jst3SR%1u;ZE&L66;re9{Nh1K>W`dNseA#qU%ZJa}jYBF#M-p8Zb8(7c> z82*Q54SLPli}!50q|{Z9)F;j2qX`Wy@C}##kaFxbLe2*`>8U{%l8+t1-Ui=p+d0*t z6eF$_$C}pacj{>~pDPOR4tkH+$e+X)RVxKIQ}3o+C)Wnw4C{AS9N)0qn+&Y~OHM*< zC13>qOAY+Lq@*crt=K(o4Bws};#iiB3dgRI`K4sPb${*h;>smNDoHgndL77<%axgW z)LsOH#0pz&5>m-1RD#@Vo=lr!dn5?ydOvaGj|jg=N<)7`+Fm0p2qj8Tgw%)=FI>ZZ z55k|5#0wzW;HdsEl9FY%c_4l9KsO9jY6&cCSP^qCXC7L2UNB@5lRFi38EL#NGGmwD z_wQbNQs+oGEV9}DpTi{OI)b!grhUVBP|YgvCmvz)(zzC16RFN~9nX9JWM%0#t3t#4x5s z5`hka8b^=RVNhM61Dx?l+R$KV*vJTSyozsAvq7NX+Ck*#l;h10L0}Q(Nao;iP%K1> z6%oRGN)RWK)ldZnzmH@eeyg6HX2>999Eu8%N3{)}h)>7}smcmhBNpW65hg=5L$u8| z&QcnV36Xvhs?Qrjf1|q70xw}AB~RE$X0N--SE%Rw-=eCQPEJF0>#-4bJ_<~MG}H(D zbbO$w8>!PSb1U8(d}Ks7u2O6KKR_ceq7UuUlBinmMIY>wv)+!tLMyFDo9p|798%V*oedva)^P znkB6BlWx3;bzjREBqUW$51a&0`1))m+3SF`8d^vs_3_9CtwLin|1^b5jUd{RFt|1v znmgsiFZmpPF-rp@>PV+3?EE2d@*AsX@~6@Enzekl&!!3EHvXJ^g0f9z9`60M61Qxx zmZI@d{;_xBz@?T-zmO<^aEo67Mmh!^DNO> z9({-K$_d1yVpNVI0lZ>B&K#_#7!^(dQ$`-u_$)5nti}Vp&vz^K7cV|pvNR1sdoT_r&kd4J zDN%4Ajr%BtL)M(-A%X0cckIU|sB@6~Y&p~5eKOT{!I1lpz-J0TYp;IdI}h-KE$rtO z2=tkb3ie%VQW79!D<_&*BxGkB%`BViJj>Q~Q>TT0&Pd$t6@k;<;UCJpKs7lsfp}Tw zDxU{xZ^Fsv_uHhPd_eYJP@@|tTnD03yUw#eB$7cPq# z}W zy)WaOJkoEe``qx?`c=K5ic7H8-{W`IPEsD?(n*MPBUlk^*}pCmmrdSh)Sr()5%_SA zuK(G$8QY)Cx*?t=#G&Jw8}%7sYm2;`%h^H6`m1NSzX6C&vS%=!z{k5*)F3Sk+!Gq>}$o((;cP;~; z7rULb`v>L}|NfgEa73Wo$pyv7&BEW_(MgsTL%_PEg@pTSPT_)p)%O!Cl^oJpglB|a z)?*N;p>5tlcG6&SW$EpvSlCE8_8=;g3U-_g3>J%owQt-Da4@9DexNRy2q}CzT$}R9 zRz=+ZE>(?9-8)$L+0fNqyk8N%q{{LhjN#_7a_+L8LfMYOOHIym8>1R0yrBE?K|=eC zSyAE{U)j|?)IRf2MY{Iu33q|D#lpH8T6_s$DtYj|FQ2uz{5Kqoy6N8DZyq1a)`BwD zIPB2*DHJ+C3W~p#>?65bk4T*&Bp32aC_0t;=(&Udr;v1{f4-+U)7~kI+wyK`lZe>(WaFVB(9>{1iv1nl^nKg5#i=b_K;7Wm)mcUGevvz%j#Q>`$^t)rk-&47 zOlF~7gz9m6INT^+9|OR(zj3R?YETU~2)Zu7QqVmAE+bboJ%5_o@Z;t`&uIcO|1ee1 z30;n(sGc-}X=YH<`e*7jjRr?}KHH_hRt?}pl6D$MJksa3aGZ7K>W$R;)sd&K*4|ef zm;POzT2e+*nj%BFijbMzGI@$KxJU=(5Af)!IN11WHPmD^;(3bkLs@;q)9O&Rb?Qj% zdQ(*LP>E<;tFAP8V)Ts^-P{wvUZaaOJJ1o=um1Ip8FFdbn_=bC+`;9Gwk(~`or*aU zs$oufI!C>huAZz=nUUsM4^0VeF-y!r5PkE6Hl8DGQ!Qo-Ly)ZEZmTA~X}o{C6gr)( zFVXc>ZsRsoW>*g(--eEk{7JcI9$!9*@GJi?W4yyPBg@u;kdyEg$B^}?G-J}wvYXp9`PDh8H z=DhX(>#RQ4N|vBYQGJ0Rt!Qe!KzlTiiF_J76Y~yk#lLmmIL#flY;QPdhPwO-TLYzUQI4`|Hj2J++qeSLJNUP43^? z@=pfuUpLu_g^d%BRE5(24r}=z)gZQiX{J-l17b-wIbX_~Iwy3Wn+;J5aVJi323dG) z-;;lY6%p_vI$`7OvKwA3jxa2#PHQ)X5(W@aOEE>?36Ha0UhCUY)R z7A8hkQx0}6He(}x-v2*FDwR376u|s{{$1Ht-Bd9&a7J!DjYNE^L9Ag0l}LI8(i-hR+v!Pu_CHVt~ORczVYh} z<9+#*cZu>!Uk6smBO~s)38-~gG~>@-CZz0rl-3bbbzI^mu-4(#j1W2|zoLdXts^h% z_*{EqjDs;V0F%?fdfC#Lk?B?NPGb()-mA1QK1OISV}^@ppE`!8fiKPk#Ev2cVL#xV7#4Z9U3Fp zj#L|8(sGcrhO2q^-<#Ii!m*Aa)Z2Pg77fmjdbFXsdECj6??ijqGh7XPy8l4jBbpo; zFiuwE2c%*c6?|&s4MFgSOpJYAk~78@WDg|>D07o3bH5c{9a`=Z2@fM}#uX-#b3ZFS zx}xu3bzaSgzw$mjqwg}&NMG6IULy*9o26eny6bOFzCE?n1lA1(=Rz_LBr! zMk2>^pD4fARbM$0dOC00?+D7rImE_+rvZXwpl6faVxKc-P(?GWbm;BTiRB0`XXyDB z7h~6>i*WhcZp!^LGEd{5Tvvp~O+wf+7Rz0OmJP4EdIp|UObtJQQSnLf`iU~M>JzeS0vkSiwW7w5anr5fA!aUm|m-6kI!jiUioY&KSO{0{P)*dtUk8a8787eYC6yV8Nby zx;LE@*1`m)Bm(by_`(h0eM^K7^LY1vCes0#ei%+SKxPO(&V-wx@ZR_Auhn2X;t=O6VklvtLGn>2Wt!3`){D}ky%F}e#G!sedPSq zGL1DIY3zWj@lLsUi@sT9&Z}IP|B$=fg+;Q$(Yf*WdH%7M>9QQ;lZo!~lR0`HIbHmZhU{@X@xE4fEhY@Mtw8?o${ zqHcZd(rB9*^{w-A`ihIH`JF~2jM99hzgjMv`c3hF?AM1dnK7xDR#YHYLk z%FG+%f=v*{L~68Xv`ABrFU50K-3C5T&#&_c08>&pis#b+5h=U_x)iGm$)xBf0)aH1 zf3%dg%1obRR^+`s6MG+=Zj4gfPZ@g-_964K<95vF!ki@`-XOXgM*O}MceGd|khA;R z8GWi8aX!GOG;4reGw7HJgW;c8SCGRh2E#u3qQ)-$@rc}3P6zcF+ou2eFCWP(BZ-Tk z=Ub5BeGzXU3Vjv3b`R$wK&uU-)=1+b1iB5+sh;O(mtJ;|uMN@bp9`-)1rxN_9u@7N zzB|N>u^sI&^F?569d53n!NtgHy`tos%0^i(7HHXHY6>+e=q-d4(lQ}-P@vQUUiI}3 zkvpp0t_^O)o0fL0CsxhCV;e~5J``^FS{t&&F8L-9Rd|XW^C&f}c;4ASbk=D{d&WWh zslT2^n$C^lqa<+qkO~4UDoo!lf?!2@4*j<{q@F&mgy})|4j}Tw!D%5XjCid^2=YUg zZJ=&@&Tkxjl#6a1UthCukAHa+JIGpud{U?S*U|#Y_YkrTmqUCxmPcxOV24cHE~516 zXwCq)eK6UjcE&|( zwf$#K{B`!M@F_-y*lI$5HI&`<3vrj9*Y6m(<-{K@bVpGzLCzVP*z@e%sPd=TdpXE_ ztKbysNq@`WR6;N5rk5``)|1J?ZkAQ1otAx!?oYp;C*>GSFYWf?tW1z|RLNSpmi3Xz z!Ua@P!&Kb27h8w87DPVPEM*dtBAhG=rN>!h9VN109(@L0>zLbVrd1vv2RzidHoSm6 zZ(O6Z_dRdNtwqGa*Nc|&iNV+I%|*l^x}5L6q07PagZQ>BZ0g%}>x7ZLCwld8JA%Zd z$PK%4=?b&VBH>MtoifrQf2o`>o+HqE_{~I$%0j0RddBdu&K-P}jXnI1?ZxatKJtl0 z1a%>X2O)u~kR>LlKgL~cBf%L6&5ZJB=Z07z|6L40(e=>mLr1GWb&UQ*4YjP|^F(+J pOHaHBd|yoc`u|wo|MRvRIXb&Lnwi6}u(7bSGs2LOi7AM~{9h&%aB~0v delta 15185 zcmajFQ;aT5(6&3aZQC~1*tTuk^Nfu(wr$(Ctu?monf-o$zGVOVXdm>I>f}DCPP!_0 zs(bezdi5@Py)Q5ze4dq}EEtXy&nnSe_IF8hX_i^Xo80Uy92)U$U;OrYtxfy0X6V?pt>C*I##qOAsZIDK^Ji;uWQI_S7t$+Z|F%g!^fcw^?~rtO+|h*3D!2FLy~z z${rE<(+YY{NpP9>g4>FPvF-6E!D$IU%^(s8$^nr(hvA~a5Zx_u+0Ajn1nRQrA7y=i zi&xRJ=AC|k&N#|x?R7Z{Z&d@e&DIoTB;Rwb6}(RM?h8^VQ#@0j;&NkJYZ`lzS-6gg zvk^XT`wqWrEF{WQplSlojpLN^WrL40Y-Sa1mcG@edM7M3k4|#yVa_`yZ>8PB!c7s@ zX>ZAh5U?d4yQ{n2wO58Zj%JJGcOlm?tDOY?fbScCCf!~J*Gt!Y^gh)d<7hTqPif7* zkHMN2lt*pCRar{gC;1c0hPOluwYC^iH$w+tu`}=z{6`Ed@}h*!-p^#SmQI~rL@W9J zxG)h9EcC3km7zp@QM@PASATlO?pu+~Y$0V`CHcL7)T-65U~CyWo_!)OCzWIbhRKYO zg5Dd@86SN}v=O5jS5BrVqwt3wCdTD*`&y3byX!!Rm|j3_h)B+eN7j^yaOR;4h1|mW zwnPdCT-y2aQ)mao;DbAi;tplE_`U6oTpz+uA^9k;xxfEDg@s>-Kr%M6tis^u{eptS z!Z67v(4o;v4*c?w$YPix9HG3)1F6cqCT|Jwp8HfsOiyDP;F7I#loc&Ah+gv+cXgde zmMp8i7MDfu2Q~>;<6)v_^9x)MkuR{xe9`?pq{;I_HYzP`{Wnxz3og_Z1&&Hxl?+=J ziL0xUj}VpX`)n2_Eg=N#8upVX~ICIv3FMNlq zR-U&)*IQY_d)ncMsVm=Uq7`9RAFAlEt5y|YG-(7%u_d04Ras1px%PGolfQyL+wJ~* z_hSqJ*@Q(NXy4L^)_6IYMMS-iA#hdusvv(2{bF5UI16P2>Ygg9pO5 z{iwk(tdiF^`(A~~9}Wk9&d(~HC?7lX#2VPbcX7ksksrUfuL4g|%&dx4g?Pe^l;!de zjov+6upm4(UxV17f})o&Dypjz+ivA6yX&q+*dtQRkhcd%*HXrvV^Dd8;N7K(>b;$6 zd>hc@@Xek(Xe83lB;1H9<%R$s-3%09hWMAN(2(Qj>lK-@L0fad3j7!R=-xg72_Fmm zN)*j4p9Pz01{4kHnirw}9B5aio>Z^!3l|k;7d5vc0n~a1j^90r=!ohzkkPZqF>qGe z%5r)`vlx$W8}ayogS1gZAUPDK3V|>IueCTHwL}~qY_VoNP>2lng$w|D#u}TJXLX1_ z5+x*)ytJYbD~oLKtms96X&u#ZoO))TUM>)_MD9R1kqA<_g<=vIB+buP!C5Ev3QQr< zdXmT=DnchHf$#bMAsT*z6M@_A%Z@>$X_6B_iLAHV~~(8+W(PDCEWd8G*n|cMxU}Xok4N1~Jd1OQgGERDwte zqq~Y+j#boP{S%^teQ{4yPotH0h_BeM6utdTGRQ@c1vYhEVfgBj9}l0HI~3&9stx|+zjz3^SnhiQwG>N#YKzIa`t*90`y%gQ z3pSdaKAE71X<7m2Ba`MAL8iv(Zjv(XbSq2PFx2($J(n^bTIgA&Vy3MWUV}9|^-%c| zVn9!}_ouhTM0MJF3N-622$7nidG9}@VyD>2xuaI?EuL?iwNL+v57#~A1~ubyyJ~TK`9JFgIbH@ zxl!n$J8=*YR(Yy)VYQNfnzh^b5E2SKdIQk|jY+|eNj?p=vRTVnc3 zD1?3#Kk~DDBz7ozkg#s@{ZC}%;&@I?E5NvuxymIMW$688*yf>rliNrnHl^tSKCuC} zhJW7T68B`)7rJ?Lg$Jq96{ovTHHfDnsN(zp6!i4no_SP0lP0jylrQ2a)5G06*IWt` z;->5u&P2#frNV`8el4kYTR@Z_fpmbJNHEO`KB`3XPSA;wt*C7>FO~xF1wBpiI6y^f zSQrbr1a^Y>#P%cE5WejBlfI~{S zRQ1;}WWnVH6p?-@DJ5t0@I+dHu(G#Ns!>zGQo_Fz_bI7W-mIV}k*jFKx+L8v{EpQt z{LkwIu&`#J9PJIj6ll>~(siifXB$`7F1B_2`MRWeXk z0Sqh_mxn^2&Nz5@aAFFiz=l-MSy0N+H*VkD&m+M?(sx&Sxj*v5Vq6Ywj6ivNy&y|8 zL4unYy~`q2UUGSnuN}{?tJ{0_4avcLyoxmTio!E)s*#Nohfgo&=h)#F^w;}vAI8ky z)Wy}=%*gJ4A_rsZU{m;TQ+VM27D$^ZJgEIQfSHv;Kmg`{MV^}_>Q<^~8W^>O1yzNT z6nd?L*vc#N6d8MiM95=;)aNcw^Vw z!QH(N@R_HFmvu4lDs-;;dyFYn#)BxO9BK7^W12qM-}6Bw;B+SM*4S(I4k>quO&>@@ zcku%0lk1mnG+mG9J&wrcp@)Nen$PwY5|m|P1e5K{Ayi(biNhNc=S|?SZ{4aW zxin55$)4e2SBeyrHxY>;)U7grZhSv{a{ozFe`e{23T0yGH1V*JaWd%)vOk?tDDC?$ zzwn9dKbIhD4vvmrjJ?oWLi+}6G6fIeYh2SqWy9_g3~uW-L$zZymW*3mW~ltO(a zdB<*5V(K(i0O+B}4N(?u+?K6;2>TKoaDjmYiC7@JX=DRM4?B(G@`FRTN{mhvgk&^K zbhI;|05HAGu)vo4OX0SYqZ2FQ@MScssLAPF;-J=daDe6s$c5CaY4=!+^b80GkRIw# z$Vuupp1{5mCYV7`neYjq@@e7;93i-a{02W)&%P)!*b!15Av0_uj(JCcNwyOoZ3uHO zbw6!T>#bOH5aDjMjEL_LFMVGJEBv-{R^32g#T$*sh-+d`IQ4VLSyQB!+!4LUH=5CXca#mwq74V~6lu!)NFMOADHb=@$jBxCn zO~C6oQAv{<%g=9ZXsqEL$uDmD1KZ&hMl147`o~}J?A-za1G479U@FwndP@|8@}F*v zy|B4QJ>4`#NsVP4ZtGt~kvX7!Gc|^E4jQ;B zdwAq|TOcS4c=G`dv;t++aX{T|5X5gI^Z~SF#6%I8ED8-UyMHof_Z#8@Zr`H^A^P+r ztikP<2y|p9dUQX<*DO{i3qirA9qcVfBAT^e&$|8m9!NqdW?6$}-?4jCnwF$GNBVt6 zjz#+0YZ;U!=ORdJ9Ri>oPiC&mgx0bzh0LiotO@nyAAY^%Gv`9t!SJzjJqBZYWnP5d zaw8wJBtroRTslIV;}ub*dxMo9%`n&kkOLT(s6Sz~S94v|12>wAx-lN$OkKTh&o?W4ZRJbdUfci9d zWOTLnInK0Z6XI;P8R!6wxVOZD<2Of2E;4|U6XyOMPa>)n_Oj2xo9K{e-9GWafx2`% zG&~5tC8oEMrr2)zL}aB(|AV`b^0M>|86UKhWo<3%vs~Uk&*aal<$6W~L&&_Q#9n=n z`xXPR2Hnp~&!GriN@05-GKA_B=_1(@rV$R*y+A#yVJcq1t*VOUcV3@ae$Act0Nntn zcm9neYOm?AmoWsFs?q&18T9(VJ-g({CaO3Y+*5y7BC-jmwP)E*6lw_ed1}qSi$i52 z(rG?043BFz^64MmT+63QD%1(7M@C~7)-wS-H_3uAA|18-9S;wHoX=h^J%eG01Hp{A zlaD5ToSzfEfF3i|q>i91@d-}XXc3i!@LYl^RN@4i=|&ToAD0oTmJ1hvct>UwpVMtu zY?z2YSp+tY@u-eVzfPn z5X2UY$=p3>v+|$6xVs;xeK`=J7n%W-g2c$z>J(v;cd>}-_8XZb;MgIm)DQ_I_eqS z?6vEchPDkp;Mt@)HlMdkP~pZBdb0lR&}cwjKv7TAg3$bEW{Vqt3pcaSmyiG|-NBr9 z*v@r85G+KEtc`MBmJWc`HSFUK_=%65V;aT8EtFPYai4)y`IRO-KLWw-=1}gxu%-}} z%Gk5HlHfD`olp;tRW{Mme>AjKu=&7N{`6wu^$zGrDwt!+xF0%Hl6vZ8vu}`t!Wl!` z&jO_D!!+>@I-Dy`-mGwT{JQ`qVY(J!Suz2P<2r@0_x$H&o_iczrYtKOZVw`^#DWZT zbaNXnHY+*iFEUMoWGpcQSTxv0qSayFA+Knm9I<)cQ?9Kr2NA7w(8|OzRTC3m>9ma{ zKjYAfL%lFm!1YBSIEW(fx3u@0ME}%IpC+SBp@nU(;4$&aF~` zr7vHltIGnPU6+~@&cl55l>GgjjCyOk7B@R3C>(YM?zOwEl(}iR*ctH`li1) zDc77m|6gB{7X1zi6aJ0@*7lA9UIi}6{ePR05nU|@G6`q@T>~YDRX$ref`l|;I3gKM zK~F>@dQh9WSx|P{T8McB@WiIn-=4nli0aA8T3e1|WVpt*D=cs|PykxxZho@id8iPR z7KBc7%OAa|SujiIy^8v)097hx`vHfJ5O*Y8wn-{zArplDMEV%GP&=U6aP87kv{KCF zqG67a-cXd2&}d$hVU>+hEeqSOQV6g=VXScTwz#wPKQiXorWu%LpyYG2@V@~vN9Xie zu^B&KAB?~61`&cP5pBJA;NZXj=wVj|^T&UpgXM`kJntfZ_C)#T2H;&=SYPbS?kbWl z?KN+mEv|8?_Kgm+Eo}uO4w|qtC7WFuB|619m`NW%e?LyE>*MAZ;9qrv#p&?^43+hM-aQ?j zPjxNr;16OYc6hQ-APYd*AfJ+?fyU|2juoCNIOqdHe|~>H3~u|n0N*p1@nx?Hgg1h- zFDqG`b$vZvTzz)sa}_WY$4HsD~gh$M*I?4|B|+-BX@;V;?Dm0oj|`H4dE3i zWz7s8({W3K+P>=;%8Eu`&R)M+b03KBbfH~VF6Z3R6mxEUx=Lw4Mn;shWYf*6dB5eR zmMOkLPa%s=i+3b34y!1?h|?a5HBLhYl1xiAoSNL|tlRByjB&fDDi$g`%9YDtFuEd7 z7(%C>Ro$Sfs7ZycFs7CJg<$X+NT`azt~&~|b*TCP;0r|tkSMYLKiY@eAawQ+I7U z_(WMg@c#L`VZ(N@CeZTLv*_5?D(5Q2#D*~~X;dpiWUrx6$8r{Qhw?ei?zNV(e*mM9 zAMK1wBz=b5;2J@Nv4E5SQJ&Qc zxTiq(lxYP3vFq?j2fbfWM4Pv?_>e%Ll)7DPJBDh2wPmnfXmX{J$Q6~1lYq&+LnFsY z8};hSpmYy3k@a=f{6QkIMl&?apv4($(MY57{Qj5W{uJR&<@aJKg-cSS95id4JPXz{ zgHV*QrYQp-LE|?p}}JC>QmEJa9qNumN+ON=rHi2%^3_glG%d z?XT~XY|C(+u+}-qB`y2o66JknkQzSqAS*X5gyY?AS zeTpGCM+Bj_ooG3DK@yPJDhjml!3vwRUU#O_-mQ7l=D(@F)<{)kFSI5xSQ_mp1o^3V zZEw_6ajVE(%pST$_Qm#B1zS9uK6A98?(QGOttzBP{kO5}C;4QM+XUCu6i0|j8UX_f ze@*nT%d!n7+jRPXK)n2UT3P7l_mmjmO>Yv#E!&@cD`dXz#U`_TDs;jj$FAuWn}I8^ z#pL8Oo9h^sTVAWpLpj`k&OjQ+U|H&0K$r%z_2|=$5x>qZTq^(B1mpvbQP9QArnzCM zaiz4?OiFdt?jQVuQ6D`YiEsC9mbc*~rh6?$(6v{mP>GDv=kcUsqLHNHjS>l<>y(jD zY!~}o%^htl$pb{Oy3AjN_BBhJqi}!A+N-@!96<@;*|cZ$<8*OhQ+o*01j~ zFHb)G%b_eZ2+wssUg7QgO`V~w&*yt(EdJHkkAt(^_mak|y`Bz|5i&7YaW90RLw1I9 z`(Xf=ThgP$%qX!-*pXWcVQ~il3#u=E&k~iUBq>hp(~tK~n?c%UQX4+8`l&>IwQ58* z{<9UnHP$xiUUGsTEyiGYsN9Pe;=XW6c9C_oI7OzxP0``XIuY6}#Q>0V9&o(7ypQv<9>`9h#hoaEZP-v9QCWB5VLy!j~ipnMm$tOC< zCCbX8NV&79|K5xL58w$LzP0H2*g})zn_Lc6CA86!Z0VX zaNZCdw>n@bmkl`5MiN!^T=J0-qxqi$>&9$W4M>ujX(NoBdcy<_jaDdmRlBLwu@DX7 zqLXTty3rtv{MGv^mt#NY^&3XO`5QvCG@5z!L5x~{Llm2O!6*%2RTCT+R~t=f70sa< zXf{{1r`Z|2B1jd#;{KTCK`y(b;;h_7pZBxH#h$5=C|o#Bk>*4FsJUT85j`rXOzo)| zCQto-i6&sv6$;A=49g4=%L)+-R#%&Rjiy~ojI-*_9DTPMBwH<{3C2mCc{YQTpXaca zY$2nDAi4u#DFhF&QZ!24b6X$vuR^bL-Wf`cWSJpnvqYuKbsvgF7w?ZaGw-HacXc4nB$lA|?_ z{!lKb#CbBO|M2eYNRJcm4+h%ZD9H~%~$krl2WCrm%LBvaDEz$HluK3M4 zA&nvzovM3O)gcA3%2)Gv$ck|7L08U|oayF0;m&hdVe_68xaF`+XU82hj%55j07r;M z1&5_O46crNsLJMJ{=r49w6hdZ!hc_+BQ>o4HQHH7>PJyZKMB9*yj?)qg;+HC5NTmi zXy$i08)pPSLN?RhQ@X9+Fl}>o*HELMj0BIbFA}~szt8Ua=6dF8b>!*}Nh5HbJBKvRQRkeYUe(*tR}A&z)BKtM9Z2L-wg{OL3#alAyPfAzusKb zj>>#6=V%hddW6Q>@9|xKZ>|biMs$^@$^RFNt~stl!LDP$u2aDXuY8ux=T7nm_G?94 zug>T#az$ABfZkQ5a(k}%m~thWk5zcAMSQG9dMss5N--m4G>ZR0={84OvZ9y~O*n&= zIK%miqMAFyImQ#YoTZdfI$G#|klwPi;kD~rDW$?g3k@!GR769F8Wd9EN-qkbNOvZO z9k@cl@xb5gln?zVo+*x}=-Dds96+dXg2WPH0=)Zzoz~(D>0a$bW6g(mQ}~%)8h#Li zdI^391QJi@#qI&1L2;6vxT_w|hp3bi_m<@d)!U*^pmYkQG~&=_5Olh^wSh%_=$#fa zWg73t8W&=j{RrF_DJS()?W=a#d~B?3$PlOtTMLS|hU^$M;N_wHUOy1awh%dgoqr#7 z00M;ceK14k`38d7_PIBNy>H($9%X&9zu)`k;pdRkk9UrpKF^Jprba!`ZGELXwI+f9 zOsrZsKktwIp8`3}&QCNK4^CIDxt6UAl3e)rm;IlYrO%}yOu^ls!y$z>55TY2$0eqD zE5Pqrx_#KJ)WJSi-oW>9Zv7}QCnH1;Y7*_iA@-f}D~*{3yb&PTA!ss7CfenK@-hmT zxb?}7CyemIoID!0_;A-OnYr$MX>gi-4qS0``q^|}3j_Qn#&7s$?c6?htvGyU4y2DQ z|0v@a-0eeJ+MO+D!QnQ|t@ys>Ah)w#ZHL?wwJkaLr@_oTtBHN(Yy=kLz4>8j__rP& zs*^m;rS9?sp8_V{GoPpH)L>Zq^v<{Fh{8eBH+hJb)ozv*x?}|t9 zlIi^!m1+rym-Zf14&BSZ=-7&3|9P@*sk>X9ygq9lE6&1<<%7y^`$t|iuPrKT8=Cdo zV_bNom>AIN%4qe{fT4p)2-_0%a$^ZL_bux8O72JS*9RK~UAw~o@3 zs>QDU^Q=I2V7Khat@3kwvc@6-#?RZjuTqTBDMiHV@j4Sf6Lz#*Od+lxHn;OIyW(w% zNW-9E;09c(pNP#@{w--~zFX1k@l-(fH=s5BFI_Y9$BbY99HqKn*b@{nIZ{0vPoaah z>Aj?j$&H9eJp>``JdU2t-O=!$zTZql>Q6&XpWlp8d(wM^vMcA8!S>azIkDUrD(aK( zm`39E)sT8fAB{1mDa%W^#-e=}5qvWz% z*!Z$5#Dv3O(@YF%AMeZ-hG`XDqq$rK+m(Gy)6`h)#g104;+BaN;S2|A(iC)Jjq_dv zu``o5hst0mv#9jH{WO%1we|O@-YqQY$Ts$iJ%3h&qtM3_T^&bXak4*9M{SEvTt`qd zjDXy#jdJ?T?2wM-wN~J!5*koiBsAmNrM(^%7r#S%up6%62-U4SS9N=8u1Ahaq0}Z~ zlOszT7?IpSjR%M|siHft_s}#atLx3UN1xm{Yk;=Svj1RB@^crk_Kc}XPL>4d(<%Lv zn%tH<0`9xp%)BzDF3_J@WpFHHmQT|`S z@!W?@x$fMAgViyaK?PnVSoN@90k+QPcAIldT7^@OL(#}Maj-Q);J0Wiw!Ow~r&*be zz|7;psjw4)2EUKoI)X0_*1$__&SPhyQyyemeZoZE<=ccSJdL{1r6`UGvtb%i55*zdBk`LN{ zx~_&UiyWlysX_geP9{osYDEYHHM>E&!Q>F=FSphn^V>23u=Jj2n?3h3>z6PUv8Hqh zgL$)?nfPRP_Uy_4iHJN~cP3QNI^crunO>t8qZ0OtKlau+6TpZ`Ap^esIP_a@wxUJ-P-**^9%9_U;<8aQUph9n_>j-rU5Yh-?O9{ zZ7r7r9t{7!zQK4_H`;_29kn)l^uPGy(y8(qf?RuKxl^kU-k;AdWVnB0rn$~bZD`a) zF$Q^fc0D+xr1y#7l8u35N!}Pi7zzWxgBpOsjj%)u;JDb(7v{M7LmvNtrB^6={Hfa` zQv`;NT*PN1;sPwz1l*0o838B6Kwv>88&~*v{=r_WgDEdICjef8eT##DJ7t{+Py*qn zGt^bu0X3FFsDngihJY6KB&AFszz4_cUD5+PD)UA`cN}TPyOT%0P$Ejq2fJkG5zvGP z2xLJ*f;bb^Mjn;~ zxjhk$&sSz#W9wLm&~Ik@@B4#c@o@rnXy^^*HHx_tAr6rActVx-K!G+ihfrb9udFEI34rc5t}J?}+Q z3aT3)stK__z&Qlvj${5-VD?ne_n88Vo(hOGTFE9OdKY%5Y^9ZWUK@!5`itnd*~fX| z>ZhdMix3$sIfa%qkB#uSJ0;OijVgi#PG*m9nICYXgN%G;lz8E#+B_%teXms1lN<+m zS+?_ad}}bPnp?BoMquYxYpNgLx~uW?ed4jYR3`ZI0sGvmfa=PIJ-0A(U{y=#iYWei zZur9kO8VcEtGI*}M^>BuA)O(d+d(0`ta$k0kLlky1GuT@`F}_5*}?`O{F6^?hg!y^ zT>NR^df>!>zOPRQiQ#9c<)|`BsgDO9NKI<%>5mDl%qU`h@wV6cgELnG#3g@|FBaJ# zq@C%c1-$@LSGE`f^FXFsTMo)yfg9#@hj>xLmSk%k@Zkg$!ac|b(0cnXYl z;Q&C!5(wzFtu(x7^qW3nF(-zSYa0Zj0v=+NmMOmjBpB%cQsliqfXy`iLQ+BQ2!jD* zaD)UK@nC~CO2Sa0$&-&NW5u?-k6lr2G%}Euv+;C-L}czfZ8&I-4kT6uD}8I1DPv|{ zCUy9c4Ac>E9+^j|Y!q)!1HvmB;}BT(Umn0Ta*XFU_y?FF1_t*VPMwF<{ljcY`8!~_ zIE+puczv13p-}v`(!+U2tATCXMm`J_k=fFZ7TuvyS+c};8VIpV0_AbZv?&#^`v(R`{?oMoL@w0s?Tpl;piDTT^N&V@9(JA`g zoj#p3ur#k|^8UA}$dQpDC2XDwE5%g1`%3~m&mVTe`Ik*b-0GjS0OJ91M8fn5HQ%U9 zc-UB2ghjbs!u@2yA?R?8#8dLfHvo{>k-`tTbQE+VMdM`)p4+1{5C_G zl26m(k#6Z2U9#eMPkkNH;&{&tuEqXa-L5$QNj*zfjhC_l8PNfh(u6}<0lfyagTIac zT5Hs5!zgg@`hj(A?3uoHlo-yyS3-A8ggtwh(pK({?BD^$Z7s+2$bj`M*8)9e*f8a7 zWXiI{{b4R-McXEjxG2U#Lkf_h$+u;XvnSwX(s^BuF8Nx~B)q7=dmtzL z7*DZR5R#9Ji@&?CjV2`uk7-j20n2+v{sK?&*4#PcHj|sl>1r@p;tDVs^oh^xfo{SH zdga5?W9OBN8I-9?1<5a4BdQ=+-EN4NE(1~CDVzo`aXrF>wbV=K@?irarZ7US|q%p>fn~*(ZThak()y1 zGn8wRIr!Swri&5UsqsW2j^QXMCZuXMWQfeOS%LOM}z7To$lqZD%0h%FxNQwI9r= zFw|mGcn44umBC;=H)>ESp$@}p*QJiEnp*t1vIWLjX7>I&TRQZ{J9!|q86Q6LG@@MG z)clcoR7)Y|=44gU)z%|^`8B&NCEy;Fqn4^KQ7cl)Vo`&P_fP7WM#Y;(M#k<%`2t)@ zsTL*#ovf&BWE@a57W|l$RiY5zT1@=V)~Pi*vr4uQe&@`}5vBY*A7p#KIV=1%Ras`B z<(?SxE4umXxowX}Pp*WnCHPNoHQw)KZvPL|GOSkLJSG$oz-^gUX0}<9@?mi*)*{xJ z@E6PZ*{zhIPC3Rb@TwG3Uj1f4Q7(Uc^)Raq*v)s9-wMFFW~{0cw3$L)K5qeC&#tBk z!f9cZ0E_;dyGzHZ8P3ij<=z*6u_IvTuk7+B*)TVaH=+^;8T-TRYFvlDMPl6 zo@UrSc17F=#2~hL#jmgSaIN2Btkq)D;|dEzU1h@G;#|6A{#fZ@Pg49)j&w`6zBFlP zTI)d0)B}jUN*!Z)up_N|iT%bMa%VB5Z|T+E!4rhBDN)Rsias2oWledqOu3S+nk-Y3 zlQp}U8VHZs_GKH~QlavpZfU{jp!QXj-|tYvc8ZQ2(;(#0@Fu+9ENk6CFY2AaX+Kq0 z6%EknAc>OJ>UQ63ZPKkP>Yd#-kwM_pM?C@|Bl=@tN2H&>UTBs4UtSj&1 z{w-hJXjEB_r>Eb9t-`!V|K~k&Xx9e7DVtdxnc#UX-?SmS-K3vs(r-PnuvTA>#G z*U#(GOl6M63-$E6AHy;a4)o9@>n>y(N|jv6O|4K%HaGqq-qrFL4*i2G=FhP{XXWu`I~{`2s4?pnV6cJGIJWUv$2~paWb))nV7ONvzah68?zX5a0>AK|0&Wo z&A_Gpzdo6L-9r;y183~s>-1lhDl%O@i8`2=m=K*NQiIWLeuG9oOcXpit~-*?Kv)!x zQUk2J>e0py*+r(MapS^FXO^i)ll_D3V?3G9+$-MCJKybN))4RMY&La_nbvAu-)5>w zgjx`$W6&07DyW|3PAeVa6aYo4*F=jf_bj&eEc{0^`aXz*XSnfedAHMs(kDaodeFN3 z^&qelfN#zCBu@A=f^!zmzCZ}7I9I@NA>hW06g>QpV|1bt|1Min8xPNfg=-AOF*v#o zmSxOp8&|LnGQ@dl>|~iuJ2!d>Q&lj{tQfLKqz20t-K2!edZBLT4R9o_54|mc$$BKp6I3 z2YOB*j5T3`PI97)L#?t{arh0Gtyn&E0+AQ(`A2bxwrt^xBadk_JXSgf_c2|N8|mFj zoj$q!BrQ3#Q9d>|0`TG@#5CkFk6>q?zcM5CSs=wUqF@*$Z%0epcb*b%ns?9%j2s{at9OZCcY z^qy$+=~wtJNxn5k}8b7f|w+z0OUlN%Q zdJPYGk#98*fU&F-OS~aePn77pWKF>yHWT?n>@TD_W9HnUN8|6We=%zcU_BrXduH7s zOg-@)Nf`W8zZ}fFGksL9#>5n{GewhsLLoc0d`VQF$S{o zB5$KSDI8;_RZMDXPA|}e)3)Vtg(cZVC|na%TbhT$0-VPW3D#0_+qA9BVV6&%aQg%Eg}59eC`&ZL4{SWEytBtwkCdpVLRBzN$_IV4e9wG z0QL^)0zqO}AH{@+u;fwcLYOa%@{S8*Uy3i(;*KqM1hD~Y?vT4-?br6D@b)ycdx&ko z94pFbUHPi})4cnqZ;#6htQGll9H{A;L`NY}!q}%Fnss_r#+IHWeiW<0VU0X;2MQ~zAI5o~x_;X+B-6OzoyFf7 zACGTw^YVI6@DDkhb~2`$7U7rVx$Wmx?g5I>xP!}@o%J2mNwi0av?#4|ct`8><6W6J zE}bcf&t1@Fd1*W0pVqrx3ARHt>0ypE#!tR(vsgyQgsodXs@{Pnp7jIHStNlbKn|}e zX?peIU_7Vx*iL-;#w&E>%+#(Of{ZQsvJn#Y4=cH{7c^sx-dMC%& zBb`NS5?3z+rx!k5znW%Rg7yyg-%k2xV>prug&x8*L;mG=Wxg|hrsO$L-)x#kmV0g5gz9C`1!nz95MPr04rJiCc_ZVSa#LN%qK#X3QxQ z%-kV3-N1jh81o+GqRBD*@UZYkQV;(b=SK88QlHGDfz^G$$CJ?H9#~)yFt(1)Xu{z> zw9*byXXtSi=+qAXSWkI!C?7Mn(~jx0uf`Wg$pq|kC`C6Y;E6M5XwV&YypBj{46ber zkUe_Sq#^??w_Emz30(e=kzr31^2{LvvPpm&B_{ibT3x$U><%pPL=7+KP2(sgh_i9R zuN_0?LXj}~uZ1qOBXjNpu)Ml(GvgDTP~YV%Q=IG(y1ey%^d6zN`u|EHjf{5b(}L!MXBJ^!81gb**} zTd@!EGNnSJNIx)G3GldJV@riFwEthe`5=E2bi@;aG3xM}p$UQ^a;d6+Ic-FXc41j#7@NTKqUzx4_=k^n z7Mq%)gJL#AL6wDan`~`_`!z#aJ(qO?-pL%1)2)Kbg)*a4He0MO{iBpip~?YgR%e5v zgM4-?ud29kk*sY*YeSWsP>tMU`1}WRJBP0(^IFeI05HY&%LVCqT+$9v1n21Zp{Alhw*6uM-vsao?|r1=1O;| zfC0kDj}&ec$u>r=fxETe(KfWEfxdN!>M8)gp{d>cYqs$Ru?1(kE{($hoDhbSl^cef KTueb6=Klee Date: Wed, 22 Jul 2026 23:53:30 +0100 Subject: [PATCH 09/35] Split the verdict vocabulary: null vs promising (stop compressing opposite results) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit's finding #5: "directional" meant two opposite things in the ledger — a genuine null (SFT-masking and GRPO found nothing) AND a big SIGNIFICANT effect capped only by a missing HARD item or n<3 (data-mix +0.59 code BPB, significant). A ledger reader couldn't tell "found nothing" from "found something big, one gate short." For a rigor factory whose product IS trustworthy negatives, that ambiguity is the core defect (batch 1 Q4). ledger.py VERDICTS now distinguishes: win — HARD-complete, significant, single-variable + iso-FLOP (§C18, unchanged) promising — a real measured (usually significant) effect, capped below win by a missing HARD item / n<3 / unresolved confound. NOT a never_repeat loss. null — measured; no effect beyond the noise floor. A first-class negative result, NOT a never_repeat loss. loss — worse than baseline; auto never_repeat (unchanged). inconclusive— could not be measured/interpreted (crash, undecidable confound, no comparand). directional — kept valid but DEPRECATED; new runs use null|promising. Relabeled only the crystal-clear historical cases (verifiable-accuracy — read each verdict.json, no guessing): - grpo-phase2 -> null ("PREDICTED NULL CONFIRMED" in its verdict.json) - sft-3seed -> null (masking does not beat the iso-FLOP control) - scaling -> promising (significant +0.474/+0.126 early, converges, n=2 cap) - hybrid pilot -> inconclusive (n=1, no comparand) 5 runs stay 'directional' pending a careful per-run read (imu1-deconfound, arch-subdrill, the two data runs whose verdict.json says "recipe-level" yet whose README records big significant effects — that contradiction must be resolved before relabeling — and cce). Regression test added; 42 ledger tests still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- research/ledger/ledger.json | 8 ++++---- research/ledger/ledger.py | 14 +++++++++++++- research/tests/test_ledger.py | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/research/ledger/ledger.json b/research/ledger/ledger.json index 3c205ee..d962461 100644 --- a/research/ledger/ledger.json +++ b/research/ledger/ledger.json @@ -1178,7 +1178,7 @@ "started": "2026-06-27", "ended": null, "status": "done", - "verdict": "directional", + "verdict": "null", "metrics": { "suite_version": "text-lm-v2", "eval_harness_note": "1 representative seed/arm scored on the standard suite (held-out \u03c3\u22480.001 across seeds; full 3-seed CI is the in-domain reasoning_verdict.json)", @@ -1475,7 +1475,7 @@ "started": "2026-07-02", "ended": null, "status": "done", - "verdict": "directional", + "verdict": "null", "metrics": { "proceed_to_phase3": false, "conclusion": "PREDICTED NULL CONFIRMED: GRPO does not beat the SFT floor; GRPO does not beat the random-reward gate. At ~1% base pass@1 there is nothing for RL to sharpen (GRPO training reward flat at ~0.9% correct with no trend [first-50 0.0091 vs last-50 0.0080]; RFT's 352 collected completions, ~0.9% of 38,400 rollouts, were too few to separate from the floor). Do NOT spend the multi-seed cohort; the reasoning gain lives in SFT/distillation, not RL at this scale.", @@ -1555,7 +1555,7 @@ "started": "2026-07-05", "ended": "2026-07-12", "status": "done", - "verdict": "directional", + "verdict": "promising", "metrics": { "trend_verdict_wikitext": "CONVERGES", "trend_code_py": "CONVERGES", @@ -1695,7 +1695,7 @@ "started": "2026-07-19", "ended": "2026-07-20", "status": "done", - "verdict": "directional", + "verdict": "inconclusive", "metrics": { "ppl_wikitext2_val": 133.4628, "ppl_code_py": 5142.6426, diff --git a/research/ledger/ledger.py b/research/ledger/ledger.py index 273a3a4..5878dae 100644 --- a/research/ledger/ledger.py +++ b/research/ledger/ledger.py @@ -68,7 +68,19 @@ # serving/RAG/safeguards benches, and the sweep parent group. "scaling-fit", "replication", "serving-bench", "rag-eval", "safeguards-round", "sweep", "serve"} -VERDICTS = {"win", "loss", "inconclusive", "directional", None} # "directional" = §C25 eval-completeness cap (battery incomplete; not a never_repeat loss) +# Verdict vocabulary (2026-07-22: split "directional", which compressed opposite +# realities — a genuine null and a big significant-but-capped effect wore the same word). +# win — HARD-complete, significant, single-variable + iso-FLOP (§C18 gate below). +# promising — a REAL, measured (usually significant) effect, capped BELOW win by a +# missing §C25 HARD item, n<3 seeds, or an unresolved confound. +# "found something, one gate short." NOT a never_repeat loss. +# null — measured; the contrast shows NO effect beyond the noise floor. +# A first-class negative result ("found nothing"), NOT a never_repeat loss. +# loss — worse than baseline; auto-appends to never_repeat. +# inconclusive— could not be measured/interpreted (crash, undecidable confound, no comparand). +# directional — DEPRECATED alias, kept valid for old entries; new runs use null|promising. +VERDICTS = {"win", "promising", "null", "loss", "inconclusive", "directional", None} +NEUTRAL_VERDICTS = {"promising", "null", "directional", "inconclusive"} # measured, not a never_repeat loss PROP_KINDS = {"new-build", "big-run", "needs-approval"} PROP_STATUS = {"open", "accepted", "declined"} PAPER_STATUS = {"drafting", "packaged", "submitted", "published", "abandoned"} # §C16 diff --git a/research/tests/test_ledger.py b/research/tests/test_ledger.py index 44c6654..24d5346 100644 --- a/research/tests/test_ledger.py +++ b/research/tests/test_ledger.py @@ -372,6 +372,24 @@ def test_the_run_type_lint_catches_the_original_bug(tmp_path): assert all(t not in ledger.RUN_TYPES for t in found.values()) # ...and flagged invalid +def test_split_verdict_vocabulary(ledger_path): + """2026-07-22: 'directional' was split into null / promising so a genuine negative + result and a big-but-capped effect stop sharing one word. Both must be accepted, and + neither may auto-append to never_repeat (only 'loss' does).""" + run(ledger_path, "add-technique", "--slug", "t", "--title", "X") + for i, verdict in enumerate(("null", "promising", "inconclusive", "directional")): + rid = f"2026-07-22_m_v{i}" + assert run(ledger_path, "add-run", "--run-id", rid, "--type", "eval") == 0 + assert run(ledger_path, "update-run", rid, "--set", f'verdict="{verdict}"') == 0, verdict + assert reload(ledger_path)["runs"][-1]["verdict"] == verdict + assert reload(ledger_path)["never_repeat"] == [], "neutral verdicts must not never_repeat" + # a 'loss' still does + rid = "2026-07-22_m_loss" + run(ledger_path, "add-run", "--run-id", rid, "--type", "ablation", "--technique-slug", "t") + run(ledger_path, "update-run", rid, "--set", 'verdict="loss"') + assert "t" in reload(ledger_path)["never_repeat"] + + def test_the_run_type_lint_accepts_a_valid_caller(tmp_path): """No false positives: the FIXED argv shape must not be flagged. Mirrors the real score_ladder.py, which reaches the CLI through a `LEDGER = ROOT / ".../ledger.py"` From e38979382ed3d8cd57725b99d9719191129a59e4 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Wed, 22 Jul 2026 23:57:48 +0100 Subject: [PATCH 10/35] Write score_arch_ladder.py + make the driver's scoring hook loud (fix the silent no-op) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found run_arch_ladder.sh's completion hook was `[ -f score_arch_ladder.py ] && python score_arch_ladder.py` — but the scorer never existed, so a completed ladder produced NO comparable numbers and nothing said so. - score_arch_ladder.py REUSES the proven JAX suite functions from the pilot's eval_suite_jax.py (load_model / ppl / load_corpora), patching the per-arm config per cell — it does NOT re-implement scoring (the 6-copies-of-score_cohort anti-pattern the audit flagged). It reports cross-arm val PPL (the valid cross-arm metric: shared tokenizer/corpora/windows) + the emergence-speed curve (ppl vs log10 tokens per arm, gap-vs-base per rung). It deliberately does NOT stamp a hand-rolled BPB — cross-study BPB comes from the single consolidated eval-harness (upgrade-plan item 6), not a copy here. - --smoke runs the pure aggregation/curve math on CPU (no GPU, no model load); PASSES. GPU end-to-end validation is pending the first rung gap (§C4.5: score only when no trainer is live). - The driver hook now logs LOUDLY on a missing or failing scorer instead of silently passing — "ladder COMPLETE but UNSCORED" is a visible state, not silence. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_arch_ladder.sh | 14 +- .../score_arch_ladder.py | 137 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh index dc78881..f9ddec8 100755 --- a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh @@ -149,7 +149,19 @@ while read -r id _; do [ -f "$LDIR/${id}.done" ] || { missing=$((missing+1)); ec if [ "$missing" -eq 0 ]; then touch "$LDIR/ladder.done" echo "===== $(date '+%F %T') ARCH LADDER COMPLETE — $LDIR/ladder.done =====" - [ -f "$LDIR/score_arch_ladder.py" ] && $PY "$LDIR/score_arch_ladder.py" + # Score at ladder end. LOUD on absence/failure — the audit found this hook was a + # silent no-op (the scorer did not exist), so a completed ladder produced NO numbers + # and nothing said so. No trainer is live here (all cells done), so §C4.5 permits it. + if [ -f "$LDIR/score_arch_ladder.py" ]; then + echo "[$(date '+%T')] scoring ladder..." + if $PY "$LDIR/score_arch_ladder.py"; then + echo "[$(date '+%T')] scoring done -> arch_ladder_scores.json" + else + echo "[$(date '+%T')] !! SCORING FAILED (rc=$?) — cells are trained but UNSCORED; run score_arch_ladder.py by hand" + fi + else + echo "[$(date '+%T')] !! NO SCORER (score_arch_ladder.py absent) — ladder COMPLETE but UNSCORED" + fi else echo "===== $(date '+%F %T') LADDER INCOMPLETE — $missing cells missing .done =====" fi diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py new file mode 100644 index 0000000..0e10add --- /dev/null +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""score_arch_ladder.py — the arch-ladder's completion/rung scorer. + +Fixes the audit finding that run_arch_ladder.sh's scoring hook was a silent no-op +(this file did not exist). It REUSES the proven JAX suite functions from the build +dir's eval_suite_jax.py (load_model / ppl / load_corpora) — it does NOT re-implement +scoring (the 6-copies-of-score_cohort anti-pattern the audit flagged). It patches the +per-arm config (mixer / attn_every / nope / checkpoint) onto that module per cell. + +Metric: val PPL with the model's own Qwen3 tokenizer on the same wikitext-2 + code +corpora the suite pins (suite_version text-lm-v2). Because every cell shares the +tokenizer, corpora, and window settings, PPL is directly COMPARABLE across arms — +this is the valid cross-arm ranking metric. It is NOT stamped as BPB: the audit +flagged that BPB was being stamped on text-lm-v2 which does not define it, and a +cross-STUDY BPB must come from the SINGLE consolidated eval-harness (upgrade-plan +item 6), not a hand-rolled copy here. So this scorer reports PPL cross-arm + the +emergence-speed curve, and defers a BPB number to eval-harness. + +Modes: + --smoke : CPU-only structural self-test of the pure aggregation/curve math (no GPU, + no model load) — safe to run beside a live trainer. + (default): GPU — score every .done cell in cells.json. §C4.5: run ONLY when no + trainer is live (the driver calls this at a rung gap / at ladder end). + +GPU-VALIDATION-PENDING: the per-cell scoring path (load_model+ppl) is proven (it scored +the pilot), but this orchestrator's end-to-end run has not yet executed on GPU — it will +at the first rung gap. Until then treat the emitted numbers as unproduced. +""" +from __future__ import annotations +import json +import math +import pathlib +import sys + +LDIR = pathlib.Path(__file__).resolve().parent +BUILD = LDIR.parent / "2026-07-19_hybrid-ssm-0.2b_build" +ROOT = LDIR.parents[2] +sys.path.insert(0, str(ROOT / "research")) + + +def load_cells(): + return json.loads((LDIR / "cells.json").read_text()) + + +def emergence_curve(per_arm_rung_ppl: dict) -> dict: + """per_arm_rung_ppl[arm] = {tokens: ppl}. Fit ppl vs log10(tokens) per arm (OLS) + and report the slope (emergence speed) + the arm-minus-base gap at each rung. + Pure math — the part this file unit-tests on CPU.""" + curves = {} + base = per_arm_rung_ppl.get("ssm_base", {}) + for arm, pts in per_arm_rung_ppl.items(): + xs = [math.log10(t) for t in sorted(pts)] + ys = [pts[t] for t in sorted(pts)] + slope = intercept = None + if len(xs) >= 2: + n = len(xs); sx = sum(xs); sy = sum(ys) + sxx = sum(x * x for x in xs); sxy = sum(x * y for x, y in zip(xs, ys)) + denom = n * sxx - sx * sx + if denom != 0: + slope = (n * sxy - sx * sy) / denom + intercept = (sy - slope * sx) / n + gap_vs_base = {t: round(pts[t] - base[t], 4) for t in sorted(pts) if t in base} + curves[arm] = {"points": {str(t): round(pts[t], 4) for t in sorted(pts)}, + "ppl_vs_log10tok_slope": (round(slope, 4) if slope is not None else None), + "intercept": (round(intercept, 4) if intercept is not None else None), + "gap_vs_base": gap_vs_base} + return curves + + +def _smoke() -> int: + # synthetic: ssm improves fastest, swa lags — assert the curve math is sane. + fake = {"ssm_base": {42_000_000: 4.7, 85_000_000: 4.2, 150_000_000: 3.9}, + "swa128": {48_000_000: 5.7, 96_000_000: 5.3, 170_000_000: 5.0}} + c = emergence_curve(fake) + assert c["ssm_base"]["ppl_vs_log10tok_slope"] < 0, "ppl must fall with tokens" + assert c["swa128"]["ppl_vs_log10tok_slope"] < 0 + assert c["ssm_base"]["gap_vs_base"][42_000_000] == 0.0 + # base has no gap key for swa's token budgets (different rung tokens) -> ok + assert set(c) == {"ssm_base", "swa128"} + print("SMOKE PASS: emergence_curve math sane (slopes negative, base gap 0)") + return 0 + + +def _score_all() -> int: + """GPU path — score every .done cell. §C4.5: caller guarantees no live trainer.""" + sys.path.insert(0, str(BUILD)) + import importlib + esj = importlib.import_module("eval_suite_jax") + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(esj.TOKENIZER_REPO) + + cells = load_cells() + per_arm_rung = {} # arm -> {tokens: {corpus: ppl}} + scored = [] + for cell in cells["cells"]: + done = LDIR / f"{cell['id']}.done" + ckpt = LDIR / f"checkpoint_{cell['id']}.pkl" + if not done.exists() or not ckpt.exists(): + continue + # patch the suite module's per-arm config, then reuse its proven loader/scorer + esj.MIXER, esj.ATTN_EVERY, esj.NOPE = cell["mixer"], cell["attn_every"], cell["nope"] + esj.TARGET_CKPT = str(ckpt) + params, forward = esj.load_model() + corpora, _ = esj.load_corpora(tok) + row = {"cell": cell["id"], "arm": cell["arm"], "tokens": cell["tokens"], "ppl": {}} + for name, c in corpora.items(): + v, n = esj.ppl(params, forward, tok, c["text"]) + row["ppl"][name] = round(v, 4) + scored.append(row) + per_arm_rung.setdefault(cell["arm"], {}).setdefault(cell["tokens"], row["ppl"]) + print(f"[scored] {cell['id']}: {row['ppl']}", flush=True) + + # per-corpus emergence curve (use the headline corpus present in most rows) + corpus = "wikitext2_val" + per_arm_ppl = {arm: {tok_n: cor.get(corpus) for tok_n, cor in rungs.items() if corpus in cor} + for arm, rungs in per_arm_rung.items()} + per_arm_ppl = {a: p for a, p in per_arm_ppl.items() if p} + out = {"suite_version": "text-lm-v2", "metric": f"val PPL ({corpus}), own tokenizer, n=1/cell", + "comparability": "cross-arm PPL only (same tokenizer/corpora/windows); " + "cross-study BPB deferred to the consolidated eval-harness", + "cells_scored": len(scored), "rows": scored, + "emergence_curve": emergence_curve(per_arm_ppl), + "caveat": "n=1 per cell -> DIRECTIONAL (§C17); mixer-type gaps carry the " + "LR-not-retuned-per-arm confound (see rung_42M_comparison.md)."} + (LDIR / "arch_ladder_scores.json").write_text(json.dumps(out, indent=2) + "\n") + print(f"[done] wrote arch_ladder_scores.json ({len(scored)} cells)") + return 0 + + +def main() -> int: + if "--smoke" in sys.argv: + return _smoke() + return _score_all() + + +if __name__ == "__main__": + raise SystemExit(main()) From b9ce4e3abe760eb690218ddf30aefcba641a9ef0 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 00:03:28 +0100 Subject: [PATCH 11/35] Add a fixed-reference noise floor (item 7) + correct the stale text-lm-v3 status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Item 7 (batch 6 Q4): self-floor noise floors on undertrained checkpoints are 15-26% of PPL (measured on the HybridSSM pilot), making the standing baseline useless for future significance calls. eval_stats.noise_floor(target, reference=None) prefers a stable reference checkpoint's subsample spread over the target's wobbly self-floor, and RETURNS the basis ("reference"|"self") so a wide self-floor is never silently read as a real significance bar. Same max-min primitive as the legacy self-floor, so it stays comparable to prior self-floored numbers. Tested. On item 6 (one real eval-harness): the audit's "6 hand-rolled copies re-implement the suite" is partly overstated — the canonical PRIMITIVES already exist and are tested (eval_metrics.bits_per_byte, eval_stats.subsample_noise_floor / seed_delta_significant), and the score_cohort copies import them. What actually duplicates is the thin per-model windowed-CE score LOOP, which is legitimate experiment-isolation glue. So the consolidation is smaller than framed; the real integrity gap was the noise-floor basis (fixed above) and the suite stamp. Also corrected a stale, contradictory line in the eval-harness suite.md status table (local-only, under .claude/): text-lm-v3 was marked "flips ACTIVE on the first GPU eval run / no v3 number yet", but the v3 downstream battery already scored 25 checkpoints 2026-06-24. Flagged the deeper governance call (flip the v3 core, or keep v3 downstream-only) as a deliberate TODO rather than deciding it unilaterally, and documented that BPB is the §C10 headline reported under v2. Co-Authored-By: Claude Opus 4.8 (1M context) --- research/eval_stats.py | 19 +++++++++++++++++++ research/tests/test_eval_stats.py | 15 +++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/research/eval_stats.py b/research/eval_stats.py index 2f206b3..20fb0c2 100644 --- a/research/eval_stats.py +++ b/research/eval_stats.py @@ -77,6 +77,25 @@ def subsample_noise_floor(values): return max(vals) - min(vals) +def noise_floor(target_subsamples, reference_subsamples=None): + """Canonical noise-floor selector (2026-07-22 decision, upgrade-plan item 7). + + The SELF-floor (subsamples of the target itself) is unusably wide on an + UNDERTRAINED checkpoint — measured 15-26% of PPL on the HybridSSM pilot, which + makes the standing baseline useless for future significance calls. When a stable + REFERENCE checkpoint's subsamples are available, compute the floor on THOSE + instead; a well-trained reference has a tight, stable subsample spread that is a + meaningful significance yardstick across runs. + + Returns (floor_abs, basis) where basis is "reference" (preferred) or + "self" (fallback) — the caller MUST record `basis` so a wide self-floor is never + silently read as a real significance bar. Uses the same max-min primitive as the + legacy self-floor, so numbers stay comparable to prior self-floored results.""" + if reference_subsamples: + return subsample_noise_floor(reference_subsamples), "reference" + return subsample_noise_floor(target_subsamples), "self" + + def _welch_df(sem_b, n_b, sem_t, n_t): """Welch–Satterthwaite degrees of freedom from the per-arm SEMs and n's. Requires n_b, n_t >= 2 (callers guarantee this before calling).""" diff --git a/research/tests/test_eval_stats.py b/research/tests/test_eval_stats.py index 4d72f1c..c640cda 100644 --- a/research/tests/test_eval_stats.py +++ b/research/tests/test_eval_stats.py @@ -114,3 +114,18 @@ def test_asymmetric_single_seed_not_significant(): # One good arm, one single-seed arm -> still not significance-testable. r = es.seed_delta_significant([28.6, 28.7, 28.5], [23.5]) assert r["significant"] is False and r["n_treatment"] == 1 + + +def test_noise_floor_prefers_reference_and_records_basis(): + """upgrade-plan item 7: prefer a stable reference checkpoint's subsamples over an + undertrained target's wide self-floor, and always report which basis was used.""" + wobbly_target = [130.0, 150.0, 138.0] # undertrained: ~20 spread (huge) + tight_reference = [12.0, 12.3, 12.1] # well-trained: ~0.3 spread + floor, basis = es.noise_floor(wobbly_target, tight_reference) + assert basis == "reference" + assert abs(floor - 0.3) < 1e-9 # max-min of the reference, not the target + # no reference -> falls back to self, flagged as such + floor2, basis2 = es.noise_floor(wobbly_target) + assert basis2 == "self" and abs(floor2 - 20.0) < 1e-9 + # empty reference is treated as absent (fallback to self), not a crash + assert es.noise_floor(wobbly_target, [])[1] == "self" From f6a23f91651800230640278cd3cfcac392b32392 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 00:04:06 +0100 Subject: [PATCH 12/35] Upgrade plan: log 2026-07-22 execution (6 items done) + corrected item-6 framing Co-Authored-By: Claude Opus 4.8 (1M context) --- research/LOOP_UPGRADE_PLAN_2026-07-22.md | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/research/LOOP_UPGRADE_PLAN_2026-07-22.md b/research/LOOP_UPGRADE_PLAN_2026-07-22.md index f6dc307..35210c3 100644 --- a/research/LOOP_UPGRADE_PLAN_2026-07-22.md +++ b/research/LOOP_UPGRADE_PLAN_2026-07-22.md @@ -44,3 +44,29 @@ Derived from the 28-decision audit (`LOOP_AUDIT_2026-07-22.md`). One-line identi ## The one sequencing question for you Item 9: insert NorMuon-at-scale after the current 85M rung (pausing the hybrid ladder ~34h), or let the whole hybrid ladder finish first? Your batch-1 answer said "NorMuon next GPU-week," which implies the former — confirm when you want it. + +## Execution log — 2026-07-22 (this session) + +DONE (committed + pushed): +- Tier 0 #2 track the record (0409620) · #3 arXiv package rebuild (66f2953). +- Tier 1 #8 verdict vocabulary split + relabel 4 clear runs (79a514f). +- Tier 1 #4 score_arch_ladder.py + loud driver hook (e389793). +- Tier 1 #7 fixed-reference noise floor + stale-v3-status correction (b9ce4e3). +- Tier 0 #1 recovery cron lines PRINTED for the user to paste (cron_logs/ created) — awaiting paste. + +CORRECTED FRAMING: +- #6 "one real eval-harness": the canonical PRIMITIVES already exist + are tested + (eval_metrics.bits_per_byte, eval_stats.subsample_noise_floor / seed_delta_significant / + noise_floor). The 6 score_cohort copies differ only in the thin per-model windowed-CE + score LOOP (legitimate experiment-isolation glue). So #6 is smaller than the audit framed; + the real integrity gaps were the noise-floor basis (#7, DONE) and the suite stamp (DONE). + Remaining #6 work (optional): extract the shared score-loop into one importable helper. + +BLOCKED (GPU + human trigger, per propose-only): +- #5 per-arm LR probe (de-confounds the 42M mixer finding) — run at a rung gap. +- #9 NorMuon-at-scale — run at a rung gap; sequencing still to confirm. +- score_arch_ladder.py GPU end-to-end validation — first rung gap. + +GOVERNANCE TODO flagged, not decided unilaterally: +- Flip the v3 core PPL/BPB section ACTIVE vs keep v3 downstream-only (suite.md). +- Relabel the 5 still-'directional' runs after resolving the data-run verdict.json vs README contradiction. From 7852728a5bd5849505bc8ebd82b14a640e6d0888 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 00:06:57 +0100 Subject: [PATCH 13/35] Un-strand finished techniques so next-best stops recommending completed work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found next-best "actively wrong today": techniques that finished their runs were stuck at status=briefed, so the loop's selector kept offering finished work as runnable. Fixed the clear cases (verifiable from run_ids + run statuses): - normuon-optimizer briefed -> done (produced the win + scaling study) - vibethinker-small-reasoning briefed -> done (5/5 runs done) - midtrain-anneal-premium-mix briefed -> done (1/1 run done) - hybrid-attention-rethink briefed -> running (owns the live arch ladder) zeta-dual-whitening correctly stays briefed (0 runs — declined at the 2026-07-14 interactive gate, genuinely awaiting launch). next-best now returns only zeta-dual-whitening as the top runnable candidate, instead of a pile of finished techniques. Technique histogram is now honest: briefed=1, candidate=14, done=4, proposal=1, running=1. Co-Authored-By: Claude Opus 4.8 (1M context) --- research/ledger/ledger.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/research/ledger/ledger.json b/research/ledger/ledger.json index d962461..e930c3e 100644 --- a/research/ledger/ledger.json +++ b/research/ledger/ledger.json @@ -143,7 +143,7 @@ "source_url": "https://arxiv.org/abs/2510.05491", "paper_date": "2025-10-06", "first_seen": "2026-06-16", - "status": "briefed", + "status": "done", "objective": "pretrain-ablation", "taxonomy": [], "score": null, @@ -160,7 +160,7 @@ "source_url": "https://arxiv.org/abs/2606.16140", "paper_date": "2026-06-15", "first_seen": "2026-06-17", - "status": "briefed", + "status": "done", "objective": "finetune", "taxonomy": [ "architecture: decoder-only", @@ -248,7 +248,7 @@ "source_url": "https://arxiv.org/abs/2606.15378", "paper_date": "2026-06-13", "first_seen": "2026-06-17", - "status": "briefed", + "status": "running", "objective": "pretrain-ablation", "taxonomy": [ "architecture:hybrid", @@ -321,7 +321,7 @@ "source_url": null, "paper_date": null, "first_seen": "2026-06-30", - "status": "briefed", + "status": "done", "objective": "pretrain-ablation", "taxonomy": [ "training-stage", From 8ab1d2b4c43d3a836bd90eaafdf82b084f3d6cf0 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 07:23:26 +0100 Subject: [PATCH 14/35] Test the safety-killers first: thermal-kill path + safe_cuda/jax_safe_env guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found the newest, most box-crash-critical code had ZERO tests, and batch 7 Q2 chose "test the killers before the features." These are the code paths that prevent (or cause) a whole-box hard-lock, so they get tested first. Thermal-kill path (sentinel.py, added because the GB10 hard-locks from heat): - gpu_thermal() parses the nvidia-smi temp + hw/sw throttle CSV, and MUST fail open — a dead/absent nvidia-smi returns (None, False), never a fabricated high temp that would SIGTERM a healthy trainer. Tested incl. the exact "Not Active" != "Active" substring trap. - hottest_soc_c() picks the max ACPI zone and fails open on unreadable zones. - Constants guard: >=2 consecutive over-limit samples before a kill (a single nvidia-smi blip must not fire), WARN < KILL, ceiling set high on purpose. L1 memory guards (safe_cuda, jax_safe_env) — new test_guards.py, pure CPU (the validation/env logic runs before any torch/jax call, so no GPU needed): - safe_cuda.guard() rejects an unsafe fraction (<=0 or >0.95) BEFORE touching torch — the check that stops a 0.99/typo'd 1.5 from handing the whole unified pool to one process; default stays the §C1 0.85; import composes PYTORCH_CUDA_ALLOC_CONF. - jax_safe_env sets PREALLOCATE=false + MEM_FRACTION=0.5 on import, and REFUSES (RuntimeError) if PREALLOCATE=true is already set — the ~90GB startup grab that would crash the box. Full gate green: 433 passed, 1 skipped, with the trainer live. Co-Authored-By: Claude Opus 4.8 (1M context) --- research/tests/test_guards.py | 91 +++++++++++++++++++++++++++++++++ research/tests/test_sentinel.py | 58 +++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 research/tests/test_guards.py diff --git a/research/tests/test_guards.py b/research/tests/test_guards.py new file mode 100644 index 0000000..614765a --- /dev/null +++ b/research/tests/test_guards.py @@ -0,0 +1,91 @@ +"""Tests for the L1 memory guards every GPU process depends on — safe_cuda (torch) +and jax_safe_env (JAX). Upgrade-plan item 16 (batch7 Q2: test the killers first): +these had NO tests, yet their pure-CPU logic — fraction-range validation, env-var +composition, and the PREALLOCATE=true refusal — is the first line of defense against +the unified-memory over-allocation that hard-crashed the box on 2026-06-08. + +All tests here are pure CPU: the validation/env logic runs BEFORE any torch/jax call, +so no GPU (and no torch/jax install) is needed to exercise the safety-critical paths. +""" +import importlib +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + + +# ------------------------------------------------------------------ safe_cuda +import safe_cuda # noqa: E402 (imports os only; torch is imported inside guard()) + + +@pytest.mark.parametrize("bad", [0.0, -0.1, 0.96, 1.0, 1.5, 2.0]) +def test_guard_rejects_unsafe_fraction_before_touching_torch(bad): + """guard() validates 0 < fraction <= 0.95 and raises BEFORE the torch call, so an + unsafe fraction is refused even on a box with no CUDA. This is the check that stops + a 0.99 (or a typo'd 1.5) from handing the whole unified pool to one process.""" + with pytest.raises(ValueError): + safe_cuda.guard(bad) + + +def test_safe_cuda_import_sets_alloc_conf(): + """Importing safe_cuda must compose PYTORCH_CUDA_ALLOC_CONF (expandable segments etc.) + so fragmentation doesn't trip the OOM-killer on the shared pool.""" + assert "PYTORCH_CUDA_ALLOC_CONF" in os.environ + assert os.environ["PYTORCH_CUDA_ALLOC_CONF"] != "" + + +def test_guard_default_fraction_is_the_documented_085(): + import inspect + sig = inspect.signature(safe_cuda.guard) + assert sig.parameters["fraction"].default == 0.85 # §C1 default; must not drift + + +# ------------------------------------------------------------------ jax_safe_env +def _reload_jax_safe_env(env): + """Import jax_safe_env fresh under a controlled environment, returning (module|exc).""" + for k in ("XLA_PYTHON_CLIENT_PREALLOCATE", "XLA_PYTHON_CLIENT_MEM_FRACTION"): + os.environ.pop(k, None) + os.environ.update(env) + sys.modules.pop("jax_safe_env", None) + try: + return importlib.import_module("jax_safe_env"), None + except Exception as e: # noqa: BLE001 + return None, e + + +def test_jax_safe_env_sets_no_prealloc_and_half_pool(): + saved = {k: os.environ.get(k) for k in + ("XLA_PYTHON_CLIENT_PREALLOCATE", "XLA_PYTHON_CLIENT_MEM_FRACTION")} + try: + mod, exc = _reload_jax_safe_env({}) + assert exc is None + assert os.environ["XLA_PYTHON_CLIENT_PREALLOCATE"] == "false" # no ~90GB startup grab + assert os.environ["XLA_PYTHON_CLIENT_MEM_FRACTION"] == "0.5" # cap at ~60GB + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + sys.modules.pop("jax_safe_env", None) + + +def test_jax_safe_env_refuses_prealloc_true(): + """If a caller already set PREALLOCATE=true (the ~90GB unified-pool grab that would + crash the box), importing the guard must REFUSE loudly rather than proceed.""" + saved = {k: os.environ.get(k) for k in + ("XLA_PYTHON_CLIENT_PREALLOCATE", "XLA_PYTHON_CLIENT_MEM_FRACTION")} + try: + mod, exc = _reload_jax_safe_env({"XLA_PYTHON_CLIENT_PREALLOCATE": "true"}) + assert mod is None and isinstance(exc, RuntimeError) + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + sys.modules.pop("jax_safe_env", None) diff --git a/research/tests/test_sentinel.py b/research/tests/test_sentinel.py index 975fae9..4e9f6c8 100644 --- a/research/tests/test_sentinel.py +++ b/research/tests/test_sentinel.py @@ -381,3 +381,61 @@ def boom(): monkeypatch.setattr(sentinel, "meminfo", boom) assert sentinel.preflight() == 1 assert "failing closed" in capsys.readouterr().out + + +# ------------------------------------------- thermal-kill path (upgrade-plan item 16) +# The thermal guard is the NEWEST, least-mature safety layer, added because the GB10 +# hard-locks from heat — and it had ZERO tests. Both readers MUST fail OPEN (an +# unreadable sensor returns None/False, never a fabricated high temp that would kill a +# healthy trainer), and the throttle parse must not confuse "Not Active" with "Active". + +class _FakeProc: + def __init__(self, stdout): self.stdout = stdout + +def test_gpu_thermal_parses_temp_and_throttle(monkeypatch): + monkeypatch.setattr(sentinel.subprocess, "run", + lambda *a, **k: _FakeProc("72, Not Active, Not Active\n")) + assert sentinel.gpu_thermal() == (72.0, False) + # hw slowdown active -> throttling True + monkeypatch.setattr(sentinel.subprocess, "run", + lambda *a, **k: _FakeProc("91, Active, Not Active\n")) + assert sentinel.gpu_thermal() == (91.0, True) + # sw slowdown active -> throttling True + monkeypatch.setattr(sentinel.subprocess, "run", + lambda *a, **k: _FakeProc("88, Not Active, Active\n")) + assert sentinel.gpu_thermal() == (88.0, True) + +def test_gpu_thermal_not_active_is_not_active(monkeypatch): + """The exact substring trap: 'Not Active' must NOT read as throttling.""" + monkeypatch.setattr(sentinel.subprocess, "run", + lambda *a, **k: _FakeProc("60, Not Active, Not Active\n")) + _, throttling = sentinel.gpu_thermal() + assert throttling is False + +def test_gpu_thermal_fails_open(monkeypatch): + """A dead/absent nvidia-smi must never fabricate heat (else it kills a healthy run).""" + def boom(*a, **k): raise OSError("no nvidia-smi") + monkeypatch.setattr(sentinel.subprocess, "run", boom) + assert sentinel.gpu_thermal() == (None, False) + monkeypatch.setattr(sentinel.subprocess, "run", + lambda *a, **k: _FakeProc("garbage no digits\n")) + assert sentinel.gpu_thermal() == (None, False) + +def test_hottest_soc_c_picks_max(monkeypatch, tmp_path): + zones = [] + for milli in (45000, 78000, 60000): # 45C, 78C, 60C + z = tmp_path / f"zone{milli}"; z.write_text(str(milli)) + zones.append(str(z)) + monkeypatch.setattr("glob.glob", lambda pat: zones) # hottest_soc_c imports glob locally + assert sentinel.hottest_soc_c() == 78.0 + +def test_hottest_soc_c_fails_open(monkeypatch): + """No readable zones -> None, never a fabricated temperature.""" + monkeypatch.setattr("glob.glob", lambda pat: []) + assert sentinel.hottest_soc_c() is None + +def test_thermal_constants_debounced_and_ordered(): + """A single nvidia-smi blip must not trigger a kill (>=2 consecutive), and WARN < KILL.""" + assert sentinel.TEMP_KILL_CONSECUTIVE >= 2, "single-sample kill would false-fire on a blip" + assert sentinel.TEMP_WARN_C < sentinel.TEMP_KILL_C + assert sentinel.TEMP_KILL_C >= 85, "thermal ceiling set high on purpose (load temps uninstrumented)" From 73ad5beec00cbd398055dac268a17b38c85f935c Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 07:23:53 +0100 Subject: [PATCH 15/35] Plan log: safety-killer tests + un-strand done (2026-07-23) Co-Authored-By: Claude Opus 4.8 (1M context) --- research/LOOP_UPGRADE_PLAN_2026-07-22.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/research/LOOP_UPGRADE_PLAN_2026-07-22.md b/research/LOOP_UPGRADE_PLAN_2026-07-22.md index 35210c3..7c4c95f 100644 --- a/research/LOOP_UPGRADE_PLAN_2026-07-22.md +++ b/research/LOOP_UPGRADE_PLAN_2026-07-22.md @@ -70,3 +70,12 @@ BLOCKED (GPU + human trigger, per propose-only): GOVERNANCE TODO flagged, not decided unilaterally: - Flip the v3 core PPL/BPB section ACTIVE vs keep v3 downstream-only (suite.md). - Relabel the 5 still-'directional' runs after resolving the data-run verdict.json vs README contradiction. + +## Execution log — 2026-07-23 (continued) +- Tier 2 #12 (partial): un-stranded 4 finished techniques so next-best is correct again (7852728). +- Tier 3 #16 (safety-killers first): thermal-kill path + safe_cuda + jax_safe_env now tested + (test_guards.py new; test_sentinel.py +6 thermal tests). Full gate 433 passed (8ab1d2b). +- All pushed to origin/harden-research-loop. +Remaining Tier 2/3 (each needs a bit of your steer): #10 adopted-run protocol, #11 c5 schema+lint / + CLAUDE.md verdict-timing, #13 dedup, #14 calibration wiring, #15 doc reframe + meta-scrub, + #17 shared-box lock, #18 loop_state fsync, #19 kdump [HUMAN]. GPU items #5/#9 still await a rung gap + go. From dcf01e0fe264463bb0c301c4563e36a54ad702fa Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 13:33:29 +0100 Subject: [PATCH 16/35] Harden dedup: catch the same paper under a different slug (arXiv id + fuzzy title) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found check-dup was exact-slug-string only — it depended on three separate LLM invocations minting IDENTICAL slugs, so the same paper could enter the ledger twice under different slugs (and a real technique already bypassed the eligibility window). Duplicate briefs/runs cost real GPU (batch 4 Q4). check-dup now, in addition to the exact-slug + never_repeat check: - extracts the arXiv id from --source-url (version-suffix stripped, so v1/v2 dedup) and flags a match against any existing technique's source_url; - fuzzy-matches --title by normalized token Jaccard (>= 0.85), so punctuation/case/ spacing differences collapse. Deterministic, no external fuzzy lib. Verified against the LIVE ledger: a re-entry of zeta-dual-whitening (arxiv 2606.14187) under a new slug is caught by both id and title, while a genuinely new candidate stays NEW. New helpers arxiv_id() / norm_title_tokens() / title_jaccard() + check-dup gains --source-url / --title. Tests added; ledger suite green. Co-Authored-By: Claude Opus 4.8 (1M context) --- research/ledger/ledger.py | 63 +++++++++++++++++++++++++++++++++-- research/tests/test_ledger.py | 32 ++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/research/ledger/ledger.py b/research/ledger/ledger.py index 5878dae..7bf17cc 100644 --- a/research/ledger/ledger.py +++ b/research/ledger/ledger.py @@ -597,14 +597,67 @@ def cmd_update_paper(led, a): # §C16 # ---------------------------------------------------------------- readers +_ARXIV_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/(\d{4}\.\d{4,5})", re.I) + + +def arxiv_id(url): + """The bare arXiv id (e.g. '2606.14187') from a source URL, else None. Strips any + version suffix so v1/v2 of the same paper dedup together.""" + if not url: + return None + m = _ARXIV_RE.search(url) + return m.group(1) if m else None + + +def norm_title_tokens(title): + """Lowercase alphanumeric token set of a title — the fuzzy-match key. Punctuation, + case, and spacing differences collapse away so 'Zeta: Dual Whitening' and + 'zeta dual whitening' match.""" + if not title: + return frozenset() + return frozenset(re.sub(r"[^a-z0-9]+", " ", title.lower()).split()) + + +def title_jaccard(a_title, b_title): + """Token Jaccard overlap of two titles in [0,1]; 0 if either is empty.""" + A, B = norm_title_tokens(a_title), norm_title_tokens(b_title) + if not A or not B: + return 0.0 + return len(A & B) / len(A | B) + + +TITLE_DUP_THRESHOLD = 0.85 # >= this token-Jaccard flags a probable duplicate title + + def cmd_check_dup(led, a) -> int: + """Dedup a candidate against the ledger. Beyond the exact-slug + never_repeat check + (which depended on three separate LLM calls minting IDENTICAL slugs — the audit's + gap), it now ALSO catches the SAME paper entering under a different slug via its + arXiv id (from --source-url) and a fuzzy title match (--title). Cheap and + deterministic — no external fuzzy lib.""" hits = [] t = find(led["techniques"], "slug", a.slug) if t is not None: - hits.append({"where": "techniques", "status": t.get("status"), - "first_seen": t.get("first_seen")}) + hits.append({"where": "techniques", "match": "slug", "slug": a.slug, + "status": t.get("status"), "first_seen": t.get("first_seen")}) if a.slug in led["never_repeat"]: - hits.append({"where": "never_repeat"}) + hits.append({"where": "never_repeat", "match": "slug", "slug": a.slug}) + + cand_arxiv = arxiv_id(getattr(a, "source_url", None)) + cand_title = getattr(a, "title", None) + if cand_arxiv or cand_title: + for other in led["techniques"]: + if other.get("slug") == a.slug: + continue # already reported above + if cand_arxiv and arxiv_id(other.get("source_url")) == cand_arxiv: + hits.append({"where": "techniques", "match": "arxiv_id", + "arxiv_id": cand_arxiv, "slug": other.get("slug")}) + elif cand_title: + j = title_jaccard(cand_title, other.get("title")) + if j >= TITLE_DUP_THRESHOLD: + hits.append({"where": "techniques", "match": "fuzzy_title", + "jaccard": round(j, 3), "slug": other.get("slug"), + "title": other.get("title")}) emit({"slug": a.slug, "verdict": "DUPLICATE" if hits else "NEW", "hits": hits}) return 1 if hits else 0 @@ -795,6 +848,10 @@ def main(argv=None) -> int: p = sub.add_parser("check-dup", parents=[common]) p.add_argument("slug") + p.add_argument("--source-url", default=None, + help="also dedup by arXiv id extracted from this URL") + p.add_argument("--title", default=None, + help="also dedup by fuzzy title match against existing techniques") p = sub.add_parser("query", parents=[common]) p.add_argument("--status", default=None) diff --git a/research/tests/test_ledger.py b/research/tests/test_ledger.py index 24d5346..fa3edfb 100644 --- a/research/tests/test_ledger.py +++ b/research/tests/test_ledger.py @@ -372,6 +372,38 @@ def test_the_run_type_lint_catches_the_original_bug(tmp_path): assert all(t not in ledger.RUN_TYPES for t in found.values()) # ...and flagged invalid +def test_arxiv_id_extraction(): + assert ledger.arxiv_id("http://arxiv.org/abs/2606.14187") == "2606.14187" + assert ledger.arxiv_id("https://arxiv.org/pdf/2606.14187v2") == "2606.14187" + assert ledger.arxiv_id("https://blog.google/some-post") is None + assert ledger.arxiv_id(None) is None + + +def test_title_jaccard_fuzzy(): + assert ledger.title_jaccard("Zeta: Dual Whitening!", "zeta dual whitening") == 1.0 + assert ledger.title_jaccard("A method for X", "A totally different thing") < 0.85 + assert ledger.title_jaccard("anything", "") == 0.0 + + +def test_check_dup_catches_arxiv_and_fuzzy_title(ledger_path): + """The audit gap: exact-slug dedup let the SAME paper re-enter under a different slug. + check-dup now also catches it by arXiv id and by fuzzy title.""" + ledger.main(["add-technique", "--slug", "zeta", "--title", "Zeta: Dual Whitening for Matrix Opt", + "--source-url", "http://arxiv.org/abs/2606.14187", "--ledger", str(ledger_path)]) + # same paper, DIFFERENT slug, via arxiv id -> DUPLICATE + assert run(ledger_path, "check-dup", "zeta-rebrand", + "--source-url", "https://arxiv.org/pdf/2606.14187v3") == 1 + # same paper, different slug, via fuzzy title -> DUPLICATE + assert run(ledger_path, "check-dup", "zeta-reworded", + "--title", "zeta dual whitening for matrix opt") == 1 + # genuinely new -> NEW (no false positive) + assert run(ledger_path, "check-dup", "new-thing", + "--source-url", "http://arxiv.org/abs/2607.00001", + "--title", "An unrelated widget method") == 0 + # exact-slug path still works with no url/title + assert run(ledger_path, "check-dup", "zeta") == 1 + + def test_split_verdict_vocabulary(ledger_path): """2026-07-22: 'directional' was split into null / promising so a genuine negative result and a big-but-capped effect stop sharing one word. Both must be accepted, and From dd9661a9b0a24c2fee0c5ffa790987ae617cb068 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 13:35:33 +0100 Subject: [PATCH 17/35] =?UTF-8?q?c5=5Fvalidate.py=20=E2=80=94=20machine-ch?= =?UTF-8?q?eckable=20=C2=A7C5=20pre-launch=20lint=20(item=2011a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit found c5_evidence.json schema-free and drifting: the flagship win has NO c5 file, one 3-seed cohort has none, and the key naming varies so nothing could machine-check §C5 compliance (batch 3 Q3). This turns "smoke: pass" from prose into a checkable gate — validate_c5() reports which §C5 items (smoke/budget/probe/eta/ resume/sentinel/guards) are missing, so a launcher (or the adopted-run protocol for manual launches) can REFUSE to spawn on incomplete evidence. Accepts both historical namings (numbered c5_0_smoke and flat smoke) — the point is that the evidence exists in a machine-readable FIELD, not the key name; an item buried in free prose reads as missing, which is the correct signal. Verified on disk: both live HybridSSM c5 files PASS 7/7; the drifted scaling-persistence outlier FAILs informatively. CLI (exit 0/1/2) + importable validate_c5(); 6 tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- research/c5_validate.py | 95 ++++++++++++++++++++++++++++++ research/tests/test_c5_validate.py | 59 +++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 research/c5_validate.py create mode 100644 research/tests/test_c5_validate.py diff --git a/research/c5_validate.py b/research/c5_validate.py new file mode 100644 index 0000000..5c320cc --- /dev/null +++ b/research/c5_validate.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""c5_validate.py — machine-checkable §C5 bounded-auto-run evidence (upgrade-plan #11). + +The audit found c5_evidence.json is schema-free and its keys drift per run: the +flagship win has NO c5 file, one full 3-seed cohort has none, and the key naming +varies so nothing could machine-check §C5 compliance. This turns "smoke: pass" from +prose into a CHECKABLE contract: a launch (or the adopted-run protocol for a manual +launch) calls validate_c5() and REFUSES to spawn if a required §C5 item is missing. + +It accepts the several historical namings for each item (both the numbered +`c5_0_smoke` style and the flat `smoke` style), because the point is to check that +the EVIDENCE exists in a machine-readable field — not to bikeshed the key name. An +item recorded only in free prose under an unrecognized key reads as MISSING, which is +the correct signal: §C5 evidence belongs in a checkable field. + +CLI: python3 research/c5_validate.py + exit 0 = all required items present; exit 1 = missing items (printed); exit 2 = bad file. +Import: from c5_validate import validate_c5 ; report = validate_c5(evidence_dict) +""" +from __future__ import annotations +import json +import sys + +# Each §C5 item -> the set of accepted keys that satisfy it (any one present = satisfied). +# §C5.1 (concurrency) is a launch-instant check, legitimately transient, so it is NOT a +# required stored field; the rest (0,2,3,4,5,6,7) must be recorded before launch. +C5_ITEMS = { + "c5.0_smoke": ("smoke", "c5_0_smoke"), + "c5.2_budget": ("budget", "c5_2_budget"), + "c5.3_probe": ("probe", "c5_3_probe"), + "c5.4_eta": ("eta_hours", "c5_4_eta_hours"), + "c5.5_resume": ("c5_5_resume", "resume_roundtrip", "resume"), + "c5.6_sentinel": ("c5_6_sentinel", "sentinel"), + "c5.7_guards": ("c5_7_guards", "guards"), +} +# Identity fields every c5 file must carry regardless of style. +REQUIRED_IDENTITY = ("run_id", "objective", "framework") + + +def _present(evidence, keys): + """A key satisfies an item if present AND its value is not None/empty.""" + for k in keys: + if k in evidence and evidence[k] not in (None, "", {}, []): + return k + return None + + +def validate_c5(evidence: dict) -> dict: + """Return {ok, missing_items, missing_identity, satisfied} for a c5_evidence dict. + `ok` is True only when every required §C5 item AND every identity field is present.""" + if not isinstance(evidence, dict): + return {"ok": False, "missing_identity": list(REQUIRED_IDENTITY), + "missing_items": list(C5_ITEMS), "satisfied": {}, + "error": "evidence is not a JSON object"} + satisfied, missing_items = {}, [] + for item, keys in C5_ITEMS.items(): + hit = _present(evidence, keys) + if hit: + satisfied[item] = hit + else: + missing_items.append(item) + missing_identity = [k for k in REQUIRED_IDENTITY + if evidence.get(k) in (None, "", {}, [])] + return {"ok": not missing_items and not missing_identity, + "missing_items": missing_items, "missing_identity": missing_identity, + "satisfied": satisfied} + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + if not argv: + print("usage: c5_validate.py ", file=sys.stderr) + return 2 + try: + evidence = json.loads(open(argv[0]).read()) + except (OSError, json.JSONDecodeError) as e: + print(f"[c5] cannot read {argv[0]}: {e}", file=sys.stderr) + return 2 + r = validate_c5(evidence) + if r["ok"]: + print(f"[c5] PASS — all §C5 items present ({len(r['satisfied'])}/7): " + f"{', '.join(sorted(r['satisfied']))}") + return 0 + if r.get("error"): + print(f"[c5] FAIL — {r['error']}") + if r["missing_identity"]: + print(f"[c5] FAIL — missing identity fields: {', '.join(r['missing_identity'])}") + if r["missing_items"]: + print(f"[c5] FAIL — missing §C5 evidence: {', '.join(r['missing_items'])}") + print(" (record each in a machine-readable field before launch — §C5)") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/tests/test_c5_validate.py b/research/tests/test_c5_validate.py new file mode 100644 index 0000000..ce35217 --- /dev/null +++ b/research/tests/test_c5_validate.py @@ -0,0 +1,59 @@ +"""Tests for c5_validate — the machine-checkable §C5 pre-launch lint (upgrade-plan #11).""" +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +import c5_validate as c5 # noqa: E402 + + +def _complete(**over): + e = {"run_id": "2026-07-23_m_x", "objective": "pretrain-ablation", "framework": "jax", + "smoke": "pass", "budget": {"tokens": 1}, "probe": {"tokens_per_sec": 1}, + "eta_hours": 4.0, "resume_roundtrip": "pass", "sentinel": "armed", + "guards": "verified"} + e.update(over) + return e + + +def test_complete_evidence_passes(): + r = c5.validate_c5(_complete()) + assert r["ok"] is True and r["missing_items"] == [] and r["missing_identity"] == [] + + +def test_accepts_both_key_stylings(): + # numbered c5_x style instead of flat style — must also satisfy + e = {"run_id": "r", "objective": "finetune", "framework": "pytorch", + "c5_0_smoke": {"result": "pass"}, "c5_2_budget": {"tokens": 1}, + "c5_3_probe": {}, "c5_4_eta_hours": 3, "c5_5_resume": "pass", + "c5_6_sentinel": "armed", "c5_7_guards": "verified"} + # c5_3_probe is empty {} -> counts as MISSING (empty value), everything else present + r = c5.validate_c5(e) + assert r["missing_items"] == ["c5.3_probe"] + e["c5_3_probe"] = {"tokens_per_sec": 1} + assert c5.validate_c5(e)["ok"] is True + + +def test_missing_items_are_reported(): + r = c5.validate_c5(_complete(guards=None, sentinel="")) + assert set(r["missing_items"]) == {"c5.7_guards", "c5.6_sentinel"} + assert r["ok"] is False + + +def test_missing_identity_fails(): + e = _complete(); del e["framework"] + r = c5.validate_c5(e) + assert "framework" in r["missing_identity"] and r["ok"] is False + + +def test_non_dict_and_empty(): + assert c5.validate_c5("nope")["ok"] is False + assert c5.validate_c5({})["ok"] is False + + +def test_cli_exit_codes(tmp_path): + good = tmp_path / "good.json"; good.write_text(__import__("json").dumps(_complete())) + bad = tmp_path / "bad.json"; bad.write_text(__import__("json").dumps(_complete(guards=None))) + assert c5.main([str(good)]) == 0 + assert c5.main([str(bad)]) == 1 + assert c5.main([str(tmp_path / "absent.json")]) == 2 From 894cb278d0215f9b96c91b4c487cb78405707c60 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 13:36:26 +0100 Subject: [PATCH 18/35] Plan log: Tier 2 dedup + c5 lint + CLAUDE.md fix (2026-07-23) Co-Authored-By: Claude Opus 4.8 (1M context) --- research/LOOP_UPGRADE_PLAN_2026-07-22.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/research/LOOP_UPGRADE_PLAN_2026-07-22.md b/research/LOOP_UPGRADE_PLAN_2026-07-22.md index 7c4c95f..25f1e54 100644 --- a/research/LOOP_UPGRADE_PLAN_2026-07-22.md +++ b/research/LOOP_UPGRADE_PLAN_2026-07-22.md @@ -79,3 +79,9 @@ GOVERNANCE TODO flagged, not decided unilaterally: Remaining Tier 2/3 (each needs a bit of your steer): #10 adopted-run protocol, #11 c5 schema+lint / CLAUDE.md verdict-timing, #13 dedup, #14 calibration wiring, #15 doc reframe + meta-scrub, #17 shared-box lock, #18 loop_state fsync, #19 kdump [HUMAN]. GPU items #5/#9 still await a rung gap + go. + +## Execution log — 2026-07-23 (Tier 2 pass) +- #13 dedup hardened: arXiv-id + fuzzy-title (dcf01e0). Verified vs live ledger; 36 tests. +- #11a c5_validate.py: machine-checkable §C5 pre-launch lint (dd9661a). Both live HybridSSM c5 files PASS 7/7. +- #11b CLAUDE.md verdict-timing contradiction corrected (entry-at-launch; verdict written at finish). [local file] +Remaining Tier 2: #10 adopted-run protocol [.claude/, bigger], #14 calibration wiring [bigger], #15 doc reframe + meta-scrub [local docs]. From 5cebed04f0cca0092b702d31ea3fa46ed09a2a7d Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 20:04:23 +0100 Subject: [PATCH 19/35] Recover the dead arch ladder + close the false-green loop defects The hand-launched HybridSSM arch ladder died: sentinel thermally killed 2 of the 85M cells (2026-07-23 14:16/14:21Z) and the box then hard-locked at 15:24 BST. The safety system worked; the driver's cool-down gate and a batch of "done" loop-hardening items did not. Recon (2 workflows, adversarially cross-checked) found the real defects; this lands the fixes, each diff reviewed. Ladder / safety: - run_arch_ladder.sh cool_down: 3-min sustained-cool dwell, DEFER on the bounded fall-through (engages the hot-spell backoff), honour the return value at the call site, COOL_C 70->58 (= sentinel KILL 90 - measured +31C load transient - 1). - score_arch_ladder.py resolved checkpoints under the wrong dir -> scored 0/7 cells and exited 0 (a silent no-op). Fixed the path, made it fail-loud, added a pure-path smoke check. - sentinel.py writes the kill marker BEFORE the SIGTERM->SIGKILL grace loop, so a caller that reaps the watcher on trainer exit no longer loses it (+ ordering test). - train_hybrid.py checkpoints/restores the PRNG key so a resumed cell continues its exact data-window stream (round-trip verified bit-exact). Loop integrity: - eval_completeness: fix a disallowed-sole-signal floor BYPASS (fired only at len(present)==1, so an n=1 val-PPL headline + any 2nd item reached promising/win); now floors to inconclusive whenever no admissible signal remains. Verdict cap aligned. - loop_state.py: parent-dir fsync + advisory lock + mode preservation (ledger.py parity) + register() setter for the flat in-flight fields. - ledger.py: unknown-run-key hygiene warning; cross-lane provenance keys allowlisted. ledger.json: 17 dangling detail_md nulled, 33 stray eval keys moved into metrics{}. - adopt_run.py (adopt + reconcile) + calibration_pairs.py (read-only join, n=0 today), both with tests. Standalone for now; wiring into their callers is the follow-up. - Docs reframed to "rigor factory / propose-only GPU"; S8 eval correctly called a bounded preflight-gated GPU read, not a training launch. Reconciled the dead run through the sanctioned CLI only: arch-ladder + orphan s1 runs -> crashed; technique hybrid-attention-rethink queued->running (it was wrongly surfaced as a fresh next-best launch); loop_state in-flight pointer cleared; sentinel_kill marker archived. sentinel liveness now exits 0. Full suite 537 passed / 1 skipped; ledger fsck clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../train_hybrid.py | 19 +- .../run_arch_ladder.sh | 66 +- .../score_arch_ladder.py | 110 +++- README.md | 9 +- research/LOOP_UPGRADE_PLAN_2026-07-22.md | 55 ++ research/adopt_run.py | 591 ++++++++++++++++++ research/calibration_pairs.py | 166 +++++ research/eval_completeness.py | 136 +++- research/ledger/ledger.json | 134 ++-- research/ledger/ledger.py | 164 ++++- research/loop_state.py | 259 +++++++- research/tests/test_adopt_run.py | 474 ++++++++++++++ research/tests/test_calibration_pairs.py | 205 ++++++ research/tests/test_eval_completeness.py | 123 +++- research/tests/test_guards.py | 323 +++++++++- research/tests/test_ledger.py | 164 +++++ research/tests/test_orchestration_chaos.py | 248 ++++++++ research/tests/test_sentinel.py | 55 ++ sentinel.py | 84 ++- 19 files changed, 3216 insertions(+), 169 deletions(-) create mode 100644 research/adopt_run.py create mode 100644 research/calibration_pairs.py create mode 100644 research/tests/test_adopt_run.py create mode 100644 research/tests/test_calibration_pairs.py diff --git a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py index 96c19cf..89c5534 100644 --- a/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py +++ b/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py @@ -107,7 +107,19 @@ def main(): params = serialization.from_bytes(params, blob["params"]) opt_state = serialization.from_bytes(opt_state, blob["opt_state"]) start_step = blob["step"] - print(f"[resume] from {a.resume} at step {start_step}", flush=True) + # Restore the PRNG stream so a resumed run continues the SAME data-window sequence it + # would have had uninterrupted. Without this, `rng` stays PRNGKey(seed) from above and the + # loop's first split at step=start_step reproduces step 0's subkey — the resumed run then + # REPLAYS its own steps 0..start_step-1 data windows across its remaining budget. That is a + # per-arm data-repetition confound the uninterrupted arms don't carry; it bit the 85M rung + # after the 2026-07-23 thermal kills (swa128_nope_85M_s0 was killed at step ~7800/11718). + if "rng" in blob: + rng = jnp.asarray(blob["rng"]) + print(f"[resume] from {a.resume} at step {start_step} (rng stream restored — exact continuation)", flush=True) + else: + print(f"[resume] from {a.resume} at step {start_step} — WARNING: checkpoint predates " + f"rng-checkpointing; data windows for steps 0..{start_step-1} will REPLAY across the " + f"remaining budget (data-repetition confound). Discard and rerun clean to avoid it.", flush=True) @jax.jit def step(params, opt_state, ids, tgt): @@ -118,7 +130,10 @@ def step(params, opt_state, ids, tgt): return params, opt_state, loss, gnorm def save(step_i): - blob = {"params": serialization.to_bytes(params), "opt_state": serialization.to_bytes(opt_state), "step": step_i} + # `rng` is captured at call time (late binding): it holds the post-split stream position at + # the end of step step_i-1, which is exactly what a resume at step_i must restore to be exact. + blob = {"params": serialization.to_bytes(params), "opt_state": serialization.to_bytes(opt_state), + "step": step_i, "rng": np.asarray(rng)} tmp = a.ckpt + ".tmp" with open(tmp, "wb") as f: pickle.dump(blob, f) diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh index f9ddec8..37ec53a 100755 --- a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh @@ -18,7 +18,17 @@ BUILD="$ROOT/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build" DATA="$ROOT/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/tokcache_170034304_300000_seed0_Qwen3-0.6B-Base.pt" LOG="$LDIR/run_arch_ladder.log" PY=python3 -COOL_C="${LADDER_COOL_C:-70}" # don't launch onto a box hotter than this +# Cool-down gate (see cool_down() for the 2026-07-23 post-mortem that set these). +# COOL_C=58 is derived from sentinel.py's own constants, not guessed: TEMP_KILL_C=90 minus +# the +31C idle->load transient measured on 2026-07-23 (66C at the 15:17:47 BST launch -> +# soc 97C 50s later, sentinel_attn1to3_85M_s0.log 14:18:37Z), minus 1C. Launching at <=58C +# is therefore the hottest start that keeps the first-minute transient under the kill line; +# the old 70C sat ABOVE the box's own recently-loaded idle floor (64-69C), so it gated +# nothing. A cold box reads 43-44C here, so 58C is attainable — just not one minute after +# a thermal kill, which is exactly the launch this blocks. +COOL_C="${LADDER_COOL_C:-58}" # don't launch onto a box hotter than this +COOL_DWELL="${LADDER_COOL_DWELL:-6}" # consecutive cool samples required (6 x 30s = 3 min) +COOL_MAX="${LADDER_COOL_MAX:-30}" # bound: 30 x 30s = 15 min, then DEFER (never launch) MAXPASS="${LADDER_MAXPASS:-100}" SEQ=2048; BATCH=4; LR=3e-3; WARMUP=200 @@ -38,17 +48,52 @@ hottest_c () { echo "$max" } -# Bounded, fail-open cool-down: wait up to ~30 min to drop below COOL_C. Unreadable => proceed. +# Cool-down gate. Returns 0 = safe to launch, 1 = still hot -> DEFER this cell. +# +# 2026-07-23 post-mortem (the 15:16-15:24 BST thrash that ended in a hard lock). The old +# gate accepted a SINGLE sample below 70C, and when its 30-min bound expired it LAUNCHED +# ANYWAY; its return value was discarded at the call site, so nothing could have stopped +# it either way. What that produced, from run_arch_ladder.log + the sentinel logs (sentinel +# stamps UTC, the driver stamps BST = UTC+1): +# 14:16:46Z sentinel thermal-kills swa128_nope_85M_s0 at gpu 85C / soc 92C +# 15:17:47 driver relaunches attn1to3_85M_s0 on ONE 66C sample, 60s after that kill +# 14:18:37Z that arm is already at soc 97C (a +31C idle->load rise in 50s) +# 14:21:37Z sentinel thermal-kills it too +# 15:22:38 driver relaunches fullattn_85M_s0 on ONE 69C sample, 61s after THAT kill +# 15:24:38 box hard-locks (no trace: crashkernel=0M, see the GB10 lockup memory note) +# Three launches, two thermal kills and one hard lock inside eight minutes. +# +# Two structural fixes: +# (a) DWELL — require COOL_DWELL consecutive samples strictly under COOL_C. One cool +# sample a minute after a 92C kill is a sensor dip, not a cool box: this box sheds to +# ~66C in ~60s and then PLATEAUS there, so a single-sample 70C gate had ~1C of +# headroom and waved through a box still full of heat. +# (b) DEFER, never launch-anyway — the bound now returns 1 so the caller drops the cell to +# the next pass and the hot-spell backoff at the end of the pass loop engages, instead +# of the driver walking down the cell list igniting one arm after another. +# Unreadable temp still fails OPEN (proceed): fabricating heat is worse than not measuring +# it, and sentinel.py takes the same stance (gpu_thermal/hottest_soc_c return None rather +# than a fake high reading). cool_down () { - local tag=$1 h - for _ in $(seq 1 60); do + local tag=$1 h streak=0 + for _ in $(seq 1 "$COOL_MAX"); do h=$(hottest_c) [ -z "$h" ] && { echo "[$(date '+%T')] [cooldown] $tag: temp unreadable, proceeding"; return 0; } - [ "$h" -lt "$COOL_C" ] && { echo "[$(date '+%T')] [cooldown] $tag: ${h}C < ${COOL_C}C, launching"; return 0; } - echo "[$(date '+%T')] [cooldown] $tag: ${h}C >= ${COOL_C}C, waiting 30s" + if [ "$h" -lt "$COOL_C" ]; then + streak=$((streak+1)) + if [ "$streak" -ge "$COOL_DWELL" ]; then + echo "[$(date '+%T')] [cooldown] $tag: ${h}C < ${COOL_C}C for $streak consecutive samples, launching" + return 0 + fi + echo "[$(date '+%T')] [cooldown] $tag: ${h}C < ${COOL_C}C, dwell $streak/${COOL_DWELL}" + else + echo "[$(date '+%T')] [cooldown] $tag: ${h}C >= ${COOL_C}C, waiting 30s (dwell reset from $streak)" + streak=0 + fi sleep 30 done - echo "[$(date '+%T')] [cooldown] $tag: still hot after 30min — launching anyway (bounded)" + echo "[$(date '+%T')] [cooldown] $tag: no ${COOL_DWELL}-sample cool spell in $((COOL_MAX * 30 / 60))min (last ${h}C) — DEFERRING" + return 1 } # Is a REAL trainer alive? A bare `pgrep -f 'train_*.py'` is not good enough: it matches any @@ -83,7 +128,12 @@ run_cell () { if trainer_alive; then echo "[$(date '+%T')] [wait] $id: another trainer is alive, deferring this pass"; return 1 fi - cool_down "$id" + # HONOUR the gate. Until 2026-07-23 this call discarded cool_down's return value, so even + # a gate that wanted to refuse could not: the driver launched regardless. A deferral must + # cost this cell the pass, so the hot-spell backoff below sees "failed, no progress". + if ! cool_down "$id"; then + echo "[$(date '+%T')] [defer] $id: no sustained cool window — deferring this pass"; return 1 + fi # unified pool headroom (shared CPU+GPU memory): wait for >= 60 GB available for _ in $(seq 1 90); do a=$(free -g | awk '/Mem:/{print $7}'); [ "${a:-0}" -ge 60 ] && break; sleep 10; done diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py index 0e10add..ccd0b9e 100644 --- a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/score_arch_ladder.py @@ -17,18 +17,31 @@ emergence-speed curve, and defers a BPB number to eval-harness. Modes: - --smoke : CPU-only structural self-test of the pure aggregation/curve math (no GPU, - no model load) — safe to run beside a live trainer. + --smoke : CPU-only structural self-test — the pure aggregation/curve math PLUS a pure + path check that every .done cell resolves to a checkpoint that exists (no + GPU, no model load, nothing unpickled) — safe to run beside a live trainer. (default): GPU — score every .done cell in cells.json. §C4.5: run ONLY when no trainer is live (the driver calls this at a rung gap / at ladder end). +2026-07-23 fix — the scorer was a SILENT NO-OP, the exact failure mode this file was +written to eliminate. It looked for checkpoint_.pkl under LDIR, but the driver +trains with `cd "$BUILD"` (run_arch_ladder.sh) so train_hybrid.py writes every checkpoint +into BUILD. Every cell therefore failed the `ckpt.exists()` test, was skipped, and the +script wrote "cells_scored: 0" and exited 0 — a green ladder with no numbers. Three +changes: resolve checkpoints under BUILD; chdir to BUILD so the suite's cwd-relative +paths resolve (load_corpora's fallback reads "model.py", which exists only there); and +FAIL LOUD (non-zero) when .done cells exist but nothing was scored, or when a .done cell +has no checkpoint on disk. + GPU-VALIDATION-PENDING: the per-cell scoring path (load_model+ppl) is proven (it scored the pilot), but this orchestrator's end-to-end run has not yet executed on GPU — it will -at the first rung gap. Until then treat the emitted numbers as unproduced. +at the first rung gap. Until then treat the emitted numbers as unproduced. The path fix +above is verified by the --smoke path check, not by a GPU run. """ from __future__ import annotations import json import math +import os import pathlib import sys @@ -42,6 +55,22 @@ def load_cells(): return json.loads((LDIR / "cells.json").read_text()) +def cell_ckpt(cell_id: str) -> pathlib.Path: + """Checkpoint path for one cell — under BUILD, not LDIR. + + run_arch_ladder.sh launches with `cd "$BUILD" && $PY train_hybrid.py ... --ckpt + "checkpoint_${id}.pkl"`, i.e. a BUILD-RELATIVE name, so that is where the trainer + writes it (verified: BUILD holds checkpoint_ssm_base_42M_s0.pkl et al, LDIR holds + none). Resolving under LDIR made every cell fail its exists() test and be skipped. + """ + return BUILD / f"checkpoint_{cell_id}.pkl" + + +def done_cells(cells: dict) -> list: + """Cells carrying the driver's .done marker (written in LDIR by train_hybrid.py).""" + return [c for c in cells["cells"] if (LDIR / f"{c['id']}.done").exists()] + + def emergence_curve(per_arm_rung_ppl: dict) -> dict: """per_arm_rung_ppl[arm] = {tokens: ppl}. Fit ppl vs log10(tokens) per arm (OLS) and report the slope (emergence speed) + the arm-minus-base gap at each rung. @@ -67,7 +96,7 @@ def emergence_curve(per_arm_rung_ppl: dict) -> dict: return curves -def _smoke() -> int: +def _smoke_curve() -> int: # synthetic: ssm improves fastest, swa lags — assert the curve math is sane. fake = {"ssm_base": {42_000_000: 4.7, 85_000_000: 4.2, 150_000_000: 3.9}, "swa128": {48_000_000: 5.7, 96_000_000: 5.3, 170_000_000: 5.0}} @@ -81,21 +110,65 @@ def _smoke() -> int: return 0 +def _smoke_paths() -> int: + """PURE PATH CHECK — no model load, no unpickle, no GPU, no import of the JAX suite. + + Every cell the driver marked .done MUST resolve to a checkpoint that exists; that is + precisely the precondition the GPU path skipped in silence until 2026-07-23, turning + a whole ladder into "cells_scored: 0, exit 0". Also checks the file the suite's + load_corpora() code-corpus FALLBACK opens by a cwd-relative name ("model.py"), since + _score_all now chdirs to BUILD to make that resolve. + """ + cells = load_cells() + done = done_cells(cells) + fails = [] + for c in done: + p = cell_ckpt(c["id"]) + if not p.exists(): + fails.append(f"{c['id']}.done exists but {p} does not") + fallback = BUILD / "model.py" # load_corpora() cwd-relative fallback source + if not fallback.exists(): + fails.append(f"corpus fallback source {fallback} missing (scorer chdirs to BUILD)") + for msg in fails: + print(f"SMOKE FAIL: {msg}") + if fails: + return 1 + print(f"SMOKE PASS: {len(done)}/{len(cells['cells'])} cells .done, every checkpoint " + f"resolves under {BUILD.name}/ (+ corpus fallback present)") + return 0 + + +def _smoke() -> int: + """Both CPU self-tests, both always run so both report; non-zero if EITHER fails.""" + rc_math = _smoke_curve() + rc_paths = _smoke_paths() + return 1 if (rc_math or rc_paths) else 0 + + def _score_all() -> int: """GPU path — score every .done cell. §C4.5: caller guarantees no live trainer.""" sys.path.insert(0, str(BUILD)) + # Score FROM the build dir. eval_suite_jax resolves some paths against the cwd — its + # load_corpora() code-corpus fallback does pathlib.Path("model.py").read_text(), and + # model.py exists only under BUILD — and the driver itself trains with `cd "$BUILD"`, + # so matching that cwd is the honest resolution. (2026-07-23: the driver invokes this + # script by absolute path from wherever it was started, so the cwd was arbitrary.) + os.chdir(BUILD) import importlib esj = importlib.import_module("eval_suite_jax") from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(esj.TOKENIZER_REPO) cells = load_cells() + done = done_cells(cells) per_arm_rung = {} # arm -> {tokens: {corpus: ppl}} scored = [] - for cell in cells["cells"]: - done = LDIR / f"{cell['id']}.done" - ckpt = LDIR / f"checkpoint_{cell['id']}.pkl" - if not done.exists() or not ckpt.exists(): + unresolved = [] # .done but no checkpoint on disk -> loud, never silent + for cell in done: + ckpt = cell_ckpt(cell["id"]) + if not ckpt.exists(): + unresolved.append({"cell": cell["id"], "expected_ckpt": str(ckpt)}) + print(f"[MISSING] {cell['id']}: .done but no checkpoint at {ckpt}", flush=True) continue # patch the suite module's per-arm config, then reuse its proven loader/scorer esj.MIXER, esj.ATTN_EVERY, esj.NOPE = cell["mixer"], cell["attn_every"], cell["nope"] @@ -118,13 +191,28 @@ def _score_all() -> int: out = {"suite_version": "text-lm-v2", "metric": f"val PPL ({corpus}), own tokenizer, n=1/cell", "comparability": "cross-arm PPL only (same tokenizer/corpora/windows); " "cross-study BPB deferred to the consolidated eval-harness", - "cells_scored": len(scored), "rows": scored, + "cells_done": len(done), "cells_scored": len(scored), "rows": scored, + "unresolved_cells": unresolved, "emergence_curve": emergence_curve(per_arm_ppl), "caveat": "n=1 per cell -> DIRECTIONAL (§C17); mixer-type gaps carry the " "LR-not-retuned-per-arm confound (see rung_42M_comparison.md)."} (LDIR / "arch_ladder_scores.json").write_text(json.dumps(out, indent=2) + "\n") - print(f"[done] wrote arch_ladder_scores.json ({len(scored)} cells)") - return 0 + print(f"[done] wrote arch_ladder_scores.json ({len(scored)} of {len(done)} .done cells)") + + # FAIL LOUD. Scoring nothing while trained cells exist is the silent no-op this file + # was written to eliminate; exiting 0 on it let run_arch_ladder.sh print "scoring done" + # over an empty result. The driver's `if $PY ...score_arch_ladder.py` branch turns a + # non-zero return into its "!! SCORING FAILED" line. + rc = 0 + if done and not scored: + print(f"[FAIL] {len(done)} cell(s) carry a .done marker but ZERO were scored — " + f"checkpoint resolution is broken, not the ladder", flush=True) + rc = 1 + if unresolved: + print(f"[FAIL] {len(unresolved)} .done cell(s) have no checkpoint on disk: " + f"{[u['cell'] for u in unresolved]}", flush=True) + rc = 1 + return rc def main() -> int: diff --git a/README.md b/README.md index 87e9338..435187d 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,11 @@ is held to multi-seed CIs, iso-FLOP matching, and a held-out noise floor. | [`SmolLM2-134(base)/`](SmolLM2-134(base)/) | Single-file PyTorch reproduction of [SmolLM2-135M](https://huggingface.co/HuggingFaceTB/SmolLM2-135M) (134,515,008 params), verified **bit-exact** vs the official weights (`max \|Δlogits\| = 0.0`). Includes from-scratch training, continued pretraining on TinyStories, multi-axis parity diagnostics, in-domain vs OOD eval, and an `lm-evaluation-harness` wrapper. | | [`Qwen3-0.6B/`](Qwen3-0.6B/) | Single-file PyTorch reproduction of [Qwen3-0.6B-Base](https://huggingface.co/Qwen/Qwen3-0.6B-Base), verified **bit-exact** (`max \|Δlogits\| = 0.0`), then a **full research lifecycle** on top (architecture / optimizer / data / post-training, below). See [its README](Qwen3-0.6B/README.md). | -> The repo is driven by a set of **local-only** Claude Code skills (an autonomous -> ML-research loop) and a FastAPI agent-harness showcase; those are kept local -> and are **not** committed. What is committed is the model code, the verify -> gates, and the experiment results below. +> The repo is driven by a set of **local-only** Claude Code skills (an ML-research +> loop whose scanning, briefing, data-prep and scoring stages run autonomously, +> while a human triggers every GPU run) and a FastAPI agent-harness showcase; +> those are kept local and are **not** committed. What is committed is the model +> code, the verify gates, and the experiment results below. ## Research lifecycle — Qwen3-0.6B diff --git a/research/LOOP_UPGRADE_PLAN_2026-07-22.md b/research/LOOP_UPGRADE_PLAN_2026-07-22.md index 25f1e54..6620352 100644 --- a/research/LOOP_UPGRADE_PLAN_2026-07-22.md +++ b/research/LOOP_UPGRADE_PLAN_2026-07-22.md @@ -85,3 +85,58 @@ Remaining Tier 2/3 (each needs a bit of your steer): #10 adopted-run protocol, # - #11a c5_validate.py: machine-checkable §C5 pre-launch lint (dd9661a). Both live HybridSSM c5 files PASS 7/7. - #11b CLAUDE.md verdict-timing contradiction corrected (entry-at-launch; verdict written at finish). [local file] Remaining Tier 2: #10 adopted-run protocol [.claude/, bigger], #14 calibration wiring [bigger], #15 doc reframe + meta-scrub [local docs]. + +## Execution log — 2026-07-23 (incident recovery + Tier 1/2/3 sweep) + +INCIDENT: the hand-launched arch ladder (driver pid 1808201) is DEAD. Timeline reconstructed from +logs + `last -x`: sentinel's thermal killer fired twice (14:16:46Z gpu85/soc92, 14:21:37Z gpu86/soc91), +then the **box hard-locked at 15:24:38 BST** (boot -1 ends there, no panic/OOM trace — the documented +kdump-less lockup). 7/15 cells done (42M rung complete + ssm_base_85M + swa128_85M). Safety system worked; +the ladder DRIVER's gate did not. Root cause was NOT a metric mismatch (driver + sentinel agree ~69C) — +it was a structural cool-down gate: 1-sample accept, 30-min fall-through that LAUNCHED anyway, discarded +return value, and COOL_C=70 sitting at the box's own idle floor. Verified via a 2-workflow recon (5 read-only +audits + adversarial cross-check) then an 8-lane fix workflow (each diff adversarially reviewed). + +DONE (committed): +- **Tier 3 cooldown fix** (#16-adjacent): run_arch_ladder.sh cool_down now requires 6 sustained sub-COOL_C + samples (3 min dwell), DEFERS on the bounded fall-through (return 1 → engages the hot-spell backoff), and + honours its return value at the call site. COOL_C 70→58 (= sentinel KILL 90 − measured +31C idle→load + transient − 1). Kept the reviewer's 63–65 suggestion OUT: 65+31=96 > 90 would re-cross the kill line. +- **Tier 1 #4 scorer FIX**: score_arch_ladder.py resolved checkpoints under LDIR but the trainer writes them + under BUILD → it scored 0/7 cells and exited 0 (the exact silent no-op it was meant to kill). Fixed the + path, made it fail-loud on 0-scored-with-.done, added a pure-path --smoke check. (GPU end-to-end still pending.) +- **sentinel marker race**: kill marker now written BEFORE the SIGTERM→SIGKILL grace loop (only 1 of 2 kills + left a marker before). + regression test pinning the ordering. +- **Tier 1 #8 verdict vocab**: eval_completeness.gate_verdict emits null/promising, never `directional`. + Fixed a floor BYPASS (disallowed-sole-signal only fired at len(present)==1, so valppl-n1 + any 2nd item — + even a figure — reached promising/win); now floors to inconclusive whenever no admissible signal remains. +- **Tier 1 #18 loop_state durability**: parent-dir fsync + advisory lock + mode preservation to match ledger.py; + new register() setter for the flat in-flight fields. +- **Tier 2 #12 ledger integrity**: 17 dangling detail_md nulled, 33 stray eval keys migrated into metrics{}, + unknown-run-key hygiene warning added; cross-lane provenance keys (c5_lint, adopted, evidence_path, + reconciled) added to RUN_ADDITIVE_KEYS. +- **Tier 2 #10 adopted-run protocol**: research/adopt_run.py (adopt + reconcile, dry-run by default) + tests. + Standalone tool for now — wiring into ablation-runner/liveness-cron is the follow-up. +- **Tier 2 #14 capture half**: research/calibration_pairs.py (read-only join, honestly prints n=0 today). No + ECE computed — 0 realised pairs. +- **Tier 3 #16 safety tests**: trainer_alive() argv[0] guard + recovery-chain (exit-4 routing, flock) tested. +- **Tier 1 #15 doc reframe**: "autonomous nightly loop" → "rigor factory / propose-only GPU" across CLAUDE.md, + AGENTS.md, README.md + strategy-doc meta-commentary scrub. (Most are gitignored; on-disk only.) +- **RNG-checkpoint fix** (user decision): train_hybrid.py now saves/restores the PRNG key so a resumed cell + continues its exact data-window stream (round-trip verified bit-exact). Discarded the swa128_nope_85M ckpt + (renamed .discarded_rng_confound_20260723) so it reruns clean — per the user's "fix RNG then rerun" choice. +- **RECONCILED the dead run** (sanctioned CLI only): arch-ladder + orphan s1-pretrain runs → `crashed`; + technique hybrid-attention-rethink queued→`running` (E-lane wrongly made it a fresh next-best launch); + loop_state in-flight pointer cleared via register(); stale sentinel_kill marker archived. `sentinel liveness` + now exits 0; next-best no longer surfaces an in-flight technique. Full suite 537 passed / 1 skipped; fsck clean. + +USER DECISIONS (GPU work — propose-only, human-triggered; NOT launched this session): +- Ladder: restart as soon as fixes land (rely on the dwell gate, no overnight window). +- Resume: fix RNG checkpointing then RERUN swa128_nope_85M clean (done: code + ckpt discarded). +- NorMuon-at-scale (#9): INSERT NOW, before the 85M rung resumes (the ladder is stopped, cheapest switch point). + → SEQUENCING TENSION between "restart ladder ASAP" and "NorMuon first": resolved as NorMuon-at-scale is next + in the GPU queue, ladder stays live/restart-ready and resumes right after. Both await the human GPU trigger. + +STILL OPEN: #5 per-arm LR probe [GPU], #9 NorMuon-at-scale launch prep [GPU/human], #17 shared-box lock, +#19 kdump/panic GRUB fix [HUMAN sudo — the box hard-locked AGAIN today, still undiagnosable], recovery-cron +paste [HUMAN], wiring adopt_run/calibration_pairs into their callers. diff --git a/research/adopt_run.py b/research/adopt_run.py new file mode 100644 index 0000000..520eb5a --- /dev/null +++ b/research/adopt_run.py @@ -0,0 +1,591 @@ +#!/usr/bin/env python3 +"""research/adopt_run.py — the ADOPTED-RUN protocol (upgrade-plan #10). + +The audit's uncomfortable finding: the out-of-loop launch is the DOMINANT mode on +this box (~24 hand-driven runs vs ~2 through /ablation-runner). The live arch-ladder +is the pattern in full — a hand-written `c5_evidence.json`, a hand-added ledger +entry, a raw `setsid nohup ... run_arch_ladder.sh`, and `loop_state.json` mutated by +an inline python snippet. The contracts had NO provision for any of that, so every +such launch was contract-VIOLATING rather than contract-GOVERNED. Pretending it does +not happen is how the record rots; this module makes the path legal and mechanically +complete instead. + +Two verbs, both DRY-RUN BY DEFAULT (nothing mutates without `--apply`): + + adopt + Make a manual launch indistinguishable from a sanctioned one: + (a) §C5 lint — validate /c5_evidence.json by importing + c5_validate.validate_c5(); a lint failure REFUSES the adoption. No + evidence, no adoption: that is the whole point of the protocol. + (b) ledger — create the run entry through research/ledger/ledger.py's + CLI (never by hand). Idempotent: an existing entry is left untouched. + (c) digest — write a stub under research/digests/ so §C9's "a digest is + always written" cannot be silently skipped by an out-of-loop launch. + (d) loop_state— register in_flight_run / train_pid / ckpt_path / resume_cmd + through loop_state.register() (the sanctioned setter), so the recovery + chain (boot_resume.sh, liveness cron) can re-adopt the trainer. + + reconcile + The other half, and the more valuable one: an in-flight run whose train_pid is + no longer alive. It (i) reports the stale pointer and (ii) with `--apply` + transitions the ledger entry to a terminal STATUS and clears the loop_state + pointer. It NEVER invents a verdict — dying is not a result, and the terminal + status itself is read off disk (a `verdict.json` in the artifacts dir means the + run finished -> `done`; its absence means it did not -> `crashed`). + +Exit codes: + 0 success / nothing to do (clean or alive) + 1 refused (§C5 lint failure, unreadable evidence, identity mismatch, dead pid at + adopt) or a step failed + 2 argparse usage error (bad flags) — every other refusal is 1 + 4 reconcile only, dry run: a STALE in-flight run was detected and needs `--apply` + (mirrors `sentinel.py liveness` exit 4 = run dead, so a cron can key on it) + +Stdlib-only, pure CPU, no network, no GPU, never launches or resumes a trainer. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +import sys +from datetime import date, datetime +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +LEDGER_PY = HERE / "ledger" / "ledger.py" +DEFAULT_LEDGER = HERE / "ledger" / "ledger.json" +DEFAULT_STATE = HERE / "loop_state.json" +DEFAULT_DIGESTS = HERE / "digests" +PROTOCOL_VERSION = "adopt-run/v1" + +# A launch is not a result: the ledger STATUS moves, the verdict never does. +TERMINAL_IF_FINISHED = "done" # artifacts carry a verdict.json -> the run finished +TERMINAL_IF_DIED = "crashed" # no verdict.json and the pid is gone -> it died +IN_FLIGHT_STATUSES = ("launched", "running", "resumed") + +# §C13 objective -> §C8 run type. A run's objective is the thing the c5 evidence +# records; `type` is the ledger's own vocabulary, so map rather than guess. +OBJECTIVE_TO_RUN_TYPE = {"pretrain-ablation": "ablation", "finetune": "finetune"} + + +def _load_module(name: str, path: Path): + """Import a sibling research/ module by ABSOLUTE PATH (cwd-independent, so the + headless loop and a cron can call this from anywhere). Registered in + sys.modules under its real name so a caller that already did `import + loop_state` shares ONE module object — two copies would mean two lock + registries flocking the same file against each other.""" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +c5_validate = _load_module("c5_validate", HERE / "c5_validate.py") +loop_state = _load_module("loop_state", HERE / "loop_state.py") + + +def _ledger_vocabulary(): + """RUN_TYPES/RUN_STATUS straight from ledger.py (§C8 is the authority), loaded + by path under a private name and deliberately NOT registered in sys.modules — + same discipline as research/eval_completeness.py, so it cannot shadow the plain + `import ledger` used by the ledger's own tests.""" + spec = importlib.util.spec_from_file_location("_adopt_ledger_vocab", LEDGER_PY) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # ledger.py's CLI is under __main__: no side effects + return frozenset(mod.RUN_TYPES), frozenset(mod.RUN_STATUS) + + +RUN_TYPES, RUN_STATUS = _ledger_vocabulary() + + +# ---------------------------------------------------------------- small helpers + +def _now_iso() -> str: + """Local wall-clock stamp, runtime-derived (§C2). loop_state never reads a + clock itself — the caller supplies `ts`, and this is that caller.""" + return datetime.now().astimezone().isoformat(timespec="seconds") + + +def _today() -> str: + return date.today().isoformat() + + +def pid_alive(pid: int) -> bool: + """Alive and not a zombie. Same test as sentinel.pid_alive (a zombie cannot + allocate, so it is dead for our purposes); duplicated rather than imported so + this module stays importable without the repo root on sys.path.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + pass # exists, owned by someone else + except (OSError, TypeError, ValueError): + return False + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().rsplit(")", 1)[1].split()[0] != "Z" + except (OSError, IndexError): + return False + + +def _result_of(value): + """§C5 items are recorded either as a bare token (`"pass"`) or as an object + (`{"result": "pass", "detail": ...}`). Return the token in both stylings.""" + if isinstance(value, dict): + return value.get("result") + return value + + +def _rel(path: Path) -> str: + """Repo-relative path when the target is inside the repo, else absolute — the + ledger stores repo-relative artifact paths.""" + p = Path(path).resolve() + try: + return str(p.relative_to(ROOT)) + except ValueError: + return str(p) + + +def _ledger_cli(ledger_path: Path, args: list) -> tuple: + """Run one research/ledger/ledger.py subcommand. Returns (rc, stdout, stderr). + The ledger CLI is the ONLY sanctioned writer (§C11) — this module never opens + ledger.json itself, for reading or writing.""" + proc = subprocess.run([sys.executable, str(LEDGER_PY)] + args + + ["--ledger", str(ledger_path)], + capture_output=True, text=True) + return proc.returncode, proc.stdout, proc.stderr + + +def ledger_run_entry(ledger_path: Path, run_id: str): + """The run entry for `run_id`, or None. Read through the CLI's `query` verb so + even the READ goes through the sanctioned interface.""" + rc, out, err = _ledger_cli(ledger_path, ["query", "--collection", "runs"]) + if rc != 0: + raise RuntimeError(f"ledger query failed (exit {rc}): {(err or out).strip()}") + try: + runs = json.loads(out).get("runs", []) + except json.JSONDecodeError as e: + raise RuntimeError(f"ledger query returned non-JSON: {e}") + return next((r for r in runs if r.get("run_id") == run_id), None) + + +# ---------------------------------------------------------------- digest stub + +def digest_stub_path(digest_dir: Path, run_id: str, started: str) -> Path: + """`_adopted_.md`, deliberately NOT `.md`: the loop's own S9 + digest owns that name, and an adopted run must never clobber (or race) it. The + stub still globs into research/digests/*.md, which is how digests are found.""" + return Path(digest_dir) / f"{started}_adopted_{run_id}.md" + + +def digest_stub_text(*, run_id, evidence, c5_report, ledger_action, state_action, + train_pid, evidence_path, artifacts_dir, ts) -> str: + """The §C9 acceptance artifact for an out-of-loop launch. It records that the + run EXISTS and is governed — and says, in words, that it holds no result.""" + ev = evidence + satisfied = ", ".join(sorted(c5_report["satisfied"])) + lines = [ + f"# Adopted-run digest stub — {run_id}", + "", + f"> **Out-of-loop launch, adopted {ts} by `{PROTOCOL_VERSION}`.** This run was", + "> started by hand — not by `/research-loop` and not by `/ablation-runner`.", + "> `research/adopt_run.py adopt` validated its §C5 evidence, recorded the ledger", + "> entry through `ledger.py`, and registered it in `loop_state.json` through", + "> `loop_state.register()`. This stub exists so §C9's \"a digest is always", + "> written\" cannot be skipped by an out-of-loop launch.", + "", + "## What was adopted", + f"- **run_id:** `{run_id}`", + f"- **model_dir:** `{ev.get('model_dir')}` · **technique:** `{ev.get('technique_slug')}`", + f"- **objective:** `{ev.get('objective')}` · **framework:** `{ev.get('framework')}`" + f" · **lifecycle_stage:** `{ev.get('lifecycle_stage')}`", + f"- **artifacts:** `{artifacts_dir}`", + f"- **§C5 evidence:** `{evidence_path}` — lint **PASS** ({len(c5_report['satisfied'])}/7: {satisfied})", + f"- **ledger:** {ledger_action} · **loop_state:** {state_action}" + + (f" · **train_pid:** {train_pid}" if train_pid else ""), + "", + "## Results", + "_None._ A launch is not a result. This stub records only that the run exists and", + "is governed; no number, no verdict, and no claim is made here. The verdict is", + "written when the run finishes and `/eval-harness` scores it (§C10), subject to", + "the §C25 per-stage battery (a missing HARD item caps it below `win`).", + "", + "## To close this run", + "1. Score the finished artifacts with `/eval-harness` (the only source of", + " cross-run comparable numbers, §C10) and stamp `suite_version`.", + "2. Write `verdict.json` in the artifacts dir, then record `metrics`/`verdict`", + " on the ledger entry via `ledger.py update-run`.", + "3. If the trainer dies instead, run `python3 research/adopt_run.py reconcile`", + " — it moves the entry to a terminal status and clears the in-flight pointer", + " WITHOUT inventing a verdict.", + "", + ] + return "\n".join(lines) + + +# ---------------------------------------------------------------- verb: adopt + +def adopt(run_dir, *, ledger_path=DEFAULT_LEDGER, state_path=DEFAULT_STATE, + digest_dir=DEFAULT_DIGESTS, train_pid=None, resume_cmd=None, + ckpt_path=None, run_type=None, ts=None, apply=False, + force=False) -> dict: + """Adopt a manual launch. Every precondition is checked BEFORE any mutation, so + a refusal leaves the ledger, the digests and loop_state untouched.""" + ts = ts or _now_iso() + run_dir = Path(run_dir) + report = {"verb": "adopt", "ok": False, "dry_run": not apply, "ts": ts, + "run_dir": str(run_dir), "run_id": None, "c5": None, + "actions": [], "errors": []} + + def err(msg): + report["errors"].append(msg) + return report + + # (a) §C5 lint — no evidence, no adoption. + if not run_dir.is_dir(): + return err(f"run dir not found: {run_dir}") + evidence_file = run_dir / "c5_evidence.json" + try: + evidence = json.loads(evidence_file.read_text()) + except (OSError, json.JSONDecodeError) as e: + return err(f"cannot read {evidence_file}: {e} — §C5 evidence must exist " + "BEFORE launch, and an adopted run is held to the same bar") + c5_report = c5_validate.validate_c5(evidence) + report["c5"] = c5_report + if not c5_report["ok"]: + return err("§C5 lint FAILED — refusing to adopt. missing_items=" + f"{c5_report['missing_items']} missing_identity=" + f"{c5_report['missing_identity']}") + sat = c5_report["satisfied"] + + # Identity. The RUN_ID is the run dir's name by convention (§C4.4); a mismatch + # means the evidence belongs to a different run and the entry would point at + # the wrong artifacts. + run_id = evidence["run_id"] + report["run_id"] = run_id + if run_dir.name != run_id and not force: + return err(f"run dir name {run_dir.name!r} != c5 run_id {run_id!r} — refusing " + "(the ledger entry would point at the wrong artifacts); --force to override") + + objective = evidence.get("objective") + run_type = run_type or OBJECTIVE_TO_RUN_TYPE.get(objective) + if run_type not in RUN_TYPES: + return err(f"cannot determine a valid ledger run type: objective={objective!r} " + f"-> {run_type!r}; pass --run-type (valid: {sorted(RUN_TYPES)})") + + # A RUN_ID starts with the launch date (§C8 YYYY-MM-DD__), so the + # start date is READ off the id rather than defaulted to today — the bug that + # once gave a run started(2026-07-13) AFTER ended(2026-07-12). + started = run_id[:10] + model_dir = evidence.get("model_dir") + if not model_dir and run_dir.parent.name == "experiments": + model_dir = run_dir.parent.parent.name + + # A pid is only worth registering if it is ALIVE. Registering a dead pid as the + # in-flight run is precisely the stale pointer `reconcile` exists to clean up. + if train_pid is not None and not pid_alive(train_pid): + return err(f"train_pid {train_pid} is not alive — refusing to register a dead " + "trainer as in-flight. Adopt without --train-pid to record a finished " + "run, or run `adopt_run.py reconcile` if this pid was the in-flight one") + + # loop_state pre-check: never strand a DIFFERENT in-flight run. + st = loop_state.load(state_path) + current = st.get("in_flight_run") + if train_pid is not None and current not in (None, run_id) and not force: + return err(f"loop_state already points at in_flight_run={current!r} — refusing to " + "overwrite it (that run would be stranded). Reconcile or finish it " + "first; --force to override") + + # ---- everything validated; from here on, mutate (or describe the mutation) ---- + evidence_path = _rel(evidence_file) + artifacts_dir = _rel(run_dir) + smoke_pass = _result_of(evidence[sat["c5.0_smoke"]]) == "pass" + adopted_block = {"protocol": PROTOCOL_VERSION, "adopted_on": ts, + "launched_out_of_loop": True, + "c5_satisfied": sorted(sat), "evidence_path": evidence_path, + "train_pid": train_pid} + + sets = ["--set", f"budget={json.dumps(evidence[sat['c5.2_budget']])}", + "--set", f"probe={json.dumps(evidence[sat['c5.3_probe']])}", + "--set", f"eta_hours={json.dumps(evidence[sat['c5.4_eta']])}", + "--set", f"evidence_path={evidence_path}", + "--set", f"adopted={json.dumps(adopted_block)}"] + for key in ("lifecycle_stage", "confound_check"): + if evidence.get(key) not in (None, "", {}, []): + sets += ["--set", f"{key}={json.dumps(evidence[key])}"] + if train_pid is not None: + sets += ["--set", "status=running"] # a live trainer IS running (§C8 vocabulary) + + argv = ["add-run", "--run-id", run_id, "--type", run_type, + "--started", started] + if model_dir: + argv += ["--model-dir", model_dir] + if evidence.get("technique_slug"): + argv += ["--technique-slug", evidence["technique_slug"]] + if objective: + argv += ["--objective", objective] + if evidence.get("framework"): + argv += ["--framework", evidence["framework"]] + if smoke_pass: + argv += ["--smoke", "pass"] + argv += ["--artifacts-dir", artifacts_dir] + sets + + # (b) ledger entry — idempotent: an existing entry is NEVER clobbered. + try: + existing = ledger_run_entry(ledger_path, run_id) + except RuntimeError as e: + return err(str(e)) + if existing is not None: + ledger_action = f"exists (status={existing.get('status')}) — left untouched" + report["actions"].append({"step": "ledger", "action": "exists", + "status": existing.get("status")}) + elif not apply: + ledger_action = "would add-run" + report["actions"].append({"step": "ledger", "action": "would-add-run", + "argv": argv}) + else: + rc, out, cli_err = _ledger_cli(ledger_path, argv) + if rc != 0: + report["actions"].append({"step": "ledger", "action": "FAILED", + "exit": rc, "stderr": cli_err.strip()}) + return err(f"ledger add-run FAILED (exit {rc}): {(cli_err or out).strip()} " + "— NOT registering loop_state; the pointer must never name a run " + "the ledger does not know") + ledger_action = "add-run" + report["actions"].append({"step": "ledger", "action": "add-run", "argv": argv}) + + # (c) digest stub — §C9 is not skippable by launching out of loop. + stub = digest_stub_path(digest_dir, run_id, started) + state_action = ("register in_flight_run/train_pid" if train_pid is not None + else "not registered (no live train_pid given)") + if stub.exists(): + report["actions"].append({"step": "digest", "action": "exists", + "path": _rel(stub)}) + else: + text = digest_stub_text(run_id=run_id, evidence=evidence, c5_report=c5_report, + ledger_action=ledger_action, state_action=state_action, + train_pid=train_pid, evidence_path=evidence_path, + artifacts_dir=artifacts_dir, ts=ts) + if apply: + stub.parent.mkdir(parents=True, exist_ok=True) + stub.write_text(text) + report["actions"].append({"step": "digest", "action": "wrote", + "path": _rel(stub)}) + else: + report["actions"].append({"step": "digest", "action": "would-write", + "path": _rel(stub), "bytes": len(text)}) + + # (d) loop_state — through register(), the sanctioned setter. Never a direct write. + if train_pid is None: + report["actions"].append({"step": "loop_state", "action": "skipped", + "why": "no --train-pid: nothing is in flight"}) + elif not apply: + report["actions"].append({"step": "loop_state", "action": "would-register", + "in_flight_run": run_id, "train_pid": train_pid, + "ckpt_path": ckpt_path, "resume_cmd": resume_cmd}) + else: + kw = {"in_flight_run": run_id, "train_pid": train_pid} + if ckpt_path is not None: + kw["ckpt_path"] = ckpt_path + if resume_cmd is not None: + kw["resume_cmd"] = resume_cmd + try: + loop_state.register(state_path, ts=ts, **kw) + except (ValueError, OSError) as e: + report["actions"].append({"step": "loop_state", "action": "FAILED", + "error": str(e)}) + return err(f"loop_state.register failed: {e}") + report["actions"].append({"step": "loop_state", "action": "registered", **kw}) + + report["ok"] = True + return report + + +# ------------------------------------------------------------ verb: reconcile + +def reconcile(*, ledger_path=DEFAULT_LEDGER, state_path=DEFAULT_STATE, + clear_resume=False, ts=None, today=None, apply=False) -> dict: + """Detect an in-flight run whose trainer is gone, and (with --apply) close the + transaction: terminal STATUS on the ledger entry + a cleared loop_state pointer. + + `state` is one of: + clean — no in_flight_run; nothing to do. + alive — the trainer is still running; nothing to do (never touch it). + unverifiable — an in_flight_run with NO train_pid: death cannot be proven, so + nothing is mutated. Fail closed (§C6) — a missing signal is not + evidence of death. + stale — the pid is gone. Report, and with --apply, reconcile. + stale-unrecorded — stale AND the ledger has no entry for it (the out-of-loop + pathology in its purest form): the pointer is cleared, but no + ledger entry is invented. `adopt` is what creates entries. + """ + ts = ts or _now_iso() + today = today or _today() + report = {"verb": "reconcile", "ok": True, "dry_run": not apply, "ts": ts, + "state": None, "run_id": None, "train_pid": None, + "actions": [], "errors": []} + + st = loop_state.load(state_path) + run_id, pid = st.get("in_flight_run"), st.get("train_pid") + report["run_id"], report["train_pid"] = run_id, pid + if st.get("_recovered"): + report["actions"].append({"step": "loop_state", "action": "note", + "why": "state file missing/corrupt — read fail-open"}) + if not run_id: + report["state"] = "clean" + return report + if pid is None: + report["state"] = "unverifiable" + report["actions"].append({"step": "liveness", "action": "skipped", + "why": f"in_flight_run={run_id!r} has no train_pid; " + "liveness cannot be proven — refusing to " + "declare it dead"}) + return report + if pid_alive(pid): + report["state"] = "alive" + report["actions"].append({"step": "liveness", "action": "alive", + "pid": pid, "why": "trainer still running — untouched"}) + return report + + # --- the trainer is gone ------------------------------------------------- + report["state"] = "stale" + report["actions"].append({"step": "liveness", "action": "dead", "pid": pid}) + try: + entry = ledger_run_entry(ledger_path, run_id) + except RuntimeError as e: + report["ok"] = False + report["errors"].append(str(e)) + return report + + new_status = None + if entry is None: + report["state"] = "stale-unrecorded" + report["actions"].append({"step": "ledger", "action": "no-entry", + "why": f"no ledger run {run_id!r} — this launch was " + "never recorded; run `adopt` to create the " + "entry. Refusing to invent one here"}) + elif entry.get("status") not in IN_FLIGHT_STATUSES: + report["actions"].append({"step": "ledger", "action": "already-terminal", + "status": entry.get("status")}) + else: + # Terminal status is READ off disk, never assumed: a verdict.json in the + # artifacts dir means the run reached its end; its absence means it did not. + adir = entry.get("artifacts_dir") + vpath = (ROOT / adir / "verdict.json") if adir else None + finished = bool(vpath and vpath.exists()) + # §C8 vocabulary, never a free-text status (drift is caught by + # test_terminal_statuses_are_ledger_vocabulary against ledger.RUN_STATUS). + new_status = TERMINAL_IF_FINISHED if finished else TERMINAL_IF_DIED + basis = (f"verdict.json present at {adir}/verdict.json" + if finished else "no verdict.json in the artifacts dir") + reconciled = {"protocol": PROTOCOL_VERSION, "reconciled_on": ts, + "dead_train_pid": pid, "basis": basis, + "verdict_untouched": True} + sets = ["--set", f"status={new_status}", "--set", f"ended={today}", + "--set", f"reconciled={json.dumps(reconciled)}"] + argv = ["update-run", run_id] + sets + if not apply: + report["actions"].append({"step": "ledger", "action": "would-update-run", + "status": new_status, "basis": basis, + "argv": argv}) + else: + rc, out, cli_err = _ledger_cli(ledger_path, argv) + if rc != 0: + report["ok"] = False + report["actions"].append({"step": "ledger", "action": "FAILED", + "exit": rc, "stderr": cli_err.strip()}) + report["errors"].append( + f"ledger update-run FAILED (exit {rc}): {(cli_err or out).strip()} " + "— loop_state pointer left in place so the stale run is not lost") + return report + report["actions"].append({"step": "ledger", "action": "update-run", + "status": new_status, "basis": basis}) + + # Clear the stale pointer. `resume_cmd`/`ckpt_path` are PRESERVED by default: + # boot_resume.sh needs BOTH in_flight_run and resume_cmd to act, so clearing the + # pointer already disarms auto-resume, and keeping the command means a human can + # still relaunch by hand from the exact recorded form. --clear-resume drops them. + kw = {"in_flight_run": None, "train_pid": None} + if clear_resume: + kw.update({"ckpt_path": None, "resume_cmd": None}) + if not apply: + report["actions"].append({"step": "loop_state", "action": "would-clear", **kw}) + else: + try: + loop_state.register(state_path, ts=ts, **kw) + except (ValueError, OSError) as e: + report["ok"] = False + report["errors"].append(f"loop_state.register failed: {e}") + return report + report["actions"].append({"step": "loop_state", "action": "cleared", **kw}) + report["new_status"] = new_status + return report + + +# ---------------------------------------------------------------- CLI + +def main(argv=None) -> int: + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--ledger", type=Path, default=DEFAULT_LEDGER) + common.add_argument("--state", type=Path, default=DEFAULT_STATE, + help="loop_state.json (written only via loop_state.register)") + common.add_argument("--ts", default=None, help="timestamp stamp; default: now (§C2)") + common.add_argument("--apply", action="store_true", + help="actually mutate; without it nothing is written") + + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("adopt", parents=[common], + help="record a manual launch as a governed run") + p.add_argument("run_dir", type=Path, help="/experiments//") + p.add_argument("--train-pid", type=int, default=None, + help="live trainer pid to register as in-flight (must be alive)") + p.add_argument("--ckpt-path", default=None) + p.add_argument("--resume-cmd", default=None, + help="the exact detached relaunch form the recovery chain replays") + p.add_argument("--run-type", default=None, choices=sorted(RUN_TYPES), + help="override the objective->type mapping (§C8 vocabulary)") + p.add_argument("--digests", type=Path, default=DEFAULT_DIGESTS) + p.add_argument("--force", action="store_true", + help="override the run-id/dir mismatch and in-flight-collision refusals") + + p = sub.add_parser("reconcile", parents=[common], + help="close out an in-flight run whose trainer is gone") + p.add_argument("--clear-resume", action="store_true", + help="also null ckpt_path/resume_cmd (default: keep them so a " + "human can still relaunch by hand)") + p.add_argument("--date", default=None, metavar="YYYY-MM-DD", + help="`ended` date; default: today (§C2)") + + a = ap.parse_args(argv) + if a.cmd == "adopt": + r = adopt(a.run_dir, ledger_path=a.ledger, state_path=a.state, + digest_dir=a.digests, train_pid=a.train_pid, + resume_cmd=a.resume_cmd, ckpt_path=a.ckpt_path, + run_type=a.run_type, ts=a.ts, apply=a.apply, force=a.force) + print(json.dumps(r, indent=2)) + return 0 if r["ok"] else 1 + r = reconcile(ledger_path=a.ledger, state_path=a.state, + clear_resume=a.clear_resume, ts=a.ts, today=a.date, apply=a.apply) + print(json.dumps(r, indent=2)) + if not r["ok"]: + return 1 + # exit 4 = "a stale in-flight run is sitting there and --apply was not given", + # the same shape as `sentinel.py liveness` exit 4, so a cron can key on it. + if r["state"] in ("stale", "stale-unrecorded") and not a.apply: + return 4 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/calibration_pairs.py b/research/calibration_pairs.py new file mode 100644 index 0000000..228f7b6 --- /dev/null +++ b/research/calibration_pairs.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""research/calibration_pairs.py — the (predicted, realised) JOIN that feeds +§C15.3.8 scorer calibration. + +`research/scorer_calibration.py` is the MATH (Brier / log-loss / reliability +bins / ECE / shrinkage); it takes two paired lists and has no idea where they +come from. This module is the missing half: it walks the ledger and produces +those lists, honestly, from + + techniques[].predicted_win_prob <-> techniques[].run_ids[] <-> runs[].verdict + +READ-ONLY by construction: it opens ledger.json for reading and has no write +path at all (no ledger.py mutator is imported). Stdlib-only, pure CPU. + +WHY THIS SHIPS WITHOUT A NUMBER. As of 2026-07-23 not one technique in the +ledger carries `predicted_win_prob` (`/idea-selection` has never run against +it), so the join yields n=0. `scorer_calibration.report` raises on an empty +pair list, and /weekly-retro puts the honest floor at ~10 pairs ("descriptive +only — n= below the calibration floor"). So this module reports the JOIN and +the HYGIENE GAPS and stops there; publishing an ECE off n=0 would be inventing +a number. Once the pairs exist, the report half is one line: + + scorer_calibration.report(**calibration_pairs.scorer_inputs()) + +CLI: + python3 research/calibration_pairs.py [--ledger PATH] [--per-run] + -> {"pairs": [...], "pending": [...], "hygiene_gaps": [...], "n": N} +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +DEFAULT_LEDGER = Path(__file__).resolve().parent / "ledger" / "ledger.json" + +# A RECORDED verdict is a realised outcome. The scorer earns credit only for a +# verified `win` (§C17/§C19: an unverified single-seed win is `inconclusive` by +# construction, so it scores 0 here); every other recorded verdict is "not a +# win" = 0. `promising`/`null` are the §C25.3 split of the old `directional`. +# +# TRAP, stated because the two look identical in prose: the verdict WORD "null" +# is a MEASURED no-effect result — realised, outcome 0 — and is NOT the same +# thing as a JSON `null` verdict, which means not-judged-yet and is EXCLUDED as +# pending. Contracts §C8 draws exactly this distinction. +OUTCOME = {"win": 1, "loss": 0, "inconclusive": 0, + "null": 0, "promising": 0, "directional": 0} + +# Statuses at or past `briefed` (a technique reaches `done` through `briefed`). +# §C15.3.8 requires predicted_win_prob from `briefed` onward, so any of these +# without the field is a hygiene gap, not a silent drop. +REACHED_BRIEFED = frozenset({"briefed", "queued", "running", "done"}) + +# /weekly-retro's honesty floor: below this the ECE bins are nearly empty, so +# report raw Brier/log-loss at most and never publish a shrinkage off it. +MIN_PAIRS_FOR_CALIBRATION = 10 + + +def load_ledger(path=None) -> dict: + """Parse ledger.json. Read-only; no validation side effects, no writes.""" + return json.loads(Path(path or DEFAULT_LEDGER).read_text()) + + +def below_calibration_floor(n: int) -> bool: + """True when n is too small for the reliability curve / shrinkage to mean + anything (the caller must then label the report descriptive-only).""" + return n < MIN_PAIRS_FOR_CALIBRATION + + +def collect(led: dict, per_run: bool = False) -> dict: + """Join predictions to realised outcomes. + + Granularity, because it changes n and therefore every calibration number: + the scorer predicts a win-probability for an IDEA, so the default emits ONE + pair per technique — outcome 1 if any of its realised runs won, else 0. + `per_run=True` emits one pair per run, which repeats the SAME prediction + across a cohort's runs and inflates n on correlated outcomes; it exists for + inspection, not for feeding the report. + + Returns {"pairs", "pending", "hygiene_gaps", "n"}. Every pair carries its + technique slug and contributing run_ids so any number is traceable back to + a ledger entry (verifiable-accuracy rule). + """ + runs = {r.get("run_id"): r for r in led.get("runs", [])} + pairs, pending, gaps = [], [], [] + + for t in led.get("techniques", []): + slug, status = t.get("slug"), t.get("status") + pwp = t.get("predicted_win_prob") + if pwp is None: + if status in REACHED_BRIEFED: + gaps.append({"kind": "briefed_without_prediction", "technique": slug, + "status": status, + "detail": "§C15.3.8 requires predicted_win_prob once " + "a technique is briefed"}) + continue + run_ids = t.get("run_ids") or [] + if not run_ids: + gaps.append({"kind": "prediction_without_run", "technique": slug, + "status": status, "predicted_win_prob": pwp, + "detail": "prediction can never be realised: no run_ids[]"}) + continue + + realised = [] + for rid in run_ids: + r = runs.get(rid) + if r is None: + gaps.append({"kind": "run_id_not_found", "technique": slug, + "run_id": rid, + "detail": "technique.run_ids[] names a run with no " + "runs[] entry"}) + continue + verdict = r.get("verdict") + if verdict is None: # JSON null = not judged yet + pending.append({"technique": slug, "run_id": rid, + "predicted_win_prob": pwp, + "run_status": r.get("status")}) + continue + realised.append((rid, OUTCOME.get(verdict, 0), verdict)) + + if not realised: + if status == "done": + gaps.append({"kind": "done_without_realised_verdict", + "technique": slug, "run_ids": run_ids, + "detail": "technique is done but no run carries a " + "verdict — outcome unrealisable"}) + continue + if per_run: + pairs.extend({"technique": slug, "predicted_win_prob": pwp, + "outcome": y, "run_ids": [rid], "verdicts": [v]} + for rid, y, v in realised) + else: + pairs.append({"technique": slug, "predicted_win_prob": pwp, + "outcome": 1 if any(y for _, y, _ in realised) else 0, + "run_ids": [rid for rid, _, _ in realised], + "verdicts": [v for _, _, v in realised]}) + + return {"pairs": pairs, "pending": pending, "hygiene_gaps": gaps, + "n": len(pairs)} + + +def scorer_inputs(result=None, **kwargs) -> dict: + """The exact keyword shape `scorer_calibration.report(preds, outcomes)` + wants, so the report half is `report(**scorer_inputs())`. Pass a `collect()` + result, or nothing to join the default ledger.""" + if result is None: + result = collect(load_ledger(kwargs.pop("ledger", None)), **kwargs) + return {"preds": [p["predicted_win_prob"] for p in result["pairs"]], + "outcomes": [p["outcome"] for p in result["pairs"]]} + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--ledger", type=Path, default=DEFAULT_LEDGER) + ap.add_argument("--per-run", action="store_true", + help="one pair per RUN instead of per technique (inflates n " + "on multi-run cohorts; inspection only)") + a = ap.parse_args(argv) + print(json.dumps(collect(load_ledger(a.ledger), per_run=a.per_run), indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/research/eval_completeness.py b/research/eval_completeness.py index 1945d0a..e001c31 100644 --- a/research/eval_completeness.py +++ b/research/eval_completeness.py @@ -4,12 +4,19 @@ 28.65/23.52/29.54 shipped on n=1 FineWeb val PPL with no downstream / no seed CI). Given a run's `lifecycle_stage` and the set of eval items it actually recorded, this decides whether the STAGE-DONE *required* battery ran. A run missing any HARD item is -capped at verdict `directional` (never `win`), stamped `incomplete-eval: `. +capped BELOW `win` and stamped `incomplete-eval: `. Layered ON TOP of the significance gates (§C10/§C13/§C17/§C18/§C21): completeness asks "did the right battery run?", significance asks "is the number real?". A run is `win` only if HARD-complete AND significant. +Verdict vocabulary (split 2026-07-22, authority = ledger.py §C8): the cap is no longer +the single word `directional`, which compressed two OPPOSITE realities — "found nothing" +and "found something big, one gate short" — into one token. The cap now preserves that +distinction: a real measured effect held back by a missing HARD item is `promising`, a +measured no-effect is `null`, an uninterpretable contrast is `inconclusive`. `directional` +remains a legal ledger value for historical entries but is NEVER emitted here. + Source of truth for the human matrix: research/eval/per_stage_eval_batteries.md. Stdlib-only, pure CPU, no network — headless-safe for the loop (research-loop S8 / ablation-runner Phase 6). Recency per §C25.7: re-research a stage if RESEARCHED_ON is @@ -17,6 +24,45 @@ """ from __future__ import annotations +import importlib.util +from pathlib import Path + +# ── Verdict vocabulary ────────────────────────────────────────────────────────────── +# The ledger is the single source of truth (§C8/§C11); this gate must never invent a word +# `ledger.py` would reject. research/ledger/ is not a package, so load it by absolute path +# (cwd-independent for the headless loop) and deliberately do NOT register it in sys.modules, +# so it cannot shadow the plain `import ledger` used elsewhere (research/tests/test_ledger.py). +_LEDGER_PY = Path(__file__).resolve().parent / "ledger" / "ledger.py" + + +def _load_ledger_vocabulary(): + spec = importlib.util.spec_from_file_location("_c25_ledger_vocab", _LEDGER_PY) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # no import-time side effects: ledger.py's CLI is under __main__ + return frozenset(mod.VERDICTS), frozenset(mod.NEUTRAL_VERDICTS) + + +VERDICTS, NEUTRAL_VERDICTS = _load_ledger_vocabulary() +DEPRECATED_VERDICTS = frozenset({"directional"}) # legal in old entries; never emitted by this gate +VERDICT_VOCAB = "split-2026-07-22" # stamped into every result so a v1-era + # `directional` is never confused with a new call + +# Significance verdicts this gate accepts from the §C13/§C17 gate. Anything else (None, a typo, +# the deprecated `directional`) is read as `inconclusive` — fail closed, per §C6: an unreadable +# signal never buys a win. +ACCEPTED_SIGNIFICANCE = frozenset({"win", "promising", "null", "loss", "inconclusive"}) + +# §C25.3 — what a HARD-incomplete battery downgrades each significance verdict TO. Every value +# is in NEUTRAL_VERDICTS: an incomplete battery can never yield `win`, and can never burn a +# `loss` into never_repeat (ledger.py auto-appends never_repeat[] on verdict=loss). +CAP_WHEN_INCOMPLETE = { + "win": "promising", # real, measured effect — short exactly the missing HARD item(s) + "promising": "promising", + "null": "null", # measured no-effect; a missing item cannot manufacture one + "loss": "inconclusive", # measured worse, but an incomplete battery may not condemn it + "inconclusive": "inconclusive", +} + REGISTRY_VERSION = "v1" RESEARCHED_ON = "2026-06-22" RECENCY_WINDOW_DAYS = 120 # §C25.7.2 @@ -25,7 +71,7 @@ "sft", "preference", "rlvr", "safety", "interpretability", "systems", "serving") # Per stage: `required` = always-HARD items (must be in the run's recorded items, or the -# run is capped to `directional`). `conditional` = HARD only when the named condition is +# run is capped below `win`). `conditional` = HARD only when the named condition is # active (e.g. an architecture change that touches attention requires the KV/latency Pareto). # Item keys are abstract identifiers the supplying skill stamps into the run's metrics. REGISTRY: dict[str, dict] = { @@ -93,7 +139,7 @@ # §C26 — every step ships a FIGURE. Global report-only item checked for ALL stages: a run # without a `figure` artifact is flagged `report_missing: [figure]` and may not be rendered to a # README/digest until `research/eval_plots.figure_for_run(run_dir)` has produced one. Tracked, not -# a HARD significance cap (a missing plot doesn't make a real result `directional`). +# a HARD significance cap (a missing plot doesn't downgrade a real result's verdict). GLOBAL_REPORT_ONLY = ("figure",) # §C25.7.3 — items that may NEVER be a stage's SOLE headline signal (auto-flagged). @@ -111,6 +157,11 @@ def check_completeness(lifecycle_stage: str, present_items, conditions=None) -> present_items: iterable of recorded eval-item keys the run actually produced. conditions: iterable of active condition flags (e.g. {"quantization"}). + + `verdict_cap` is the CEILING — the strongest verdict this run may carry — not the final + call: `None` when the HARD battery is complete (a `win` is permitted), `promising` when a + HARD item is missing, `inconclusive` when the stage is unknown (§C25.1). gate_verdict() + picks the actual word inside that ceiling from the significance signal. """ present = set(present_items or ()) conds = set(conditions or ()) @@ -118,6 +169,7 @@ def check_completeness(lifecycle_stage: str, present_items, conditions=None) -> if lifecycle_stage not in REGISTRY: return {"stage": lifecycle_stage, "known": False, "complete": False, "missing_hard": [], "verdict_cap": "inconclusive", "report_missing": report_missing, + "verdict_vocab": VERDICT_VOCAB, "reason": f"unknown lifecycle_stage '{lifecycle_stage}' — cannot be win (§C25.1)"} spec = REGISTRY[lifecycle_stage] required = list(spec["required"]) @@ -125,44 +177,76 @@ def check_completeness(lifecycle_stage: str, present_items, conditions=None) -> if cond in conds: required += items missing = [k for k in required if k not in present] - bad_sole = sorted(present & DISALLOWED_SOLE_SIGNAL) if len(present) == 1 else [] + # §C25.7.3: a disallowed item is the SOLE headline whenever it is present AND — after setting + # aside report-only artifacts (§C26 `figure`) and the disallowed items themselves — NO admissible + # effect-measurement signal remains. The old `len(present) == 1` test let a founding-mistake + # headline (e.g. `valppl_n1_stage_headline`) escape the floor the instant ANY second item was + # recorded, even a report-only figure — so a confounded n=1 val-PPL run flanked by a plot could + # still reach `promising`/`win`. Judge sole-ness by what's left after the report-only set, not by count. + disallowed_present = present & DISALLOWED_SOLE_SIGNAL + admissible = present - DISALLOWED_SOLE_SIGNAL - set(GLOBAL_REPORT_ONLY) + bad_sole = sorted(disallowed_present) if (disallowed_present and not admissible) else [] complete = not missing and not bad_sole return { "stage": lifecycle_stage, "known": True, "complete": complete, "required": required, "present": sorted(present), "missing_hard": missing, "disallowed_sole_signal": bad_sole, "report_missing": report_missing, # §C26: [figure] if no plot - "verdict_cap": None if complete else "directional", + # ceiling, not the call: a disallowed-sole-signal run has no admissible effect measurement, + # so its ceiling is `inconclusive` (matching gate_verdict), not `promising` (see docstring). + "verdict_cap": None if complete else ("inconclusive" if bad_sole else "promising"), "registry_version": REGISTRY_VERSION, "researched_on": RESEARCHED_ON, + "verdict_vocab": VERDICT_VOCAB, } def gate_verdict(lifecycle_stage: str, present_items, significance_verdict: str, conditions=None) -> dict: """Compose §C25.3: completeness caps significance. - significance_verdict ∈ {win, loss, inconclusive} from the existing §C13/§C17 gate.""" + + significance_verdict: the §C13/§C17 call, one of ACCEPTED_SIGNIFICANCE. Anything else is + read as `inconclusive` (fail closed). Returns the verdict to record in the ledger — never + `win` unless the HARD battery is complete, and never the deprecated `directional`. + """ + sig = significance_verdict if significance_verdict in ACCEPTED_SIGNIFICANCE else "inconclusive" c = check_completeness(lifecycle_stage, present_items, conditions) - if not c["complete"]: - verdict = c["verdict_cap"] # "directional" (known but missing HARD) | "inconclusive" (unknown stage, §C25.1) - why = ("incomplete-eval: " + ", ".join(c["missing_hard"] + c.get("disallowed_sole_signal", [])) - if c["known"] else c["reason"]) - elif significance_verdict == "win": + if not c["known"]: + verdict, why = c["verdict_cap"], c["reason"] # unknown stage → inconclusive (§C25.1) + elif not c["complete"]: + why = "incomplete-eval: " + ", ".join(c["missing_hard"] + c["disallowed_sole_signal"]) + if c["disallowed_sole_signal"]: + # the run's ONLY signal is disallowed as a headline (§C25.7.3) → there is no + # admissible effect measurement at all, so this cannot even be `promising`. + verdict = "inconclusive" + else: + verdict = CAP_WHEN_INCOMPLETE[sig] + if sig == "loss": + why += " — measured worse, but an incomplete battery may not burn a never_repeat loss (§C25.3)" + elif sig == "win": verdict, why = "win", "HARD-complete and significant (§C25.3.5)" else: - verdict, why = significance_verdict, "HARD-complete; significance gate decides" - return {"verdict": verdict, "completeness": c, "significance_verdict": significance_verdict, "why": why} + verdict, why = sig, "HARD-complete; significance gate decides" + # Postcondition — the reader that keeps this gate and ledger.VERDICTS in lockstep (§C8/§C11): + # every emitted word is a current ledger verdict, and an incomplete battery emits only a + # NEUTRAL one — never `win`, never a `loss` (which auto-appends to never_repeat[]). + if verdict not in VERDICTS or verdict in DEPRECATED_VERDICTS: + raise ValueError(f"§C25 gate emitted '{verdict}', not a current ledger verdict") + if not c["complete"] and verdict not in NEUTRAL_VERDICTS: + raise ValueError(f"§C25 cap violated: incomplete battery emitted '{verdict}'") + return {"verdict": verdict, "completeness": c, "significance_verdict": significance_verdict, + "significance_read_as": sig, "why": why} def _self_test(): # data stage: all required present + no active condition -> complete full = REGISTRY["data"]["required"] assert check_completeness("data", full)["complete"], "full data battery should be complete" - # drop one required -> directional, names the missing item + # drop one required -> capped below win, names the missing item part = check_completeness("data", full[:-1]) - assert not part["complete"] and part["verdict_cap"] == "directional" + assert not part["complete"] and part["verdict_cap"] == "promising" assert "second_lr_recheck" in part["missing_hard"], part # the founding mistake: a pretrain/base-eval headline on ONLY n=1 val PPL bad = check_completeness("base-eval", ["valppl_n1_stage_headline"]) - assert not bad["complete"] and bad["verdict_cap"] == "directional" + assert not bad["complete"] and bad["disallowed_sole_signal"] == ["valppl_n1_stage_headline"] # conditional fires only when active: architecture touching attention needs the Pareto arch_req = REGISTRY["architecture"]["required"] assert check_completeness("architecture", arch_req)["complete"] # no condition @@ -170,16 +254,30 @@ def _self_test(): assert "kv_ttft_itl_pareto" in c_attn["missing_hard"] # condition makes it HARD # unknown stage cannot win assert check_completeness("frobnicate", ["x"])["verdict_cap"] == "inconclusive" - # gate_verdict composition: complete+significant=win; incomplete caps to directional + # gate_verdict composition: complete+significant=win; incomplete caps below win, and the + # 2026-07-22 split keeps "found something, one gate short" apart from "found nothing" assert gate_verdict("data", full, "win")["verdict"] == "win" - assert gate_verdict("data", full[:-1], "win")["verdict"] == "directional" + assert gate_verdict("data", full[:-1], "win")["verdict"] == "promising" + assert gate_verdict("data", full[:-1], "null")["verdict"] == "null" assert gate_verdict("data", full, "loss")["verdict"] == "loss" + # an incomplete battery may not condemn an arm to never_repeat (§C25.3) + assert gate_verdict("data", full[:-1], "loss")["verdict"] == "inconclusive" + # the only signal is §C25.7.3-disallowed -> no admissible effect, so not even `promising` + assert gate_verdict("base-eval", ["valppl_n1_stage_headline"], "win")["verdict"] == "inconclusive" + # unreadable significance fails closed + assert gate_verdict("data", full, "banana")["verdict"] == "inconclusive" + # the deprecated word is never emitted, from any stage/battery/significance combination + for st, spec in REGISTRY.items(): + for items in (spec["required"], spec["required"][1:], []): + for s in sorted(ACCEPTED_SIGNIFICANCE): + assert gate_verdict(st, items, s)["verdict"] not in DEPRECATED_VERDICTS # every stage is well-formed (no key collisions between required and conditional) for st, spec in REGISTRY.items(): req = set(spec["required"]) for items in spec.get("conditional", {}).values(): assert not (set(items) & req), f"{st}: conditional overlaps required" - print(f"eval_completeness self-test PASS — {len(STAGES)} stages, registry {REGISTRY_VERSION} ({RESEARCHED_ON})") + print(f"eval_completeness self-test PASS — {len(STAGES)} stages, registry {REGISTRY_VERSION} " + f"({RESEARCHED_ON}), verdict vocab {VERDICT_VOCAB}") if __name__ == "__main__": diff --git a/research/ledger/ledger.json b/research/ledger/ledger.json index e930c3e..2823f9c 100644 --- a/research/ledger/ledger.json +++ b/research/ledger/ledger.json @@ -549,7 +549,15 @@ "ended": null, "status": "done", "verdict": null, - "metrics": {}, + "metrics": { + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 280.56, + "code_ppl": 438.67, + "self_floor": true, + "suite_version": "text-lm-v2", + "wikitext2_floor_abs": 1.3, + "wikitext2_ppl": 37.01 + }, "artifacts_dir": null, "lineage": { "git_commit": "41824a5", @@ -563,15 +571,8 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-faithful.md", - "objective": "pretrain-ablation", - "suite_version": "text-lm-v2", - "self_floor": true, - "wikitext2_ppl": 37.01, - "wikitext2_floor_abs": 1.3, - "code_ppl": 438.67, - "code_corpus_id": "codeparrot_clean_valid", - "code_floor_abs": 280.56 + "detail_md": null, + "objective": "pretrain-ablation" }, { "run_id": "2026-06-16_qwen3-0.6b_eval-modernized", @@ -587,7 +588,15 @@ "ended": null, "status": "done", "verdict": null, - "metrics": {}, + "metrics": { + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 111.92, + "code_ppl": 129.42, + "self_floor": true, + "suite_version": "text-lm-v2", + "wikitext2_floor_abs": 1.07, + "wikitext2_ppl": 27.8 + }, "artifacts_dir": null, "lineage": { "git_commit": "41824a5", @@ -601,15 +610,8 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-modernized.md", - "objective": "pretrain-ablation", - "suite_version": "text-lm-v2", - "self_floor": true, - "wikitext2_ppl": 27.8, - "wikitext2_floor_abs": 1.07, - "code_ppl": 129.42, - "code_corpus_id": "codeparrot_clean_valid", - "code_floor_abs": 111.92 + "detail_md": null, + "objective": "pretrain-ablation" }, { "run_id": "2026-06-16_qwen3-0.6b_eval-prope25", @@ -625,7 +627,15 @@ "ended": null, "status": "done", "verdict": null, - "metrics": {}, + "metrics": { + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 298.9, + "code_ppl": 447.3, + "self_floor": true, + "suite_version": "text-lm-v2", + "wikitext2_floor_abs": 1.7, + "wikitext2_ppl": 38.08 + }, "artifacts_dir": null, "lineage": { "git_commit": "41824a5", @@ -639,15 +649,8 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-prope25.md", - "objective": "pretrain-ablation", - "suite_version": "text-lm-v2", - "self_floor": true, - "wikitext2_ppl": 38.08, - "wikitext2_floor_abs": 1.7, - "code_ppl": 447.3, - "code_corpus_id": "codeparrot_clean_valid", - "code_floor_abs": 298.9 + "detail_md": null, + "objective": "pretrain-ablation" }, { "run_id": "2026-06-16_qwen3-0.6b_eval-prope10", @@ -663,7 +666,15 @@ "ended": null, "status": "done", "verdict": null, - "metrics": {}, + "metrics": { + "code_corpus_id": "codeparrot_clean_valid", + "code_floor_abs": 690.96, + "code_ppl": 1356.31, + "self_floor": true, + "suite_version": "text-lm-v2", + "wikitext2_floor_abs": 3.67, + "wikitext2_ppl": 69.63 + }, "artifacts_dir": null, "lineage": { "git_commit": "41824a5", @@ -677,15 +688,8 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-16_qwen3-0.6b_eval-prope10.md", + "detail_md": null, "objective": "pretrain-ablation", - "suite_version": "text-lm-v2", - "self_floor": true, - "wikitext2_ppl": 69.63, - "wikitext2_floor_abs": 3.67, - "code_ppl": 1356.31, - "code_corpus_id": "codeparrot_clean_valid", - "code_floor_abs": 690.96, "note": "step-4000 checkpoint; run stopped early at ~step 5400 (undertrained vs 18150-step peers)" }, { @@ -859,7 +863,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-18_qwen3-0.6b_imu1-deconfound-p1.md", + "detail_md": null, "objective": "pretrain-ablation", "arm_plan": { "arms": [ @@ -930,7 +934,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-21_qwen3-0.6b_arch-subdrill-p2.md", + "detail_md": null, "objective": "pretrain-ablation", "arm_plan": { "arms": [ @@ -1014,7 +1018,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-24_qwen3-06b_dclm-edu-dataprep.md", + "detail_md": null, "objective": "pretrain-ablation", "dataset": { "id": "HuggingFaceTB/dclm-edu", @@ -1075,7 +1079,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-24_qwen3-0.6b_data-dclm-vs-fineweb.md", + "detail_md": null, "objective": "pretrain-ablation", "lifecycle_stage": "data", "confound_check": { @@ -1140,7 +1144,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-06-26_qwen3-0.6b_data-mix-composition.md", + "detail_md": null, "objective": "pretrain-ablation", "lifecycle_stage": "data", "confound_check": { @@ -1418,7 +1422,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-01_qwen3-0.6b_rlvr-phase1-passk.md", + "detail_md": null, "objective": "finetune", "lifecycle_stage": "rlvr", "note": "rlvr Phase-1 pass@k go/no-go (plan.md Phase 1, pre-registered rule in run_phase1_passk.py). DECISION: GO -> P1 prompt-set prep unlocked; Phase-2 GRPO training remains behind the needs-approval rlvr-method-plan proposal." @@ -1457,7 +1461,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-02_qwen3-0.6b_grpo-prompts-dataprep.md", + "detail_md": null, "objective": "finetune", "lifecycle_stage": "rlvr", "note": "P1 of rlvr plan (unlocked by Phase-1 GO): GRPO training prompts, eval-decontaminated (drops) + SFT-flagged (kept). Phase-2 GRPO training still needs-approval." @@ -1537,7 +1541,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-02_qwen3-0.6b_grpo-phase2.md", + "detail_md": null, "objective": "finetune", "lifecycle_stage": "rlvr", "note": "measured pace 362s/step (no-KV-cache decode dominates) -> revised ETA ~88h total (grpo ~30h + random ~30h + rft ~26h + evals); launched 2026-07-02 07:39" @@ -1570,7 +1574,8 @@ ], "headline_capped": true, "corpora_agree": true, - "conclusion": "DIRECTIONAL (CONVERGES shape on wikitext-2): the top budget (420M) rung is n=2 seeds (<3, \u00a7C17), so this is a directional trend, NOT a headline win \u2014 add a 3rd 420M seed (and the 840M rung) to earn more. Shape: The NorMuon advantage CONVERGES toward 0 with budget \u2014 an early-training speedup, as the IMU-1 RESULT.md Limitation #3 predicted it might." + "conclusion": "DIRECTIONAL (CONVERGES shape on wikitext-2): the top budget (420M) rung is n=2 seeds (<3, \u00a7C17), so this is a directional trend, NOT a headline win \u2014 add a 3rd 420M seed (and the 840M rung) to earn more. Shape: The NorMuon advantage CONVERGES toward 0 with budget \u2014 an early-training speedup, as the IMU-1 RESULT.md Limitation #3 predicted it might.", + "suite_version": "text-lm-v2" }, "artifacts_dir": "Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence", "lineage": { @@ -1585,10 +1590,9 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-05_qwen3-0.6b_scaling-persistence.md", + "detail_md": null, "objective": "pretrain-ablation", - "lifecycle_stage": "scaling", - "suite_version": "text-lm-v2" + "lifecycle_stage": "scaling" }, { "run_id": "2026-07-19_qwen3-0.6b_interp-cka-repconvergence", @@ -1618,7 +1622,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-19_qwen3-0.6b_interp-cka-repconvergence.md", + "detail_md": null, "lifecycle_stage": "interpretability", "metric": "linear_cka", "headline": "Pre-registered CKA null: NorMuon does NOT reach AdamW@420M representation earlier by more than the across-seed band; directional-but-sub-noise at 42M, reverses by 168M. A caught fp16-overflow confound faked an earlier null.", @@ -1663,7 +1667,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-19_qwen3-0.6b_cce-fused-ce.md", + "detail_md": null, "lifecycle_stage": "systems", "headline": "CCE fused linear CE: correctness PASS (torch+Triton vs naive fp32 oracle, re-verified GB10 2026-07-19, |Dloss|<1e-6); MEMORY win validated on GB10 (naive 61.4GB->CCE 2.04GB at 32768 tok, naive OOMs at 65536); throughput wall-clock CCE fastest (298ms vs 1038 torch.compile). Roofline off-box (propose-only). Convergence@pretrain UNVALIDATED (needs iso-FLOP A/B).", "spec": "research/kernel/SPEC_cce_fused_linear_ce.md", @@ -1704,7 +1708,11 @@ "suite_version": "text-lm-v2", "self_floor": true, "corpus_wikitext": "Salesforce/wikitext:wikitext-2-raw-v1:validation@b08601e", - "corpus_code": "codeparrot/codeparrot-clean-valid:train@4db92d2 (streaming first-N)" + "corpus_code": "codeparrot/codeparrot-clean-valid:train@4db92d2 (streaming first-N)", + "best_val_loss": 3.7839, + "final_train_loss": 3.8149, + "final_val_loss": 3.9245, + "step0_loss": 12.4317 }, "artifacts_dir": "HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build", "lineage": { @@ -1735,7 +1743,6 @@ "tok_per_step": 8192, "note": "batch reduced 8->4 after OOM; CE-under-autodiff memory follow-up recorded" }, - "step0_loss": 12.4317, "kill_reason": "sentinel memory kill at step 580 (pool 81.3% >= 0.80); reached loss 12.4->6.6 on real FineWeb-Edu; ckpt@step400 resumable but needs a memory fix before resume (SSM-scan + chunked-CE hold ~61GB)", "resume_needs": "remat/reduce-batch/reduce-SSM-state, then clear marker", "memory_fix": "nn.remat blocks: 61.5GB->16.6GB alloc, pool 81%->~42%; resumed from step-400 ckpt", @@ -1745,9 +1752,6 @@ "c5_evidence_provenance": "RECONSTRUCTED 2026-07-20 AFTER launch - the 2026-07-19 launch created this ledger entry but never wrote c5_evidence.json, so the SS-C5 \"evidence recorded BEFORE launch\" requirement was NOT met for this run. Per-item src tags (log/derived/attestation/not-captured) are in the evidence file.", "open_gate_gap": "verify.py last ran 2026-07-19 12:52, model.py modified 22:27 to add nn.remat -> the verify gate has NOT been re-run against the training model. Re-run + capture verify.log before scoring (GPU work; waits for the arm to finish per SS-C4.5).", "completed": "2026-07-20", - "final_train_loss": 3.8149, - "final_val_loss": 3.9245, - "best_val_loss": 3.7839, "steps": 21156, "incomplete_eval": [ "iso-flop sibling arms (none yet)", @@ -1785,7 +1789,7 @@ "eta_hours": 16.6, "started": "2026-07-21", "ended": null, - "status": "launched", + "status": "crashed", "verdict": null, "metrics": {}, "artifacts_dir": null, @@ -1801,7 +1805,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-20_hybrid-ssm-0.2b_pretrain-ssm-base-s1.md", + "detail_md": null, "objective": "pretrain-ablation", "lifecycle_stage": "architecture", "arm": "ssm_base_s1", @@ -1811,7 +1815,8 @@ "design": "seed ladder base cell (mixer=ssm, attn_every=2, nope off): identical to s0, only --seed 0->1. C17 paired-seed; same tokcache.", "resume_roundtrip": "pass (proven on s0: step-400 ckpt resume after sentinel kill)", "guards": "jax_safe_env before jax (train_hybrid.py:10); chunked CE (model.py:136); verified 2026-07-20", - "planned_launch": "after s0 eval-suite completes (C4.5); fresh smoke + sentinel watch at launch; status set to launched then" + "planned_launch": "after s0 eval-suite completes (C4.5); fresh smoke + sentinel watch at launch; status set to launched then", + "note": "launched 2026-07-21; no _s1 checkpoint/log/done on disk, superseded same day by the arch-ladder \u2014 abandoned launch, no result" }, { "run_id": "2026-07-21_hybrid-ssm-0.2b_arch-ladder", @@ -1840,7 +1845,7 @@ "eta_hours": 141.7, "started": "2026-07-21", "ended": null, - "status": "launched", + "status": "crashed", "verdict": null, "metrics": {}, "artifacts_dir": "HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder", @@ -1856,7 +1861,7 @@ "wall_clock_min": null, "gpu_hours": null }, - "detail_md": "research/ledger/runs/2026-07-21_hybrid-ssm-0.2b_arch-ladder.md", + "detail_md": null, "objective": "pretrain-ablation", "lifecycle_stage": "architecture", "confound_check": { @@ -1933,7 +1938,8 @@ "finding": "SSM mixer >> SWA/full-attn (but LR-not-retuned-per-arm confound); attention fraction 1:1~1:3 negligible; NoPE worse not better", "artifact": "rung_42M_comparison.md", "bpb": "deferred \u00a7C4.5" - } + }, + "note": "driver died in the 2026-07-23 15:24 BST hard-lock after 2 thermal SIGTERM kills (14:16Z, 14:21Z); 7/15 cells complete with on-disk .done markers, resumable via run_arch_ladder.sh (cooldown gate fixed this session)" } ], "proposals": [ diff --git a/research/ledger/ledger.py b/research/ledger/ledger.py index 7bf17cc..2245d06 100644 --- a/research/ledger/ledger.py +++ b/research/ledger/ledger.py @@ -28,6 +28,7 @@ query [--status S] [--type T] [--since YYYY-MM-DD] [--collection C] next-best --cutoff YYYY-MM-DD [--limit N] [--include-candidates] [--objective any|pretrain-ablation|finetune (§C13 filter)] + fsck [--fix] [--repo-root DIR] # integrity report; exit 1 if repairable status # human summary --set VALUES are JSON-parsed when possible ( --set score=8.5 @@ -38,8 +39,9 @@ verdict becomes loss, appends its slug to never_repeat[]. Exit codes: - 0 success (check-dup: slug is NEW / not blocked) - 1 check-dup only: DUPLICATE (slug in techniques[] or never_repeat[]) + 0 success (check-dup: slug is NEW / not blocked; fsck: clean or repaired) + 1 check-dup: DUPLICATE (slug in techniques[] or never_repeat[]). + fsck without --fix: repairable integrity issues remain 2 validation / usage / schema error 3 target entry not found (update-* on a missing slug or run_id) """ @@ -59,6 +61,7 @@ SCHEMA_VERSION = 1 DEFAULT_LEDGER = Path(__file__).resolve().parent / "ledger.json" +REPO_ROOT = Path(__file__).resolve().parents[2] # detail_md paths are repo-relative TOP_KEYS = ("techniques", "runs", "proposals", "never_repeat") TECH_STATUS = {"candidate", "briefed", "queued", "running", "done", "rejected", "proposal"} @@ -94,6 +97,61 @@ DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") RUN_ID_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_.+_.+$") # YYYY-MM-DD__ +# ------------------------------------------------- run-key hygiene (§C8/§C10) +# `--set k=v` accepts ANY key, which is what makes the CLI usable for the additive +# keys the contracts keep introducing — and also how 24 live runs accumulated 60+ +# ad-hoc top-level keys, several of them measured EVAL numbers stranded outside an +# empty metrics{} (the four 2026-06-16 eval runs). Everything a consumer compares +# across runs reads metrics{}; a number at top level is invisible to it. +# So: recognize the schema keys, WARN on the rest, and never hard-fail — a +# hard-fail would make the already-written ledger unloadable, i.e. unwritable. +RUN_CORE_KEYS = frozenset({ + "run_id", "type", "model_dir", "technique_slug", "budget", "probe", "smoke", + "framework", "eta_hours", "started", "ended", "status", "verdict", "metrics", + "artifacts_dir", "lineage", "cost", "detail_md", "objective"}) +# Additive keys the contracts name explicitly (§C8 "additive key, ledger.py only"). +RUN_ADDITIVE_KEYS = frozenset({ + "confound_check", # §C18 single-variable / iso-FLOP gate (validated above) + "lifecycle_stage", # §C25.1 + "launched_by", # §C11 launch provenance + "c5_lint", # §C5 pre-launch lint result stamped by /ablation-runner (pass) + "adopted", "evidence_path", "reconciled", # §C10 adopted-run protocol (research/adopt_run.py) + "is_remote", "cluster_shape", # §C20 off-box runs + "brief_path", "prior_run_id", "note"}) # §C5.2 brief inheritance (+its one-line note) +RUN_KEYS = RUN_CORE_KEYS | RUN_ADDITIVE_KEYS + +# Eval-shaped names get their own, louder advisory: those are the ones that +# silently break cross-run comparison. Canonical target shape = the metrics{} of +# run 2026-06-27_qwen3-0.6b_sft-3seed. +EVAL_KEY_EXACT = frozenset({"suite_version", "self_floor"}) +EVAL_KEY_SUFFIXES = ("_ppl", "_bpb", "_loss", "_acc", "_floor_abs", + "_corpus_id", "_ci95") + + +def is_eval_key(key: str) -> bool: + """True for a top-level run key that names a measured eval quantity (or the + §C10 suite stamp) and therefore belongs INSIDE metrics{}. Deliberately narrow: + it must never sweep up prose/bookkeeping keys (`note`, `train_pid`, `arm_plan`), + because fsck --fix MOVES what this matches.""" + return key in EVAL_KEY_EXACT or key.endswith(EVAL_KEY_SUFFIXES) + + +def unknown_run_keys(r: dict): + return sorted(k for k in r if k not in RUN_KEYS) + + +def warn_unknown_run_keys(run_id, keys): + """Advisory (never fatal) for top-level run keys outside the §C8 schema.""" + evalish = [k for k in keys if is_eval_key(k)] + other = [k for k in keys if not is_eval_key(k)] + if evalish: + warn(f"run[{run_id}]: EVAL-SHAPED top-level keys {evalish} — comparable " + "numbers must live INSIDE metrics{} (§C8/§C10); nothing that reads " + "metrics{} will ever see them. Fix: ledger.py fsck --fix") + if other: + warn(f"run[{run_id}]: unrecognized top-level keys {other} — not in the §C8 " + "run schema. Numbers belong in metrics{}, prose in detail_md") + def fail(msg: str, code: int = 2): print(json.dumps({"error": msg}), file=sys.stderr) @@ -466,6 +524,9 @@ def cmd_add_run(led, a): "status": "launched", "verdict": None, "metrics": {}, "artifacts_dir": a.artifacts_dir, "lineage": _new_lineage(), "cost": _new_cost(), # §C8 + # The PLANNED path of the detail doc — the owning skill writes it later. + # If it is never written, `fsck --fix` nulls the pointer rather than + # leaving the entry claiming a document that does not exist. "detail_md": f"research/ledger/runs/{a.run_id}.md"} r["lineage"]["git_commit"] = git_head_commit() # auto-capture repo HEAD # §C13: a run's objective is one of the two concrete values — never `any`, @@ -477,6 +538,9 @@ def cmd_add_run(led, a): patch = parse_sets(a.set) merge_subdicts(r, patch) # §C8: a partial lineage/cost --set merges, not replaces r.update(patch) + # Only the --set keys can be off-schema here (every other key above is the + # factory shape), so the advisory names exactly what THIS call invented. + warn_unknown_run_keys(r["run_id"], [k for k in patch if k not in RUN_KEYS]) # §C5 evidence must be recorded for ablation|finetune regardless of the # initial status — a --set status=... override must not mute the warning. if r["type"] in ("ablation", "finetune"): @@ -519,6 +583,10 @@ def cmd_update_run(led, a): r["framework"] = a.framework merge_subdicts(r, patch) # §C8: incremental lineage/cost updates merge, not replace r.update(patch) + # Warn on EVERY off-schema --set, not just the first one to introduce a key: + # the point is that the ad-hoc surface stops growing silently, and re-setting + # an ad-hoc key is exactly how it keeps growing. Stateless, so no false quiet. + warn_unknown_run_keys(r["run_id"], [k for k in patch if k not in RUN_KEYS]) if r.get("verdict") == "loss": block_never_repeat(led, r.get("technique_slug")) return r @@ -721,6 +789,84 @@ def taste_key(t): emit(elig[: a.limit] if a.limit else elig) +# ------------------------------------------------------------- integrity (fsck) + +def dangling_detail_md(led, repo_root=REPO_ROOT): + """Run entries whose `detail_md` names a document that is not on disk. + A pointer to a file that does not exist is a claim the ledger cannot back.""" + return [{"run_id": r.get("run_id"), "detail_md": r["detail_md"]} + for r in led.get("runs", []) + if isinstance(r.get("detail_md"), str) and r["detail_md"] + and not (Path(repo_root) / r["detail_md"]).exists()] + + +def stray_eval_keys(led): + """Eval-shaped top-level run keys that belong inside metrics{} (§C8/§C10). + `collides` = metrics{} already holds that key with a DIFFERENT value — two + values under one name, which a machine must not silently pick between.""" + out = [] + for r in led.get("runs", []): + m = r.get("metrics") or {} + for k in unknown_run_keys(r): + if is_eval_key(k): + out.append({"run_id": r.get("run_id"), "key": k, + "collides": k in m and m[k] != r[k]}) + return out + + +def other_unknown_keys(led): + """Non-eval off-schema top-level run keys. REPORTED ONLY — never auto-moved: + `arm_plan`/`train_pid`/`headline` are bookkeeping or prose, and guessing a + destination for them would invent structure the ledger never recorded.""" + out = [] + for r in led.get("runs", []): + ks = [k for k in unknown_run_keys(r) if not is_eval_key(k)] + if ks: + out.append({"run_id": r.get("run_id"), "keys": ks}) + return out + + +def cmd_fsck(led, a) -> int: + """Integrity report over runs[], with two SAFE repairs behind --fix. + + Repair 1 — a dangling `detail_md` is NULLED, not back-filled. A run .md is a + narrative artifact (hypothesis, config, caveats) written by the owning skill; + generating one from the entry would produce a document containing nothing the + entry does not already say, indistinguishable later from a real write-up. The + honest record of "no detail doc was ever written" is `null`, and it is + machine-checkable. `add-run` re-plans the path for each new run, so the + repair is re-runnable rather than one-shot. + + Repair 2 — an eval-shaped top-level key is MOVED into metrics{} with its value + byte-identical (`metrics[k] = r.pop(k)`); a value collision is skipped and + reported. Both repairs are idempotent: a second --fix finds nothing to do.""" + repo_root = a.repo_root or REPO_ROOT + dangling = dangling_detail_md(led, repo_root) + stray = stray_eval_keys(led) + fixed = {"detail_md_nulled": [], "eval_keys_moved": [], "skipped_collision": []} + if a.fix: + for d in dangling: + find(led["runs"], "run_id", d["run_id"])["detail_md"] = None + fixed["detail_md_nulled"].append(d["run_id"]) + for s in stray: + r = find(led["runs"], "run_id", s["run_id"]) + if s["collides"]: + fixed["skipped_collision"].append(f"{s['run_id']}.{s['key']}") + continue + r.setdefault("metrics", {})[s["key"]] = r.pop(s["key"]) + fixed["eval_keys_moved"].append(f"{s['run_id']}.{s['key']}") + if dangling or stray: + save(a.ledger, led) + emit({"dangling_detail_md": dangling, "eval_keys_at_top_level": stray, + "other_unknown_run_keys": other_unknown_keys(led), + "fixed": fixed if a.fix else None}) + # Exit 1 = repairable issues are still there (CI-gateable): unfixed, or a + # collision --fix deliberately refused to resolve. Off-schema non-eval keys + # are advisory and never move the exit code. + remaining = fixed["skipped_collision"] if a.fix else (dangling or stray) + return 1 if remaining else 0 + + def cmd_status(led, a): def counts(items): c = {} @@ -742,6 +888,10 @@ def counts(items): print(f" open: {p['slug']} kind={p.get('kind')} md={p.get('md_path')}") print(f"never_repeat ({len(led['never_repeat'])}): " + (", ".join(led["never_repeat"]) or "none")) + dangling, stray = dangling_detail_md(led), stray_eval_keys(led) + if dangling or stray: # silent once clean, so the line means "act on this" + print(f"integrity: {len(dangling)} dangling detail_md, {len(stray)} eval " + "key(s) outside metrics{} — run `ledger.py fsck --fix`") papers = led.get("papers", []) # §C16 print(f"papers ({len(papers)}): {counts(papers)}") for p in papers: @@ -871,6 +1021,14 @@ def main(argv=None) -> int: help="§C13 filter: keep techniques of this objective (an " "`any` technique fits either); skipped when val is `any`") + p = sub.add_parser("fsck", parents=[common]) + p.add_argument("--fix", action="store_true", + help="apply the safe repairs (null dangling detail_md; move " + "eval-shaped top-level keys into metrics{}); idempotent") + p.add_argument("--repo-root", type=Path, default=None, + help="root that detail_md paths resolve against " + "(default: this checkout)") + sub.add_parser("status", parents=[common]) a = ap.parse_args(argv) @@ -894,6 +1052,8 @@ def main(argv=None) -> int: return 0 if a.cmd == "check-dup": return cmd_check_dup(led, a) + if a.cmd == "fsck": # writes only under --fix, and only if dirty + return cmd_fsck(led, a) if a.cmd == "query": cmd_query(led, a) elif a.cmd == "next-best": diff --git a/research/loop_state.py b/research/loop_state.py index 05f26f7..6093a93 100644 --- a/research/loop_state.py +++ b/research/loop_state.py @@ -11,6 +11,14 @@ Design rules (all tested): - Atomic write: tempfile + os.replace in the same dir; prior version -> .bak. + - DURABLE write (§C8 parity with ledger.save): fsync(file) -> mode-preserving + chmod -> os.replace -> fsync(PARENT DIR). The parent-dir fsync is what makes + the rename itself survive the documented GB10 hard-lock; without it the file + contents are on disk but the directory entry can be lost on reboot, and the + @reboot recovery chain (boot_resume.sh) loses the in-flight run. + - LOCKED read-modify-write (§C8 parity with ledger.acquire_lock): every mutator + holds an exclusive advisory flock on .lock, so the loop, the + @reboot recovery hook and the liveness cron cannot lose-update each other. - FAIL-OPEN read: a missing OR corrupt state file returns a safe fresh default (flagged `_recovered`) and NEVER raises — a watchdog/liveness probe must keep running even if the state file is garbage. @@ -18,22 +26,31 @@ died, not from S0. - auto-resume cap: at most MAX_AUTO_RESUMES automatic resumes; the next dead run is classified `crashed`, not resumed (prevents an infinite resume loop). + - register() is the sanctioned setter for the recovery-critical flat in-flight + fields, so callers stop open-coding their own load->mutate->save (which skips + the .bak, the parent-dir fsync and the lock). - timestamps are caller-supplied (derived at runtime, §C2) so this module never reads a wall clock and stays deterministic/testable. + +Exit codes (CLI): 0 success; 2 validation/usage error. """ from __future__ import annotations import argparse +import contextlib +import fcntl import json import os import shutil import sys import tempfile +import time from pathlib import Path SCHEMA_VERSION = 1 STAGES = tuple(f"S{i}" for i in range(10)) # S0..S9 MAX_AUTO_RESUMES = 2 +LOCK_TIMEOUT_S = 10.0 # bounded wait for the advisory lock — see acquire_lock() # Recovery-critical flat in-flight fields (§C5) — the ones /ablation-runner writes @@ -58,6 +75,87 @@ def default_state() -> dict: "updated": None} +def acquire_lock(path, timeout: float = LOCK_TIMEOUT_S): + """§C8 cross-process safety, mirroring ledger.acquire_lock: an exclusive + advisory lock on .lock, serializing the load->mutate->save + sequence so the @reboot recovery (research/boot_resume.sh), the liveness + cron and the loop itself cannot lose-update each other. Fail-open: if + locking is unavailable, return None and proceed (a missing lock must never + brick the recovery path). + + ONE deliberate difference from ledger.py: the wait is BOUNDED. ledger.py + blocks forever on LOCK_EX, but this file sits on the once-per-boot recovery + path whose callers impose no timeout of their own, so an unbounded block + would silently forfeit the recovery. After `timeout` we proceed unlocked — + a lost update is recoverable, a hung @reboot hook is not.""" + p = Path(path) + try: + fd = os.open(str(p.with_name(p.name + ".lock")), os.O_CREAT | os.O_RDWR, 0o644) + except OSError: + return None + deadline = time.monotonic() + max(0.0, timeout) + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return fd + except OSError: + if time.monotonic() >= deadline: + os.close(fd) + return None + time.sleep(0.02) + + +def release_lock(fd): + if fd is not None: + try: + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + +_LOCKS: dict[str, list] = {} # realpath of the state file -> [depth, fd] + + +@contextlib.contextmanager +def locked(path, timeout: float = LOCK_TIMEOUT_S): + """Hold the advisory lock across a whole load->mutate->save block. Every + mutator below enters it, and an external caller may wrap several mutations + in ONE critical section. + + Re-entrant within a process: flock is per open-file-description, so a nested + os.open + LOCK_EX would deadlock against ourselves — nested entries just bump + a depth counter and reuse the outermost fd.""" + key = os.path.realpath(str(path)) + slot = _LOCKS.get(key) + if slot is None: + slot = _LOCKS[key] = [0, acquire_lock(path, timeout)] + slot[0] += 1 + try: + yield + finally: + slot[0] -= 1 + if slot[0] <= 0: + _LOCKS.pop(key, None) + release_lock(slot[1]) + + +def _fsync_parent_dir(p: Path) -> None: + """§C8 durability: fsync the PARENT DIR so the os.replace is itself + persisted. Without this, on the documented GB10 unified-memory hard-crash + the new directory entry can be lost on reboot — the state file comes back + absent or truncated even though its contents were fsynced, and the recovery + chain loses the in-flight run it was supposed to re-adopt. Best-effort: a + filesystem that refuses the directory fsync must never fail the write.""" + try: + dfd = os.open(str(p.parent), os.O_DIRECTORY) + try: + os.fsync(dfd) + finally: + os.close(dfd) + except OSError: + pass + + def load(path) -> dict: """Fail-open: missing/corrupt -> fresh default flagged `_recovered=True`. Never raises. A valid file is returned as-is (with `_recovered=False`).""" @@ -77,7 +175,9 @@ def load(path) -> dict: def save(path, state: dict) -> None: - """Atomic write + .bak snapshot. Strips the transient `_recovered` flag.""" + """Atomic + DURABLE write, plus the .bak snapshot; strips the transient + `_recovered` flag. Same sequence as ledger.save (§C8): fsync the contents, + preserve the file mode, os.replace, then fsync the parent directory.""" if state.get("stage") not in STAGES: raise ValueError(f"stage must be one of {STAGES}, got {state.get('stage')!r}") p = Path(path) @@ -92,7 +192,13 @@ def save(path, state: dict) -> None: f.write("\n") f.flush() os.fsync(f.fileno()) + # mkstemp creates the temp file 0600 and os.replace would keep that — + # preserve the existing state file's mode (or 0644 on first creation), + # or a root/cron-written state silently stops being readable by the + # other sanctioned writers. + os.chmod(tmp, (os.stat(p).st_mode & 0o777) if p.exists() else 0o644) os.replace(tmp, p) # atomic on POSIX, same filesystem + _fsync_parent_dir(p) except BaseException: if os.path.exists(tmp): os.unlink(tmp) @@ -106,13 +212,78 @@ def advance(path, stage: str, ts: str | None = None, in_flight=...) -> dict: verbatim across the transition (load returns a valid file as-is).""" if stage not in STAGES: raise ValueError(f"unknown stage {stage!r}") - st = load(path) - st["stage"] = stage - if ts is not None: - st["updated"] = ts - if in_flight is not ...: - st["in_flight_run"] = in_flight - save(path, st) + with locked(path): + st = load(path) + st["stage"] = stage + if ts is not None: + st["updated"] = ts + if in_flight is not ...: + st["in_flight_run"] = in_flight + save(path, st) + return st + + +def _check_run_id(v): + if v is None or isinstance(v, str): + return v + raise ValueError(f"in_flight_run must be the RUN_ID string or None, " + f"never a nested object (§C5): got {type(v).__name__}") + + +def _check_pid(v): + if v is None: + return None + if isinstance(v, bool) or not isinstance(v, int) or v <= 0: + raise ValueError(f"train_pid must be a positive int or None, got {v!r}") + return v + + +def _check_str_or_path(name, v): + if v is None: + return None + if isinstance(v, os.PathLike): + return os.fspath(v) + if not isinstance(v, str): + raise ValueError(f"{name} must be a string or None, got {type(v).__name__}") + return v + + +def _check_resumes(v): + if isinstance(v, bool) or not isinstance(v, int) or v < 0: + raise ValueError(f"auto_resumes must be an int >= 0, got {v!r}") + return v + + +def register(path, *, in_flight_run=..., train_pid=..., ckpt_path=..., + resume_cmd=..., auto_resumes=..., ts: str | None = None) -> dict: + """Set the recovery-critical FLAT in-flight fields (§C5) atomically, under + the lock. This is the sanctioned writer for what /ablation-runner records at + launch and what boot_resume.sh re-adopts after a reboot — it exists so those + callers stop open-coding their own load->mutate->save of loop_state.json (an + open-coded write skips the .bak snapshot, the parent-dir fsync and the lock, + i.e. exactly the durability this module owns). + + Only the fields you pass are touched (the `...` sentinel means "leave + unchanged"); pass None to CLEAR one — a finished run has no in-flight + pid/ckpt. The stage is never moved: registering a trainer is not a state + transition (use advance() for that).""" + updates = {} + if in_flight_run is not ...: + updates["in_flight_run"] = _check_run_id(in_flight_run) + if train_pid is not ...: + updates["train_pid"] = _check_pid(train_pid) + if ckpt_path is not ...: + updates["ckpt_path"] = _check_str_or_path("ckpt_path", ckpt_path) + if resume_cmd is not ...: + updates["resume_cmd"] = _check_str_or_path("resume_cmd", resume_cmd) + if auto_resumes is not ...: + updates["auto_resumes"] = _check_resumes(auto_resumes) + with locked(path): + st = load(path) + st.update(updates) + if ts is not None: + st["updated"] = ts + save(path, st) return st @@ -124,27 +295,31 @@ def resume_point(path) -> str: def record_resume(path, ts: str | None = None, cap: int = MAX_AUTO_RESUMES) -> dict: """Account one dead-run recovery attempt. While auto_resumes < cap, increment and decide `resume`; once the cap is reached, decide `crashed` (no further - auto-resume) WITHOUT incrementing past the cap.""" - st = load(path) - used = int(st.get("auto_resumes", 0)) - if used < cap: - st["auto_resumes"] = used + 1 - decision = "resume" - else: - decision = "crashed" - if ts is not None: - st["updated"] = ts - save(path, st) + auto-resume) WITHOUT incrementing past the cap. The whole read-modify-write + is locked: two recovery paths racing must not BOTH read `auto_resumes=1` + and both decide `resume` (that is how a resume loop re-crashes the box).""" + with locked(path): + st = load(path) + used = int(st.get("auto_resumes", 0)) + if used < cap: + st["auto_resumes"] = used + 1 + decision = "resume" + else: + decision = "crashed" + if ts is not None: + st["updated"] = ts + save(path, st) return {"decision": decision, "auto_resumes": st["auto_resumes"], "cap": cap} def reset_resumes(path, ts: str | None = None) -> dict: """A clean completed iteration clears the resume counter for the next one.""" - st = load(path) - st["auto_resumes"] = 0 - if ts is not None: - st["updated"] = ts - save(path, st) + with locked(path): + st = load(path) + st["auto_resumes"] = 0 + if ts is not None: + st["updated"] = ts + save(path, st) return st @@ -157,6 +332,20 @@ def main(argv=None) -> int: a_ = sub.add_parser("advance"); a_.add_argument("stage"); a_.add_argument("--ts", default=None) sub.add_parser("resume-point") rr = sub.add_parser("record-resume"); rr.add_argument("--ts", default=None) + # `register` is the shell-side entry point for the §C5 flat in-flight fields + # (the sanctioned replacement for a caller's own load-mutate-save heredoc). + rg = sub.add_parser("register", help="set the flat in-flight recovery fields (§C5)") + rg.add_argument("--in-flight-run", default=None) + rg.add_argument("--train-pid", type=int, default=None) + rg.add_argument("--ckpt-path", default=None) + rg.add_argument("--resume-cmd", default=None) + rg.add_argument("--auto-resumes", type=int, default=None) + rg.add_argument("--ts", default=None) + rg.add_argument("--clear", action="append", default=[], + choices=sorted(IN_FLIGHT_FIELDS), + help="explicitly null a field (a finished run clears these); " + "repeatable. Omitted fields are left unchanged. " + "(auto_resumes is a counter — zero it with --auto-resumes 0.)") a = ap.parse_args(argv) if a.cmd == "show": print(json.dumps(load(a.path), indent=2)) @@ -166,6 +355,28 @@ def main(argv=None) -> int: print(resume_point(a.path)) elif a.cmd == "record-resume": print(json.dumps(record_resume(a.path, a.ts), indent=2)) + elif a.cmd == "register": + given = {"in_flight_run": a.in_flight_run, "train_pid": a.train_pid, + "ckpt_path": a.ckpt_path, "resume_cmd": a.resume_cmd, + "auto_resumes": a.auto_resumes} + kw = {k: v for k, v in given.items() if v is not None} + for f in a.clear: + if f in kw: + print(f"error: --clear {f} conflicts with the value given for it", + file=sys.stderr) + return 2 + kw[f] = None + if not kw: + print("error: register needs at least one field to set or --clear", + file=sys.stderr) + return 2 + try: + st = register(a.path, ts=a.ts, **kw) + except ValueError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + print(json.dumps({k: st.get(k) for k in + IN_FLIGHT_FIELDS + ("auto_resumes", "updated")}, indent=2)) return 0 diff --git a/research/tests/test_adopt_run.py b/research/tests/test_adopt_run.py new file mode 100644 index 0000000..989d524 --- /dev/null +++ b/research/tests/test_adopt_run.py @@ -0,0 +1,474 @@ +"""Tests for research/adopt_run.py — the adopted-run protocol (upgrade-plan #10). + +The defect being guarded: the out-of-loop launch is the DOMINANT mode on this box +(~24 hand-driven runs vs ~2 through /ablation-runner), and it used to leave a +hand-written c5 file, a hand-added ledger entry and a hand-mutated loop_state.json +behind. Every test below runs against tmp_path fixtures — the real ledger.json and +loop_state.json are never read or written here (nor by adopt_run itself: it goes +through ledger.py's CLI and loop_state.register()). +""" +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import adopt_run +import ledger +import loop_state as ls + +RUN_ID = "2026-07-23_testmodel_adopt-me" +TECH = "adopt-me" + + +# ------------------------------------------------------------------ fixtures + +def _complete_evidence(**over): + """A §C5-complete evidence dict in the numbered styling the live HybridSSM + c5_evidence.json uses (verified against + HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence.json).""" + e = {"run_id": RUN_ID, "model_dir": "TestModel", "objective": "pretrain-ablation", + "framework": "jax", "technique_slug": TECH, "lifecycle_stage": "architecture", + "c5_0_smoke": {"result": "pass", "detail": "1 step, exit 0"}, + "c5_1_concurrency": {"result": "pass"}, + "c5_2_budget": {"tokens_total": 42_000_000, "source": "brief"}, + "c5_3_probe": {"tokens_per_sec": 2863, "peak_mem_gb": 16.6, "fits": True}, + "c5_4_eta_hours": 4.5, + "c5_5_resume": {"result": "pass"}, + "c5_6_sentinel": {"result": "armed"}, + "c5_7_guards": {"result": "verified"}} + e.update(over) + return e + + +@pytest.fixture +def run_dir(tmp_path): + """/experiments// carrying a complete c5_evidence.json.""" + d = tmp_path / "TestModel" / "experiments" / RUN_ID + d.mkdir(parents=True) + (d / "c5_evidence.json").write_text(json.dumps(_complete_evidence(), indent=2)) + return d + + +@pytest.fixture +def seeded_ledger(ledger_path): + """The conftest ledger + the technique entry, because §C8 referential integrity + hard-fails an `ablation` run whose technique_slug has no technique entry.""" + ledger.main(["add-technique", "--slug", TECH, "--title", "Adopt Me", + "--ledger", str(ledger_path)]) + return ledger_path + + +@pytest.fixture +def state_path(tmp_path): + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + return p + + +@pytest.fixture +def digests(tmp_path): + return tmp_path / "digests" + + +def _runs(ledger_path): + return json.loads(ledger_path.read_text())["runs"] + + +def _entry(ledger_path, run_id=RUN_ID): + return next((r for r in _runs(ledger_path) if r["run_id"] == run_id), None) + + +def _dead_pid(): + """A pid that is definitely gone: spawn, wait, reap. Not a zombie (we waited), + so pid_alive must read it as dead.""" + p = subprocess.Popen([sys.executable, "-c", "pass"]) + p.wait() + return p.pid + + +def _adopt(run_dir, seeded_ledger, state_path, digests, **kw): + return adopt_run.adopt(run_dir, ledger_path=seeded_ledger, state_path=state_path, + digest_dir=digests, ts="2026-07-23T18:00:00+01:00", **kw) + + +# ------------------------------------------------------------------ vocabulary + +def test_terminal_statuses_are_ledger_vocabulary(): + """§C8 is the authority on run status/type words. If ledger.py ever renames one, + this fails HERE instead of at 3am inside a reconcile that then writes a status + the ledger rejects.""" + assert adopt_run.RUN_STATUS == ledger.RUN_STATUS + assert adopt_run.RUN_TYPES == ledger.RUN_TYPES + assert adopt_run.TERMINAL_IF_FINISHED in ledger.RUN_STATUS + assert adopt_run.TERMINAL_IF_DIED in ledger.RUN_STATUS + assert set(adopt_run.IN_FLIGHT_STATUSES) <= ledger.RUN_STATUS + assert set(adopt_run.OBJECTIVE_TO_RUN_TYPE) == ledger.RUN_OBJECTIVES + assert set(adopt_run.OBJECTIVE_TO_RUN_TYPE.values()) <= ledger.RUN_TYPES + + +# ------------------------------------------------------------------ pid liveness + +def test_pid_alive_reads_reality(): + assert adopt_run.pid_alive(os.getpid()) is True + assert adopt_run.pid_alive(_dead_pid()) is False + + +# ------------------------------------------------------------------ adopt: refusals + +def test_adopt_refuses_incomplete_c5_and_writes_nothing( + run_dir, seeded_ledger, state_path, digests): + """No evidence, no adoption — and a refusal must be TOTAL: an incomplete c5 file + must not leave a half-adopted run (entry but no digest, or a pointer with no entry).""" + bad = _complete_evidence() + del bad["c5_3_probe"] + (run_dir / "c5_evidence.json").write_text(json.dumps(bad)) + + r = _adopt(run_dir, seeded_ledger, state_path, digests, apply=True) + + assert r["ok"] is False + assert r["c5"]["missing_items"] == ["c5.3_probe"] + assert any("§C5 lint FAILED" in e for e in r["errors"]) + assert _runs(seeded_ledger) == [] + assert not digests.exists() + assert ls.load(state_path)["in_flight_run"] is None + + +def test_adopt_refuses_run_id_dir_mismatch(run_dir, seeded_ledger, state_path, digests): + """The evidence's run_id IS the artifacts pointer; a mismatch would file the + entry against the wrong directory.""" + (run_dir / "c5_evidence.json").write_text( + json.dumps(_complete_evidence(run_id="2026-07-23_othermodel_other-run"))) + r = _adopt(run_dir, seeded_ledger, state_path, digests, apply=True) + assert r["ok"] is False and "refusing" in r["errors"][0] + assert _runs(seeded_ledger) == [] + # --force is the documented override + r = _adopt(run_dir, seeded_ledger, state_path, digests, apply=True, force=True) + assert r["ok"] is True + assert _entry(seeded_ledger, "2026-07-23_othermodel_other-run") is not None + + +def test_adopt_refuses_a_dead_train_pid(run_dir, seeded_ledger, state_path, digests): + """Registering a dead pid as in-flight manufactures exactly the stale pointer + `reconcile` exists to clean up.""" + r = _adopt(run_dir, seeded_ledger, state_path, digests, + train_pid=_dead_pid(), apply=True) + assert r["ok"] is False + assert "not alive" in r["errors"][0] and "reconcile" in r["errors"][0] + assert _runs(seeded_ledger) == [] + assert ls.load(state_path)["in_flight_run"] is None + + +def test_adopt_refuses_to_strand_another_in_flight_run( + run_dir, seeded_ledger, state_path, digests): + ls.register(state_path, in_flight_run="2026-07-21_other_run", train_pid=4242) + r = _adopt(run_dir, seeded_ledger, state_path, digests, + train_pid=os.getpid(), apply=True) + assert r["ok"] is False and "stranded" in r["errors"][0] + assert ls.load(state_path)["in_flight_run"] == "2026-07-21_other_run" + assert _runs(seeded_ledger) == [] + + +def test_adopt_refuses_unknown_run_type(run_dir, seeded_ledger, state_path, digests): + (run_dir / "c5_evidence.json").write_text( + json.dumps(_complete_evidence(objective="sideways"))) + r = _adopt(run_dir, seeded_ledger, state_path, digests, apply=True) + assert r["ok"] is False and "run type" in r["errors"][0] + assert _runs(seeded_ledger) == [] + + +def test_adopt_does_not_register_a_pointer_when_the_ledger_write_fails( + run_dir, tmp_path, state_path, digests): + """Ordering guarantee: the loop_state pointer must never name a run the ledger + does not know. A missing ledger makes ledger.py exit non-zero.""" + missing = tmp_path / "no_such_ledger.json" + r = adopt_run.adopt(run_dir, ledger_path=missing, state_path=state_path, + digest_dir=digests, train_pid=os.getpid(), apply=True) + assert r["ok"] is False and r["errors"] + assert ls.load(state_path)["in_flight_run"] is None + assert not digests.exists() + + +# ------------------------------------------------------------------ adopt: dry run + +def test_adopt_dry_run_mutates_nothing(run_dir, seeded_ledger, state_path, digests): + r = _adopt(run_dir, seeded_ledger, state_path, digests, train_pid=os.getpid()) + + assert r["ok"] is True and r["dry_run"] is True + actions = {a["step"]: a["action"] for a in r["actions"]} + assert actions == {"ledger": "would-add-run", "digest": "would-write", + "loop_state": "would-register"} + assert _runs(seeded_ledger) == [] + assert not digests.exists() + assert ls.load(state_path)["in_flight_run"] is None + + +# ------------------------------------------------------------------ adopt: apply + +def test_adopt_apply_writes_entry_digest_and_pointer( + run_dir, seeded_ledger, state_path, digests): + pid = os.getpid() + resume = "setsid nohup bash run_arch_ladder.sh >/dev/null 2>&1 it died + assert e["ended"] == "2026-07-23" + assert e["verdict"] is None # THE assertion: no verdict was invented + assert e["metrics"] == {} + assert e["reconciled"]["dead_train_pid"] == in_flight + assert e["reconciled"]["verdict_untouched"] is True + assert json.loads(seeded_ledger.read_text())["never_repeat"] == [] + + st = ls.load(state_path) + assert st["in_flight_run"] is None and st["train_pid"] is None + # the relaunch form is PRESERVED by default (clearing in_flight_run already + # disarms boot_resume.sh, and a human may still want the exact command) + assert st["resume_cmd"] == "setsid nohup bash run.sh &" + assert st["ckpt_path"] == "ckpt.pkl" + + +def test_reconcile_marks_done_when_the_run_actually_finished( + in_flight, run_dir, seeded_ledger, state_path): + """The terminal status is READ off disk, not assumed: a verdict.json means the + run reached its end, so `done` — and the verdict field is STILL not touched.""" + (run_dir / "verdict.json").write_text(json.dumps({"verdict": "null"})) + r = _reconcile(seeded_ledger, state_path, apply=True) + e = _entry(seeded_ledger) + assert e["status"] == "done" + assert e["verdict"] is None, "reconcile must never copy a verdict.json into the ledger" + assert "verdict.json present" in r["actions"][1]["basis"] + + +def test_reconcile_clear_resume_drops_the_relaunch_form( + in_flight, seeded_ledger, state_path): + _reconcile(seeded_ledger, state_path, apply=True, clear_resume=True) + st = ls.load(state_path) + assert st["resume_cmd"] is None and st["ckpt_path"] is None + + +def test_reconcile_stale_unrecorded_clears_the_pointer_without_inventing_an_entry( + seeded_ledger, state_path): + """The out-of-loop pathology in its purest form: loop_state points at a run the + ledger has never heard of. Clear the pointer; do NOT fabricate a run entry.""" + ls.register(state_path, in_flight_run="2026-07-21_ghost_never-recorded", + train_pid=_dead_pid()) + r = _reconcile(seeded_ledger, state_path, apply=True) + assert r["state"] == "stale-unrecorded" + assert _runs(seeded_ledger) == [] + assert ls.load(state_path)["in_flight_run"] is None + + +def test_reconcile_does_not_re_close_a_terminal_entry( + in_flight, seeded_ledger, state_path): + ledger.main(["update-run", RUN_ID, "--set", 'status="done"', + "--set", 'verdict="promising"', "--ledger", str(seeded_ledger)]) + r = _reconcile(seeded_ledger, state_path, apply=True) + actions = {a["step"]: a["action"] for a in r["actions"]} + assert actions["ledger"] == "already-terminal" + e = _entry(seeded_ledger) + assert e["status"] == "done" and e["verdict"] == "promising" # untouched + assert ls.load(state_path)["in_flight_run"] is None # pointer still cleared + + +# ------------------------------------------------------------------ CLI contract + +def test_cli_exit_codes(run_dir, seeded_ledger, state_path, digests): + """0 ok / 1 refused / 4 stale-detected-in-dry-run (mirrors sentinel liveness).""" + base = ["--ledger", str(seeded_ledger), "--state", str(state_path)] + assert adopt_run.main(["adopt", str(run_dir), "--digests", str(digests)] + base) == 0 + assert adopt_run.main(["reconcile"] + base) == 0 # nothing in flight -> clean + + ls.register(state_path, in_flight_run=RUN_ID, train_pid=_dead_pid()) + assert adopt_run.main(["reconcile"] + base) == 4 # detected, not applied + assert adopt_run.main(["reconcile", "--apply"] + base) == 0 + assert adopt_run.main(["reconcile"] + base) == 0 # now clean + + (run_dir / "c5_evidence.json").write_text("{ not json") + assert adopt_run.main(["adopt", str(run_dir), "--digests", str(digests)] + base) == 1 + + +def test_cli_help_lists_both_verbs(capsys): + with pytest.raises(SystemExit): + adopt_run.main(["--help"]) + out = capsys.readouterr().out + assert "adopt" in out and "reconcile" in out + + +def test_module_never_names_the_live_state_files_for_writing(): + """Guard the guard (§C8/§C11): adopt_run must reach ledger.json only through + ledger.py's CLI and loop_state.json only through loop_state.register(). The + literal filenames may appear as DEFAULT PATHS, but never in an open().""" + src = Path(adopt_run.__file__).read_text() + assert "open(" not in src.replace('open(f"/proc/', "") # only the /proc pid probe + assert src.count("write_text") == 1 # the digest stub is the ONE file it writes + assert "stub.write_text(text)" in src + assert "json.dump(" not in src # dumps-to-argv only; never dump-to-file + assert "loop_state.register(" in src # the sanctioned loop_state setter + assert "[sys.executable, str(LEDGER_PY)]" in src # the sanctioned ledger writer diff --git a/research/tests/test_calibration_pairs.py b/research/tests/test_calibration_pairs.py new file mode 100644 index 0000000..cc4d479 --- /dev/null +++ b/research/tests/test_calibration_pairs.py @@ -0,0 +1,205 @@ +"""Regression tests for research/calibration_pairs.py — the (predicted, +realised) join behind §C15.3.8 scorer calibration. + +The join is the half that can lie: it decides which outcomes count as realised, +what a JSON-null verdict means versus the verdict WORD "null", and how many +pairs a multi-run cohort contributes. Each of those is pinned below. + +Every test builds its own in-memory/temp ledger; the real ledger is never read +or written (the one test that touches a file asserts the module never writes). +""" +import hashlib +import json + +import calibration_pairs as cp +import scorer_calibration + + +def led(techniques=(), runs=()): + """A minimal ledger dict — collect() reads only techniques[] and runs[].""" + return {"schema_version": 1, "techniques": list(techniques), + "runs": list(runs), "proposals": [], "never_repeat": []} + + +def tech(slug, pwp=None, status="candidate", run_ids=()): + return {"slug": slug, "title": slug, "status": status, + "predicted_win_prob": pwp, "run_ids": list(run_ids)} + + +def run(run_id, verdict, status="done"): + return {"run_id": run_id, "type": "ablation", "status": status, + "verdict": verdict, "metrics": {}} + + +# ------------------------------------------------------------------ the join + +def test_empty_ledger_yields_nothing(): + out = cp.collect(led()) + assert out == {"pairs": [], "pending": [], "hygiene_gaps": [], "n": 0} + + +def test_win_scores_one_and_traces_to_its_run(): + out = cp.collect(led([tech("t", 0.7, "done", ["2026-07-01_m_t"])], + [run("2026-07-01_m_t", "win")])) + assert out["n"] == 1 + p = out["pairs"][0] + assert p["predicted_win_prob"] == 0.7 and p["outcome"] == 1 + assert p["technique"] == "t" and p["run_ids"] == ["2026-07-01_m_t"] + + +def test_every_non_win_verdict_scores_zero(): + """Only a VERIFIED win earns the scorer credit (§C17/§C19). The §C25.3 split + words (null/promising) and the deprecated `directional` are all realised 0s.""" + for i, verdict in enumerate(("loss", "inconclusive", "null", "promising", + "directional")): + rid = f"2026-07-0{i + 1}_m_t" + out = cp.collect(led([tech(f"t{i}", 0.6, "done", [rid])], + [run(rid, verdict)])) + assert out["n"] == 1, verdict + assert out["pairs"][0]["outcome"] == 0, verdict + + +def test_json_null_verdict_is_pending_not_a_zero(): + """The trap this module exists to not fall into: an unjudged run must be + EXCLUDED, not scored 0 — scoring it would punish the scorer for a run that + has not finished. Distinct from the verdict WORD "null", asserted above.""" + out = cp.collect(led([tech("t", 0.9, "running", ["2026-07-21_m_t"])], + [run("2026-07-21_m_t", None, status="launched")])) + assert out["n"] == 0 and out["pairs"] == [] + assert out["pending"] == [{"technique": "t", "run_id": "2026-07-21_m_t", + "predicted_win_prob": 0.9, + "run_status": "launched"}] + + +def test_verdict_word_null_and_json_null_do_not_collide(): + """Both in ONE ledger: the word null is a realised 0, JSON null is pending.""" + out = cp.collect(led([tech("measured", 0.5, "done", ["2026-07-01_m_a"]), + tech("inflight", 0.5, "running", ["2026-07-21_m_b"])], + [run("2026-07-01_m_a", "null"), + run("2026-07-21_m_b", None)])) + assert out["n"] == 1 and out["pairs"][0]["technique"] == "measured" + assert [p["technique"] for p in out["pending"]] == ["inflight"] + + +def test_technique_granularity_is_one_pair_per_idea(): + """Default granularity: the scorer predicted once, so it is graded once — + a 3-run cohort must not contribute 3 correlated pairs and treble n.""" + t = tech("t", 0.8, "done", ["2026-07-01_m_t", "2026-07-02_m_t", "2026-07-03_m_t"]) + rs = [run("2026-07-01_m_t", "loss"), run("2026-07-02_m_t", "win"), + run("2026-07-03_m_t", "null")] + agg = cp.collect(led([t], rs)) + assert agg["n"] == 1 + assert agg["pairs"][0]["outcome"] == 1 # any realised win -> 1 + assert agg["pairs"][0]["verdicts"] == ["loss", "win", "null"] + per_run = cp.collect(led([t], rs), per_run=True) + assert per_run["n"] == 3 + assert [p["outcome"] for p in per_run["pairs"]] == [0, 1, 0] + + +def test_aggregate_is_zero_when_no_run_won(): + out = cp.collect(led([tech("t", 0.8, "done", ["2026-07-01_m_t", "2026-07-02_m_t"])], + [run("2026-07-01_m_t", "loss"), + run("2026-07-02_m_t", "promising")])) + assert out["n"] == 1 and out["pairs"][0]["outcome"] == 0 + + +def test_pending_runs_do_not_block_a_realised_sibling(): + out = cp.collect(led([tech("t", 0.4, "running", ["2026-07-01_m_t", "2026-07-21_m_t"])], + [run("2026-07-01_m_t", "win"), + run("2026-07-21_m_t", None, status="running")])) + assert out["n"] == 1 and out["pairs"][0]["run_ids"] == ["2026-07-01_m_t"] + assert len(out["pending"]) == 1 + + +# ------------------------------------------------------------ hygiene checks + +def test_briefed_without_prediction_is_a_hygiene_gap(): + """§C15.3.8 requires the field from `briefed` onward — the gap is REPORTED, + never silently dropped, which is the whole reason today's n is 0.""" + out = cp.collect(led([tech("b", None, "briefed"), tech("d", None, "done"), + tech("q", None, "queued"), tech("r", None, "running")])) + assert {g["technique"] for g in out["hygiene_gaps"]} == {"b", "d", "q", "r"} + assert {g["kind"] for g in out["hygiene_gaps"]} == {"briefed_without_prediction"} + + +def test_candidate_without_prediction_is_not_a_gap(): + """A candidate has not been briefed yet, so no prediction is owed.""" + out = cp.collect(led([tech("c", None, "candidate"), + tech("p", None, "proposal"), + tech("x", None, "rejected")])) + assert out["hygiene_gaps"] == [] + + +def test_prediction_without_runs_is_a_gap(): + out = cp.collect(led([tech("t", 0.6, "briefed", [])])) + assert out["n"] == 0 + assert [g["kind"] for g in out["hygiene_gaps"]] == ["prediction_without_run"] + + +def test_dangling_run_id_is_a_gap_not_a_crash(): + out = cp.collect(led([tech("t", 0.6, "done", ["2026-07-01_m_missing"])])) + assert out["n"] == 0 + g = out["hygiene_gaps"][0] + assert g["kind"] == "run_id_not_found" and g["run_id"] == "2026-07-01_m_missing" + + +def test_done_technique_with_no_verdict_anywhere_is_a_gap(): + out = cp.collect(led([tech("t", 0.6, "done", ["2026-07-01_m_t"])], + [run("2026-07-01_m_t", None)])) + assert out["n"] == 0 + assert [g["kind"] for g in out["hygiene_gaps"]] == ["done_without_realised_verdict"] + + +# ------------------------------------------- hand-off contract + read-only-ness + +def test_scorer_inputs_feed_scorer_calibration_report(): + """The point of the module: its output must drop straight into the §C15.3.8 + math with no adapter. `report(**scorer_inputs(...))` is the whole report half.""" + out = cp.collect(led( + [tech(f"t{i}", p, "done", [f"2026-07-{i + 1:02d}_m_t"]) + for i, p in enumerate((0.9, 0.8, 0.2, 0.1))], + [run("2026-07-01_m_t", "win"), run("2026-07-02_m_t", "win"), + run("2026-07-03_m_t", "loss"), run("2026-07-04_m_t", "loss")])) + kw = cp.scorer_inputs(out) + assert kw == {"preds": [0.9, 0.8, 0.2, 0.1], "outcomes": [1, 1, 0, 0]} + rep = scorer_calibration.report(**kw) # the one-liner, exercised + assert rep["n"] == 4 + # a scorer this well-ordered has a small Brier; pin the exact value so a + # silent re-mapping of outcomes cannot pass this test. + assert round(rep["brier"], 4) == round(((0.1 ** 2) * 2 + (0.2 ** 2) * 2) / 4, 4) + + +def test_empty_join_would_raise_in_the_math_which_is_why_we_stop_short(): + """Documents the guard that makes an n=0 calibration report impossible: + scorer_calibration refuses an empty pair list, so calibration_pairs must be + read for its hygiene gaps, not asked for an ECE.""" + kw = cp.scorer_inputs(cp.collect(led())) + assert kw == {"preds": [], "outcomes": []} + assert cp.below_calibration_floor(0) and cp.below_calibration_floor(9) + assert not cp.below_calibration_floor(cp.MIN_PAIRS_FOR_CALIBRATION) + try: + scorer_calibration.report(**kw) + except ValueError as e: + assert "at least one" in str(e) + else: + raise AssertionError("scorer_calibration.report accepted an empty pair list") + + +def test_module_never_writes_the_ledger(tmp_path): + p = tmp_path / "ledger.json" + p.write_text(json.dumps(led([tech("t", 0.5, "done", ["2026-07-01_m_t"])], + [run("2026-07-01_m_t", "win")]), indent=2)) + before = hashlib.md5(p.read_bytes()).hexdigest() + assert cp.main(["--ledger", str(p)]) == 0 + assert cp.main(["--ledger", str(p), "--per-run"]) == 0 + assert cp.scorer_inputs(ledger=p)["outcomes"] == [1] + assert hashlib.md5(p.read_bytes()).hexdigest() == before + + +def test_cli_prints_the_four_keys(tmp_path, capsys): + p = tmp_path / "ledger.json" + p.write_text(json.dumps(led([tech("t", None, "briefed")]))) + assert cp.main(["--ledger", str(p)]) == 0 + out = json.loads(capsys.readouterr().out) + assert set(out) == {"pairs", "pending", "hygiene_gaps", "n"} + assert out["n"] == 0 and len(out["hygiene_gaps"]) == 1 diff --git a/research/tests/test_eval_completeness.py b/research/tests/test_eval_completeness.py index 8e7b992..996fab3 100644 --- a/research/tests/test_eval_completeness.py +++ b/research/tests/test_eval_completeness.py @@ -1,5 +1,6 @@ """§C25 eval-completeness gate tests — pure CPU, stdlib.""" import sys, pathlib +import pytest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) import eval_completeness as EC @@ -18,20 +19,110 @@ def test_full_battery_complete_per_stage(): assert r["complete"] and r["verdict_cap"] is None, st -def test_missing_hard_caps_to_directional(): +def test_missing_hard_caps_below_win(): + # §C25.3: a missing HARD item caps the run at `promising` — the strongest verdict an + # incomplete battery may carry. (Was the single word `directional` before the + # 2026-07-22 vocabulary split; the cap itself is unchanged.) for st, spec in EC.REGISTRY.items(): if not spec["required"]: continue r = EC.check_completeness(st, spec["required"][1:]) # drop first required - assert not r["complete"] and r["verdict_cap"] == "directional", st + assert not r["complete"] and r["verdict_cap"] == "promising", st + assert r["verdict_cap"] in EC.NEUTRAL_VERDICTS assert spec["required"][0] in r["missing_hard"] +def test_hard_incomplete_never_yields_win(): + # the cap, exhaustively: no stage × no significance verdict may win with a HARD item absent + for st, spec in EC.REGISTRY.items(): + if not spec["required"]: + continue + partial = spec["required"][1:] + for sig in sorted(EC.ACCEPTED_SIGNIFICANCE): + g = EC.gate_verdict(st, partial, sig) + assert g["verdict"] != "win", (st, sig) + assert g["verdict"] in EC.NEUTRAL_VERDICTS, (st, sig, g["verdict"]) + assert "incomplete-eval" in g["why"], (st, sig) + + +def test_null_and_promising_do_not_collapse(): + # the whole point of the 2026-07-22 split: under the SAME incomplete battery, + # "found something, one gate short" and "found nothing" must not wear the same word. + partial = EC.REGISTRY["sft"]["required"][1:] + effect = EC.gate_verdict("sft", partial, "win")["verdict"] + no_effect = EC.gate_verdict("sft", partial, "null")["verdict"] + assert effect == "promising" + assert no_effect == "null" + assert effect != no_effect + + +def test_incomplete_loss_is_not_burned_as_never_repeat(): + # §C25.3: the cap "is NOT a never_repeat loss". ledger.py auto-appends never_repeat[] on + # verdict=loss, so an incomplete battery must never emit one. + partial = EC.REGISTRY["preference"]["required"][1:] + g = EC.gate_verdict("preference", partial, "loss") + assert g["verdict"] == "inconclusive" and g["verdict"] != "loss" + assert g["verdict"] in EC.NEUTRAL_VERDICTS + assert "never_repeat" in g["why"] + + +def test_deprecated_directional_is_never_emitted(): + assert "directional" in EC.DEPRECATED_VERDICTS + seen = set() + for st, spec in EC.REGISTRY.items(): + req = spec["required"] + for items in (req, req[1:], [], ["valppl_n1_stage_headline"]): + for sig in sorted(EC.ACCEPTED_SIGNIFICANCE): + seen.add(EC.gate_verdict(st, items, sig)["verdict"]) + assert "directional" not in seen, seen + assert seen <= (EC.VERDICTS - EC.DEPRECATED_VERDICTS), seen + + +def test_vocabulary_matches_the_ledger(): + # the NEUTRAL_VERDICTS reader: this gate speaks the ledger's vocabulary (§C8 authority), + # and the two must not drift apart. + sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "ledger")) + import ledger + assert EC.VERDICTS == frozenset(ledger.VERDICTS) + assert EC.NEUTRAL_VERDICTS == frozenset(ledger.NEUTRAL_VERDICTS) + assert "win" not in EC.NEUTRAL_VERDICTS and "loss" not in EC.NEUTRAL_VERDICTS + assert set(EC.CAP_WHEN_INCOMPLETE.values()) <= EC.NEUTRAL_VERDICTS + assert set(EC.CAP_WHEN_INCOMPLETE) == EC.ACCEPTED_SIGNIFICANCE # every input has a cap + + +def test_cap_postcondition_raises_on_vocabulary_drift(monkeypatch): + # what makes NEUTRAL_VERDICTS load-bearing: if the cap table is ever edited so an + # incomplete battery maps onto `win`, gate_verdict refuses instead of writing it. + monkeypatch.setitem(EC.CAP_WHEN_INCOMPLETE, "win", "win") + partial = EC.REGISTRY["sft"]["required"][1:] + with pytest.raises(ValueError, match="cap violated"): + EC.gate_verdict("sft", partial, "win") + + def test_founding_mistake_is_blocked(): - # a base-eval / pretrain headline on ONLY n=1 val PPL must never be a win + # a base-eval headline on ONLY n=1 val PPL: the sole signal is disallowed as a headline + # (§C25.7.3), so no admissible effect measurement exists — not even `promising`. g = EC.gate_verdict("base-eval", ["valppl_n1_stage_headline"], "win") - assert g["verdict"] == "directional" - assert "incomplete-eval" in g["why"] + assert g["verdict"] == "inconclusive" + assert "incomplete-eval" in g["why"] and "valppl_n1_stage_headline" in g["why"] + + +def test_founding_mistake_not_bypassed_by_a_second_item(): + # Regression (2026-07-23): the disallowed-sole-signal floor once fired only when EXACTLY one + # item was present (`len(present) == 1`), so pairing the n=1 val-PPL headline with ANY second + # item — even a report-only figure — let a confounded run reach `promising`/`win`. A figure is + # not an admissible effect measurement, so the run must still floor to `inconclusive`. + g = EC.gate_verdict("base-eval", ["valppl_n1_stage_headline", "figure"], "win") + assert g["verdict"] == "inconclusive", g + assert g["completeness"]["disallowed_sole_signal"] == ["valppl_n1_stage_headline"] + assert g["completeness"]["verdict_cap"] == "inconclusive" # ceiling matches the actual downgrade + # A REAL admissible signal alongside it clears the sole-signal floor: only the HARD-battery cap + # remains, so a real (but incomplete) effect is `promising` — never floored for sole-ness. + real_item = EC.REGISTRY["base-eval"]["required"][0] + assert real_item not in EC.DISALLOWED_SOLE_SIGNAL + g2 = EC.gate_verdict("base-eval", ["valppl_n1_stage_headline", real_item], "win") + assert not g2["completeness"]["disallowed_sole_signal"], g2 + assert g2["verdict"] == "promising", g2 def test_conditional_only_fires_when_active(): @@ -42,10 +133,30 @@ def test_conditional_only_fires_when_active(): def test_unknown_stage_cannot_win(): - assert EC.gate_verdict("nope", ["a", "b"], "win")["verdict"] == "inconclusive" + g = EC.gate_verdict("nope", ["a", "b"], "win") + assert g["verdict"] == "inconclusive" + assert "unknown lifecycle_stage" in g["why"] def test_complete_and_significant_is_win(): full = EC.REGISTRY["serving"]["required"] assert EC.gate_verdict("serving", full, "win")["verdict"] == "win" assert EC.gate_verdict("serving", full, "loss")["verdict"] == "loss" + # a complete battery is what earns the right to a first-class negative / capped call + assert EC.gate_verdict("serving", full, "null")["verdict"] == "null" + assert EC.gate_verdict("serving", full, "promising")["verdict"] == "promising" + + +def test_unreadable_significance_fails_closed(): + full = EC.REGISTRY["serving"]["required"] + for junk in (None, "", "directional", "WIN", "banana"): + g = EC.gate_verdict("serving", full, junk) + assert g["verdict"] == "inconclusive", junk + assert g["significance_verdict"] == junk # raw input kept for provenance + assert g["significance_read_as"] == "inconclusive" + + +def test_every_result_stamps_the_vocabulary(): + known = EC.check_completeness("systems", EC.REGISTRY["systems"]["required"]) + unknown = EC.check_completeness("frobnicate", ["x"]) + assert known["verdict_vocab"] == unknown["verdict_vocab"] == EC.VERDICT_VOCAB diff --git a/research/tests/test_guards.py b/research/tests/test_guards.py index 614765a..572a462 100644 --- a/research/tests/test_guards.py +++ b/research/tests/test_guards.py @@ -4,12 +4,25 @@ composition, and the PREALLOCATE=true refusal — is the first line of defense against the unified-memory over-allocation that hard-crashed the box on 2026-06-08. -All tests here are pure CPU: the validation/env logic runs BEFORE any torch/jax call, -so no GPU (and no torch/jax install) is needed to exercise the safety-critical paths. +Item 16's remainder (added 2026-07-23, the day the box hard-locked at 15:24 and the +ladder went unrecovered) covers the two *shell* killers that were named but untested: + * run_arch_ladder.sh's trainer_alive() — the §C4.5 one-GPU-job-at-a-time gate, whose + argv[0]-must-be-python check exists because a bare pgrep matched greps/editors/agent + sessions and wrongly deferred all 15 cells on 2026-07-21 22:03; + * the recovery chain liveness_cron.sh -> cron_runner.sh — sentinel's exit-4 routing and + the flock that stops a resume overlapping the nightly fire. + +All tests here are pure CPU and hermetic: the shell scripts are driven from tmp_path with +fake sentinel/claude/cron_runner stand-ins, so no trainer, no GPU work, no real ledger and +no real loop_state are ever touched. Fixture processes are `sleep`-alikes only. """ +import contextlib +import fcntl import importlib import os +import subprocess import sys +import time from pathlib import Path import pytest @@ -89,3 +102,309 @@ def test_jax_safe_env_refuses_prealloc_true(): else: os.environ[k] = v sys.modules.pop("jax_safe_env", None) + + +# =========================================================== trainer_alive() (§C4.5) +# The arch-ladder driver's one-GPU-job-at-a-time gate. A FALSE POSITIVE stalls the +# ladder (2026-07-21: all 15 cells deferred by a pgrep that matched a grep); a FALSE +# NEGATIVE lets two trainers share the ~119 GB pool and OOM-kills the box. Tests drive +# the SHIPPED function text, extracted verbatim from run_arch_ladder.sh — never a +# reimplementation — against controlled sleeping fixtures. No fixture does GPU work. + +LADDER_DRIVER = (ROOT / "HybridSSM-0.2B" / "experiments" + / "2026-07-21_hybrid-ssm-0.2b_arch-ladder" / "run_arch_ladder.sh") +PGREP_PATTERN = r"train_[A-Za-z0-9_]*\.py" # the pattern the shipped function pgreps on + + +def _extract_shell_function(script: Path, name: str) -> str: + """Return the verbatim `name () { ... }` block from a bash script. + + Fails loudly rather than silently yielding an empty body: if the driver renames or + reshapes the guard, every test below must break instead of passing vacuously. + """ + lines = script.read_text().splitlines() + opens = [i for i, ln in enumerate(lines) if ln.startswith(f"{name} () {{")] + assert len(opens) == 1, f"{name}() not found exactly once in {script}" + i = opens[0] + ends = [j for j in range(i + 1, len(lines)) if lines[j] == "}"] + assert ends, f"unterminated {name}() in {script}" + body = "\n".join(lines[i:ends[0] + 1]) + assert PGREP_PATTERN in body, "shipped pgrep pattern changed — fixture names are stale" + return body + + +def _trainer_alive_probe(tmp_path: Path, name: str = "probe.sh") -> Path: + """A standalone script that sources the shipped trainer_alive() and exits with its + status (0 = a real trainer is alive). `set -uo pipefail` mirrors the driver.""" + (tmp_path / "fn.sh").write_text(_extract_shell_function(LADDER_DRIVER, "trainer_alive") + "\n") + probe = tmp_path / name + probe.write_text('set -uo pipefail\nsource "$(dirname "$0")/fn.sh"\ntrainer_alive\n') + return probe + + +def _probe_rc(probe: Path) -> int: + return subprocess.run(["bash", str(probe)], capture_output=True, text=True, timeout=60).returncode + + +def _require_idle_box(probe: Path): + """Every case below is a transition test from 'no trainer'. If a real trainer is live + on this box, skip rather than report a meaningless result.""" + if _probe_rc(probe) == 0: + pytest.skip("a real trainer is alive on this box — cannot test the idle transition") + + +@pytest.fixture +def spawn(): + """Start throwaway fixture processes and guarantee they die with the test.""" + started = [] + + def _spawn(argv): + p = subprocess.Popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + started.append(p) + deadline = time.time() + 10 + while time.time() < deadline: + out = subprocess.run(["pgrep", "-f", PGREP_PATTERN], + capture_output=True, text=True).stdout.split() + if str(p.pid) in out: + return p + time.sleep(0.05) + pytest.fail(f"fixture pid {p.pid} never matched {PGREP_PATTERN}") + + yield _spawn + for p in started: + p.kill() + p.wait(timeout=10) + + +def test_trainer_alive_is_false_on_an_idle_box(tmp_path): + """Baseline: with no trainer running the gate must NOT defer — otherwise the ladder + never launches a cell (the 2026-07-21 stall).""" + assert _probe_rc(_trainer_alive_probe(tmp_path)) == 1 + + +def test_trainer_alive_detects_a_real_python_trainer(tmp_path, spawn): + """True positive: a genuine `python3 .../train_*.py` must be seen, or §C4.5 is broken + and two trainers can share the unified pool.""" + probe = _trainer_alive_probe(tmp_path) + _require_idle_box(probe) + trainer = tmp_path / "train_fixture.py" + trainer.write_text("import time\ntime.sleep(120)\n") # sleeps only — no GPU, no model + spawn([sys.executable, str(trainer)]) + assert _probe_rc(probe) == 0 + + +def test_trainer_alive_detects_a_versioned_interpreter(tmp_path, spawn): + """The check is `case basename in python*)`, not an exact `python3` — a trainer started + as python3.x still counts. (The driver's own sentinel re-pgrep uses `python[0-9.]*`.)""" + probe = _trainer_alive_probe(tmp_path) + _require_idle_box(probe) + interp = tmp_path / "python3.99" + interp.symlink_to(sys.executable) + trainer = tmp_path / "train_versioned.py" + trainer.write_text("import time\ntime.sleep(120)\n") + spawn([str(interp), str(trainer)]) + assert _probe_rc(probe) == 0 + + +def test_trainer_alive_ignores_a_shell_that_merely_mentions_a_trainer(tmp_path, spawn): + """THE 2026-07-21 22:03 REGRESSION: a shell/grep/agent session whose cmdline contains + train_*.py is matched by `pgrep -f` but its argv[0] is a shell, not a python + interpreter. Counting it deferred all 15 cells while sentinel reported trainers=none.""" + probe = _trainer_alive_probe(tmp_path) + _require_idle_box(probe) + named = tmp_path / "train_hybrid.py" + named.write_text("# not executed by this test\n") + # trailing `:` keeps bash resident: with a lone final simple command bash execs it and + # the cmdline (and the false positive we are testing for) disappears. + spawn(["bash", "-c", f"grep -c . {named} >/dev/null; sleep 120; :"]) + assert _probe_rc(probe) == 1 + + +def test_trainer_alive_ignores_a_monitoring_command_on_the_trainer_file(tmp_path, spawn): + """Same class, non-shell argv[0]: a `tail -f train_hybrid.py` (or an editor holding the + file open) mentions the trainer but is not one.""" + probe = _trainer_alive_probe(tmp_path) + _require_idle_box(probe) + watched = tmp_path / "train_hybrid.py" + watched.write_text("# not executed by this test\n") + spawn(["tail", "-f", str(watched)]) + assert _probe_rc(probe) == 1 + + +def test_trainer_alive_never_counts_itself(tmp_path): + """The `[ "$p" = "$$" ] && continue` line. Run the probe so that it is itself matched by + the pgrep AND has a python argv[0] (`exec -a python3 bash `): + without the self-skip the driver would see its own pid and defer forever. Verified + non-vacuous — deleting that line from the extracted body flips this rc to 0.""" + probe = _trainer_alive_probe(tmp_path, name="train_selfprobe.py") + r = subprocess.run(["bash", "-c", 'exec -a python3 bash "$1"', "_", str(probe)], + capture_output=True, text=True, timeout=60) + assert r.returncode == 1, r.stdout + r.stderr + + +# ================================================= recovery chain (§C6): exit 4 -> resume +# liveness_cron.sh routes ONLY sentinel's exit 4 ("in-flight run's PID is dead") to +# cron_runner.sh "/research-loop resume"; cron_runner.sh's flock stops that resume +# overlapping a live session. On 2026-07-23 this chain was never armed (no BuildFromScratch +# crontab lines) so the dead ladder went unrecovered — the scripts must at least be correct +# for when they ARE armed. Both scripts hardcode their absolute paths, so each test copies +# the shipped text and substitutes ONLY those constants (count==1 asserted, so a drift in +# the script breaks the test loudly instead of testing a stale string). Nothing real is +# invoked: sentinel, claude and (where not under test) cron_runner are fakes in tmp_path. + +LIVENESS_CRON = ROOT / "research" / "liveness_cron.sh" +CRON_RUNNER = ROOT / "research" / "cron_runner.sh" +REPO_CONST = "REPO=/home/yashb98/Downloads/BuildFromScratch" +CLAUDE_CONST = "CLAUDE=/home/yashb98/.local/bin/claude" +LOCK_CONST = "LOCK=/tmp/research-loop.lock" + + +def _repoint(script: Path, dest: Path, subs): + """Copy a shipped script to dest with ONLY the given constant lines rewritten.""" + text = script.read_text() + for old, new in subs: + assert text.count(old) == 1, f"expected exactly one {old!r} in {script}" + text = text.replace(old, new) + dest.write_text(text) + dest.chmod(0o755) + return dest + + +def _recorder(path: Path, record: Path) -> Path: + """An executable stand-in that appends its argv to `record` and exits 0.""" + path.write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" >> "{record}"\nexit 0\n') + path.chmod(0o755) + return path + + +def _fake_repo(tmp_path, liveness_rc, real_cron_runner=False): + """A fake REPO tree for liveness_cron.sh: a sentinel.py that exits `liveness_rc`, and a + cron_runner.sh that is either a recorder or the SHIPPED script (repointed at a fake + claude). Returns (patched liveness_cron, cron_record, claude_record, lock).""" + repo = tmp_path / "repo" + (repo / "research").mkdir(parents=True) + (repo / "sentinel.py").write_text( + "import os, sys\n" + "print('[fake-sentinel]', sys.argv[1:])\n" + "sys.exit(int(os.environ['FAKE_LIVENESS_RC']))\n") + cron_record = tmp_path / "cron_runner_argv.txt" + claude_record = tmp_path / "claude_argv.txt" + lock = tmp_path / "research-loop.lock" + if real_cron_runner: + _recorder(tmp_path / "claude", claude_record) + _repoint(CRON_RUNNER, repo / "research" / "cron_runner.sh", + [(REPO_CONST, f"REPO={repo}"), + (CLAUDE_CONST, f"CLAUDE={tmp_path / 'claude'}"), + (LOCK_CONST, f"LOCK={lock}")]) + else: + _recorder(repo / "research" / "cron_runner.sh", cron_record) + patched = _repoint(LIVENESS_CRON, tmp_path / "liveness_cron.sh", + [(REPO_CONST, f"REPO={repo}")]) + return patched, cron_record, claude_record, lock + + +def _run_liveness(patched, rc, *args): + env = dict(os.environ, FAKE_LIVENESS_RC=str(rc)) + return subprocess.run(["bash", str(patched), *args], + capture_output=True, text=True, timeout=120, env=env) + + +@contextlib.contextmanager +def _hold_lock(lock: Path): + """Hold cron_runner's flock from this process, exactly as a live session would.""" + fd = os.open(str(lock), os.O_CREAT | os.O_WRONLY, 0o644) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + + +def test_liveness_cron_routes_exit4_to_a_resume(tmp_path): + """sentinel exit 4 = an in-flight run whose PID is dead. This is the ONLY code that + turns that signal into recovery; if it stops firing, a dead run stays dead (2026-07-23).""" + patched, cron_record, _, _ = _fake_repo(tmp_path, 4) + r = _run_liveness(patched, 4) + assert r.returncode == 0, r.stderr + assert cron_record.read_text().splitlines() == ["/research-loop resume"] + + +@pytest.mark.parametrize("rc", [0, 1, 2, 3, 5]) +def test_liveness_cron_launches_nothing_except_on_exit4(tmp_path, rc): + """rc 0 = idle or running fine; anything else = probe glitch. Both must fail OPEN: a + spurious session could relaunch on top of a healthy run and break §C4.5.""" + patched, cron_record, _, _ = _fake_repo(tmp_path, rc) + r = _run_liveness(patched, rc) + assert r.returncode == 0, r.stderr + assert not cron_record.exists() + + +def test_liveness_cron_logs_the_probe_result(tmp_path): + """The cron is unattended: its only observability is research/cron_logs/liveness.log.""" + patched, _, _, _ = _fake_repo(tmp_path, 0) + _run_liveness(patched, 0) + log = (tmp_path / "repo" / "research" / "cron_logs" / "liveness.log").read_text() + assert "liveness rc=0 -> no action" in log + + +def test_liveness_cron_does_not_hang_on_a_bad_settle_arg(tmp_path): + """The @reboot cron passes a settle delay (180). A non-numeric arg must not become + `sleep abc` or an infinite wait — the reboot recovery would never fire.""" + patched, cron_record, _, _ = _fake_repo(tmp_path, 4) + started = time.time() + r = _run_liveness(patched, 4, "abc") + assert r.returncode == 0, r.stderr + assert time.time() - started < 30 + assert cron_record.read_text().splitlines() == ["/research-loop resume"] + + +def test_recovery_chain_reaches_the_cli_with_a_resume_prompt(tmp_path): + """End-to-end through the SHIPPED cron_runner.sh: exit 4 must land as + `claude -p "/research-loop resume"`. The CLI itself is a recorder — no session, no + trainer, no GPU. (The real relaunch decision happens inside that session's S1/§C5.)""" + patched, _, claude_record, _ = _fake_repo(tmp_path, 4, real_cron_runner=True) + r = _run_liveness(patched, 4) + assert r.returncode == 0, r.stderr + assert claude_record.read_text().splitlines() == [ + "-p", "/research-loop resume", "--dangerously-skip-permissions"] + + +def test_flock_blocks_a_second_concurrent_session(tmp_path): + """The flock is what keeps the 30-min liveness cron, the @reboot cron and the nightly + fire from running three Claude sessions at once against one ledger and one GPU.""" + _, _, claude_record, lock = _fake_repo(tmp_path, 4, real_cron_runner=True) + runner = tmp_path / "repo" / "research" / "cron_runner.sh" + with _hold_lock(lock): + r = subprocess.run([str(runner), "/research-loop"], capture_output=True, + text=True, timeout=120) + assert r.returncode == 0, r.stderr # skipping a fire is not an error + assert not claude_record.exists() + logs = list((tmp_path / "repo" / "research" / "cron_logs").glob("*.log")) + assert any("skipping this fire" in p.read_text() for p in logs) + + +def test_flock_is_released_so_the_next_fire_runs(tmp_path): + """A lock that is never released would silently retire the loop — the failure mode is + indistinguishable from 'nothing to do', so assert the fire after the holder exits.""" + _, _, claude_record, lock = _fake_repo(tmp_path, 4, real_cron_runner=True) + runner = tmp_path / "repo" / "research" / "cron_runner.sh" + with _hold_lock(lock): + subprocess.run([str(runner), "/research-loop"], capture_output=True, + text=True, timeout=120) + r = subprocess.run([str(runner), "/research-loop"], capture_output=True, + text=True, timeout=120) + assert r.returncode == 0, r.stderr + assert claude_record.read_text().splitlines() == [ + "-p", "/research-loop", "--dangerously-skip-permissions"] + + +def test_recovery_resume_cannot_overlap_a_live_session(tmp_path): + """The composite §C6 property: even when liveness says 'dead run, recover', the resume + must be swallowed by the flock while a session still holds it — otherwise a slow + nightly session plus a 30-min liveness fire could launch two loops (and two trainers).""" + patched, _, claude_record, lock = _fake_repo(tmp_path, 4, real_cron_runner=True) + with _hold_lock(lock): + r = _run_liveness(patched, 4) + assert r.returncode == 0, r.stderr + assert not claude_record.exists() diff --git a/research/tests/test_ledger.py b/research/tests/test_ledger.py index fa3edfb..8bd0488 100644 --- a/research/tests/test_ledger.py +++ b/research/tests/test_ledger.py @@ -422,6 +422,170 @@ def test_split_verdict_vocabulary(ledger_path): assert "t" in reload(ledger_path)["never_repeat"] +# ------------------------------------------- run-key hygiene + fsck (§C8/§C10) +# 2026-07-23: an audit of the live ledger found 24 runs carrying 60+ ad-hoc +# top-level keys — among them the PPLs of four 2026-06-16 eval runs sitting +# beside an EMPTY metrics{}, invisible to everything that compares runs. `--set` +# accepts any key by design (that is how additive contract keys land), so the +# fix is an advisory, not a rejection: the existing ledger must stay writable. + + +def stderr_of(capsys): + return capsys.readouterr().err + + +def warnings_of(capsys): + """The warn() lines as text — one JSON object per line on stderr.""" + return [json.loads(ln)["warning"] for ln in stderr_of(capsys).splitlines() if ln] + + +def test_unknown_top_level_run_key_warns_but_succeeds(ledger_path, capsys): + assert run(ledger_path, "add-run", "--run-id", "2026-07-23_m_t", "--type", "eval", + "--set", "train_pid=1234") == 0 + err = stderr_of(capsys) + assert "unrecognized top-level keys" in err and "train_pid" in err + assert reload(ledger_path)["runs"][0]["train_pid"] == 1234 # stored, not dropped + + +def test_eval_shaped_key_gets_its_own_louder_warning(ledger_path, capsys): + assert run(ledger_path, "add-run", "--run-id", "2026-07-23_m_t", "--type", "eval", + "--set", "wikitext2_ppl=37.01", "--set", "train_pid=1") == 0 + # two SEPARATE advisories, each naming only its own keys — the eval one must + # not be diluted into the generic "we don't know this key" line. + ws = warnings_of(capsys) + evalish = [w for w in ws if "EVAL-SHAPED top-level keys" in w] + other = [w for w in ws if "unrecognized top-level keys" in w] + assert len(evalish) == 1 and len(other) == 1, ws + assert "wikitext2_ppl" in evalish[0] and "train_pid" not in evalish[0] + assert "train_pid" in other[0] and "wikitext2_ppl" not in other[0] + assert "metrics{}" in evalish[0] and "fsck --fix" in evalish[0] + + +def test_schema_and_contract_keys_do_not_warn(ledger_path, capsys): + """No crying wolf: the §C8 core + the additive keys the contracts name + (§C18 confound_check, §C25.1 lifecycle_stage, §C11 launched_by, §C20 + is_remote, §C5.2 prior_run_id/note) must pass silently.""" + assert run(ledger_path, "add-run", "--run-id", "2026-07-23_m_t", "--type", "eval", + "--set", "lifecycle_stage=base-eval", "--set", 'launched_by="manual"', + "--set", "is_remote=false", "--set", 'note="one line"', + "--set", 'prior_run_id="2026-07-22_m_t"', + "--set", 'metrics={"ppl":1.0}') == 0 + assert stderr_of(capsys) == "" + + +def test_update_run_warns_on_each_ad_hoc_set(ledger_path, capsys): + run(ledger_path, "add-run", "--run-id", "2026-07-23_m_t", "--type", "eval") + capsys.readouterr() + assert run(ledger_path, "update-run", "2026-07-23_m_t", "--set", "arm=x") == 0 + assert "arm" in stderr_of(capsys) + # re-setting the SAME ad-hoc key warns again — the surface is still growing + assert run(ledger_path, "update-run", "2026-07-23_m_t", "--set", "arm=y") == 0 + assert "arm" in stderr_of(capsys) + + +def test_is_eval_key_is_narrow_enough_to_move_on(): + # fsck --fix MOVES what this matches, so a false positive would relocate prose. + for k in ("wikitext2_ppl", "code_floor_abs", "final_val_loss", "suite_version", + "self_floor", "headline_bpb", "task_acc", "delta_ci95", "code_corpus_id"): + assert ledger.is_eval_key(k), k + for k in ("note", "train_pid", "arm_plan", "headline", "guards", "purpose", + "evidence_path", "resume_cmd", "steps", "design"): + assert not ledger.is_eval_key(k), k + + +def _dirty_ledger(ledger_path, repo_root): + """One run with a WRITTEN detail doc, one with a dangling pointer, and stray + eval keys beside an empty metrics{} — the live ledger's three defects.""" + run(ledger_path, "add-run", "--run-id", "2026-07-23_m_kept", "--type", "eval") + run(ledger_path, "add-run", "--run-id", "2026-07-23_m_gone", "--type", "eval", + "--set", "wikitext2_ppl=37.01", "--set", 'suite_version="text-lm-v2"', + "--set", "train_pid=99") + doc = repo_root / "research" / "ledger" / "runs" + doc.mkdir(parents=True, exist_ok=True) + (doc / "2026-07-23_m_kept.md").write_text("# real detail doc\n") + + +def test_fsck_reports_without_fixing(ledger_path, tmp_path, capsys): + _dirty_ledger(ledger_path, tmp_path) + before = hashlib.md5(ledger_path.read_bytes()).hexdigest() + capsys.readouterr() + assert run(ledger_path, "fsck", "--repo-root", str(tmp_path)) == 1 # repairable + out = json.loads(capsys.readouterr().out) + assert [d["run_id"] for d in out["dangling_detail_md"]] == ["2026-07-23_m_gone"] + assert {s["key"] for s in out["eval_keys_at_top_level"]} == {"wikitext2_ppl", + "suite_version"} + assert out["other_unknown_run_keys"] == [{"run_id": "2026-07-23_m_gone", + "keys": ["train_pid"]}] + assert out["fixed"] is None + assert hashlib.md5(ledger_path.read_bytes()).hexdigest() == before # read-only + + +def test_fsck_fix_nulls_only_the_dangling_pointer(ledger_path, tmp_path, capsys): + """The honest repair: a pointer to a document that was never written becomes + null. It is NOT back-filled from the entry — a generated stub would carry + nothing the entry does not already say while looking like a real write-up.""" + _dirty_ledger(ledger_path, tmp_path) + capsys.readouterr() + assert run(ledger_path, "fsck", "--fix", "--repo-root", str(tmp_path)) == 0 + runs = {r["run_id"]: r for r in reload(ledger_path)["runs"]} + assert runs["2026-07-23_m_gone"]["detail_md"] is None + assert runs["2026-07-23_m_kept"]["detail_md"] == \ + "research/ledger/runs/2026-07-23_m_kept.md" + assert not (tmp_path / "research/ledger/runs/2026-07-23_m_gone.md").exists() + + +def test_fsck_fix_moves_eval_keys_into_metrics_preserving_values(ledger_path, + tmp_path): + _dirty_ledger(ledger_path, tmp_path) + assert run(ledger_path, "fsck", "--fix", "--repo-root", str(tmp_path)) == 0 + r = {x["run_id"]: x for x in reload(ledger_path)["runs"]}["2026-07-23_m_gone"] + assert r["metrics"] == {"wikitext2_ppl": 37.01, "suite_version": "text-lm-v2"} + assert "wikitext2_ppl" not in r and "suite_version" not in r + assert r["train_pid"] == 99 # non-eval ad-hoc key is reported, never moved + + +def test_fsck_fix_is_idempotent(ledger_path, tmp_path, capsys): + _dirty_ledger(ledger_path, tmp_path) + run(ledger_path, "fsck", "--fix", "--repo-root", str(tmp_path)) + after_first = ledger_path.read_bytes() + capsys.readouterr() + assert run(ledger_path, "fsck", "--fix", "--repo-root", str(tmp_path)) == 0 + out = json.loads(capsys.readouterr().out) + assert out["fixed"] == {"detail_md_nulled": [], "eval_keys_moved": [], + "skipped_collision": []} + assert ledger_path.read_bytes() == after_first # byte-identical re-run + + +def test_fsck_refuses_to_clobber_a_metrics_collision(ledger_path, tmp_path, capsys): + """Same name, two DIFFERENT values: a machine must not pick. Report and skip.""" + run(ledger_path, "add-run", "--run-id", "2026-07-23_m_c", "--type", "eval", + "--set", 'metrics={"wikitext2_ppl":37.01}', "--set", "wikitext2_ppl=99.9") + capsys.readouterr() + assert run(ledger_path, "fsck", "--fix", "--repo-root", str(tmp_path)) == 1 + out = json.loads(capsys.readouterr().out) + assert out["fixed"]["skipped_collision"] == ["2026-07-23_m_c.wikitext2_ppl"] + r = reload(ledger_path)["runs"][0] + assert r["metrics"]["wikitext2_ppl"] == 37.01 and r["wikitext2_ppl"] == 99.9 + + +def test_fsck_moves_an_identical_duplicate(ledger_path, tmp_path): + """Same name, SAME value = no information at stake; the top-level copy goes.""" + run(ledger_path, "add-run", "--run-id", "2026-07-23_m_d", "--type", "eval", + "--set", 'metrics={"self_floor":true}', "--set", "self_floor=true") + assert run(ledger_path, "fsck", "--fix", "--repo-root", str(tmp_path)) == 0 + r = reload(ledger_path)["runs"][0] + assert r["metrics"]["self_floor"] is True and "self_floor" not in r + + +def test_fsck_clean_ledger_exits_zero(ledger_path, tmp_path, capsys): + run(ledger_path, "add-run", "--run-id", "2026-07-23_m_t", "--type", "eval", + "--set", "detail_md=null") + capsys.readouterr() + assert run(ledger_path, "fsck", "--repo-root", str(tmp_path)) == 0 + out = json.loads(capsys.readouterr().out) + assert out["dangling_detail_md"] == [] and out["eval_keys_at_top_level"] == [] + + def test_the_run_type_lint_accepts_a_valid_caller(tmp_path): """No false positives: the FIXED argv shape must not be flagged. Mirrors the real score_ladder.py, which reaches the CLI through a `LEDGER = ROOT / ".../ledger.py"` diff --git a/research/tests/test_orchestration_chaos.py b/research/tests/test_orchestration_chaos.py index 7cb9e1f..9444a20 100644 --- a/research/tests/test_orchestration_chaos.py +++ b/research/tests/test_orchestration_chaos.py @@ -2,6 +2,14 @@ resume / dead-run / auto-resume-cap / fail-open behaviour from spec into execution-verified assertions (§C22, §C4 recovery). Stdlib-only (CI-safe).""" import json +import os +import stat +import subprocess +import sys +import time +from pathlib import Path + +import pytest import loop_state as ls @@ -120,3 +128,243 @@ def test_bad_stage_rejected_on_write(tmp_path): bad = ls.default_state(); bad["stage"] = "ZZ" with pytest.raises(ValueError): ls.save(p, bad) + + +# -------------------------------------------------------------------------- +# Crash-durability parity with ledger.py (§C8): the parent-dir fsync, the +# mode-preserving replace, and the advisory lock. loop_state.json is THE +# recovery-critical file — after a GB10 hard-lock it is what boot_resume.sh +# reads to know a run was in flight, so a write that survives only in the +# page cache loses the run. +# -------------------------------------------------------------------------- + +def test_save_fsyncs_the_parent_directory(tmp_path, monkeypatch): + """A crash between os.replace and the directory-metadata flush can bring the + state file back absent/truncated even though its CONTENTS were fsynced. Spy + on os.fsync and assert at least one call targeted a DIRECTORY fd.""" + p = tmp_path / "loop_state.json" + real_fsync = os.fsync + synced = {"file": 0, "dir": 0} + + def spy(fd): + try: + synced["dir" if stat.S_ISDIR(os.fstat(fd).st_mode) else "file"] += 1 + except OSError: # never let the spy break the write + pass + return real_fsync(fd) + + monkeypatch.setattr(os, "fsync", spy) + ls.save(p, ls.default_state()) + assert synced["file"] >= 1, "save() never fsynced the file contents" + assert synced["dir"] >= 1, "save() never fsynced the PARENT DIR (rename not durable)" + + +def test_save_preserves_the_state_file_mode(tmp_path): + """mkstemp creates 0600 and os.replace carries that onto the target, so + without an explicit chmod one write silently makes the state unreadable to + the other sanctioned writers (loop / @reboot hook / liveness cron).""" + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + assert stat.S_IMODE(p.stat().st_mode) == 0o644 # first creation, not 0600 + p.chmod(0o640) + ls.advance(p, "S4") + assert stat.S_IMODE(p.stat().st_mode) == 0o640 # preserved across the replace + + +# A concurrent writer: load -> (widened window) -> save, all inside the module's +# advisory lock, recording its own critical-section [enter, exit] window. +# CLOCK_MONOTONIC is system-wide on Linux, so windows from separate processes +# are directly comparable. +_WRITER = r''' +import json, sys, time +import loop_state as ls + +state_path, out_path, start_at = sys.argv[1], sys.argv[2], float(sys.argv[3]) +while time.monotonic() < start_at: # release every writer at the same instant + time.sleep(0.001) +with ls.locked(state_path): + enter = time.monotonic() + st = ls.load(state_path) + n = st["auto_resumes"] + time.sleep(0.05) # widen the read->write window + st["auto_resumes"] = n + 1 + ls.save(state_path, st) + left = time.monotonic() +with open(out_path, "w") as f: + json.dump([enter, left], f) +''' + + +def test_lock_serialises_concurrent_writers(tmp_path): + """Multiple writers are real here — boot_resume.sh, liveness_cron.sh and the + loop itself all write this file. Four processes each increment auto_resumes + under the lock; assert (a) no lost update and (b) no two critical sections + overlapped. Unlocked, all four would read the same value and the counter + would land at 1.""" + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + env = dict(os.environ, PYTHONPATH=str(Path(ls.__file__).resolve().parent)) + n_writers = 4 + start_at = time.monotonic() + 1.0 + procs = [] + for i in range(n_writers): + out = tmp_path / f"window_{i}.json" + procs.append((out, subprocess.Popen( + [sys.executable, "-c", _WRITER, str(p), str(out), repr(start_at)], + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True))) + windows = [] + for out, proc in procs: + _, err = proc.communicate(timeout=120) + assert proc.returncode == 0, f"writer failed: {err}" + windows.append(json.loads(out.read_text())) + + assert ls.load(p)["auto_resumes"] == n_writers, "lost update — writers raced" + windows.sort() + for (_, earlier_exit), (later_enter, _) in zip(windows, windows[1:]): + assert earlier_exit <= later_enter, "critical sections overlapped" + assert (tmp_path / "loop_state.json.lock").exists() + + +def test_lock_is_reentrant_within_one_process(tmp_path): + """flock is per open-file-description, so a nested os.open + LOCK_EX would + deadlock against ourselves. Timed, because a self-deadlock degrades to the + bounded LOCK_TIMEOUT_S fail-open wait rather than an outright hang: finishing + fast IS the assertion.""" + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + t0 = time.monotonic() + with ls.locked(p): + with ls.locked(p): + ls.advance(p, "S6", ts="2026-07-23") # a mutator that locks again + ls.register(p, train_pid=4242) + elapsed = time.monotonic() - t0 + assert elapsed < ls.LOCK_TIMEOUT_S / 2, f"nested lock stalled for {elapsed:.1f}s" + st = ls.load(p) + assert st["stage"] == "S6" and st["train_pid"] == 4242 + assert ls._LOCKS == {}, "lock fd not released when the outermost block exited" + + +def test_acquire_lock_fails_open_when_contended(tmp_path): + """A held lock must never brick a writer forever: with the wait exhausted, + acquire_lock returns None (proceed unlocked) instead of blocking. A lost + update is recoverable; a hung @reboot hook forfeits the once-per-boot + recovery entirely.""" + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + held = ls.acquire_lock(p) + assert held is not None + try: + assert ls.acquire_lock(p, timeout=0.0) is None + finally: + ls.release_lock(held) + fd = ls.acquire_lock(p, timeout=0.0) + assert fd is not None # released -> free again + ls.release_lock(fd) + + +# -------------------------------------------------------------------------- +# register(): the sanctioned setter for the §C5 flat in-flight fields, so +# boot_resume.sh / ablation-runner stop open-coding load-mutate-save (which +# skips the .bak, the parent-dir fsync and the lock). +# -------------------------------------------------------------------------- + +def test_register_roundtrips_the_recovery_fields(tmp_path): + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + ls.register(p, in_flight_run="2026-07-23_qwen3_x", train_pid=4242, + ckpt_path="/x/exp/ckpt_step_900.pt", + resume_cmd="bash run_arms.sh --resume", auto_resumes=0, + ts="2026-07-23") + st = ls.load(p) + assert st["in_flight_run"] == "2026-07-23_qwen3_x" + assert st["train_pid"] == 4242 + assert st["ckpt_path"] == "/x/exp/ckpt_step_900.pt" + assert st["resume_cmd"] == "bash run_arms.sh --resume" + assert st["auto_resumes"] == 0 and st["updated"] == "2026-07-23" + assert st["stage"] == "S0" # registering a trainer is NOT a transition + assert list(tmp_path.glob(".loopstate_*.tmp")) == [] + + +def test_register_touches_only_the_fields_passed(tmp_path): + """boot_resume.sh's real call: write back ONLY the new trainer pid after a + relaunch, without disturbing the run id / ckpt / resume_cmd it just used.""" + p = tmp_path / "loop_state.json" + st = ls.default_state() + st.update(stage="S7", in_flight_run="2026-07-19_qwen3_x", + train_pid=111, ckpt_path="/x/ckpt.pt", resume_cmd="bash r.sh", + auto_resumes=1) + ls.save(p, st) + ls.register(p, train_pid=222) + back = ls.load(p) + assert back["train_pid"] == 222 + assert back["in_flight_run"] == "2026-07-19_qwen3_x" + assert back["ckpt_path"] == "/x/ckpt.pt" and back["resume_cmd"] == "bash r.sh" + assert back["auto_resumes"] == 1 and back["stage"] == "S7" + + +def test_register_clears_fields_when_a_run_finishes(tmp_path): + p = tmp_path / "loop_state.json" + st = ls.default_state() + st.update(in_flight_run="2026-07-19_qwen3_x", train_pid=111, + ckpt_path="/x/ckpt.pt", resume_cmd="bash r.sh") + ls.save(p, st) + ls.register(p, in_flight_run=None, train_pid=None, ckpt_path=None, + resume_cmd=None, auto_resumes=0) + back = ls.load(p) + for k in ls.IN_FLIGHT_FIELDS: + assert back[k] is None # nothing to resume + assert back["auto_resumes"] == 0 + + +def test_register_rejects_shapes_the_recovery_chain_cannot_use(tmp_path): + """boot_resume.sh does `[ -z "$RUN_ID" ]` on a stringified field and + `pgrep`/`sentinel watch --pid` on the pid — a nested object or a bogus pid + would silently break the @reboot recovery instead of failing loudly.""" + p = tmp_path / "loop_state.json" + ls.save(p, ls.default_state()) + with pytest.raises(ValueError): + ls.register(p, in_flight_run={"run_id": "x"}) # §C5: the RUN_ID string + with pytest.raises(ValueError): + ls.register(p, train_pid=0) + with pytest.raises(ValueError): + ls.register(p, train_pid=-7) + with pytest.raises(ValueError): + ls.register(p, train_pid=True) # bool is an int subclass + with pytest.raises(ValueError): + ls.register(p, ckpt_path=17) + with pytest.raises(ValueError): + ls.register(p, auto_resumes=-1) + assert ls.load(p)["train_pid"] is None # nothing was written + + +def test_register_on_a_corrupt_state_fails_open_and_repairs(tmp_path): + """The realistic post-hard-lock case: the state file came back truncated. + register must still land a usable recovery pointer (fail-open load -> + canonical schema -> durable save), not raise on the recovery path.""" + p = tmp_path / "loop_state.json" + p.write_text("{ truncated") + ls.register(p, in_flight_run="2026-07-23_qwen3_x", train_pid=9, + resume_cmd="bash r.sh") + back = ls.load(p) + assert back["_recovered"] is False # a valid file again + assert back["in_flight_run"] == "2026-07-23_qwen3_x" and back["train_pid"] == 9 + assert back["stage"] == "S0" and "_recovered" not in p.read_text() + + +def test_register_cli_is_the_replacement_for_open_coded_writes(tmp_path): + """The exact shell surface boot_resume.sh should use instead of its inline + `st = ls.load(p); st['train_pid'] = pid; ls.save(p, st)` heredoc.""" + p = str(tmp_path / "loop_state.json") + ls.save(p, ls.default_state()) + assert ls.main(["--path", p, "register", "--train-pid", "1234", + "--in-flight-run", "2026-07-23_qwen3_x", + "--resume-cmd", "bash run_arms.sh", "--ts", "2026-07-23"]) == 0 + st = ls.load(p) + assert st["train_pid"] == 1234 and st["in_flight_run"] == "2026-07-23_qwen3_x" + assert ls.main(["--path", p, "register", "--clear", "train_pid"]) == 0 + assert ls.load(p)["train_pid"] is None + # usage errors are exit 2, never a silent no-op + assert ls.main(["--path", p, "register"]) == 2 # nothing to do + assert ls.main(["--path", p, "register", "--train-pid", "5", + "--clear", "train_pid"]) == 2 # contradictory + assert ls.main(["--path", p, "register", "--train-pid", "0"]) == 2 # invalid pid diff --git a/research/tests/test_sentinel.py b/research/tests/test_sentinel.py index 4e9f6c8..db17680 100644 --- a/research/tests/test_sentinel.py +++ b/research/tests/test_sentinel.py @@ -215,6 +215,61 @@ def test_watch_escalates_to_sigkill_after_grace(tmp_path, monkeypatch): assert "survived" in log and "SIGKILL" in log # escalation path taken +def test_watch_writes_marker_before_grace_elapses(tmp_path, monkeypatch): + # Regression for the lost-marker race (2026-07-23): the kill marker MUST be written the moment + # the kill decision is final — right after SIGTERM, BEFORE the SIGTERM->SIGKILL grace loop runs. + # If it is written only AFTER the grace loop, a SIGTERM-ignoring trainer holds the marker hostage + # for the whole grace window, and a caller that reaps the sentinel right after the trainer exits + # (run_arch_ladder.sh does: `wait "$tpid"; kill "$spid"`) loses it. That is exactly why only 1 of + # that day's 2 thermal kills left a marker, and why any recovery logic counting the marker undercounts. + import threading + monkeypatch.setattr(sentinel, "meminfo", lambda: (1000, 50)) # 95% pressure -> kill on sample 1 + ready = tmp_path / "ready" + code = ("import signal,time,sys,pathlib;" + "signal.signal(signal.SIGTERM, signal.SIG_IGN);" + "pathlib.Path(sys.argv[1]).write_text('1');" + "time.sleep(60)") + child = subprocess.Popen(["python3", "-c", code, str(ready)]) + for _ in range(100): + if ready.exists(): + break + time.sleep(0.05) + assert ready.exists(), "child never installed its SIGTERM handler" + + marker = tmp_path / "kill.json" + grace = 4.0 + result = {} + + def _run(): + result["rc"] = sentinel.watch(child.pid, kill_at=0.80, log_path=str(tmp_path / "w.log"), + interval=0.05, grace=grace, marker_path=str(marker)) + + t = threading.Thread(target=_run, daemon=True) + start = time.monotonic() + t.start() + appeared_at = None + while time.monotonic() - start < grace + 3: + if marker.exists(): + appeared_at = time.monotonic() - start + break + time.sleep(0.02) + try: + assert appeared_at is not None, "marker never appeared" + # The SIGTERM-ignoring child forces the FULL grace; a correctly-ordered marker predates it + # clearly. If the write had moved back after the grace loop, appeared_at would be ~grace. + assert appeared_at < grace * 0.5, ( + f"marker appeared at {appeared_at:.2f}s, not clearly before the {grace:.0f}s grace " + f"— ordering regression: the marker is being written AFTER the grace loop") + finally: + t.join(timeout=grace + 5) + try: + child.kill() + except ProcessLookupError: + pass + child.wait() + assert result.get("rc") == 3 + + # ------------------------------------------- pgrep self-match regression (#1 MLOps blocker) # 2026-06-17 digest, Health: the GPU-free watcher's `pgrep -f 'train.*\.py'` # self-matched its OWN command line (the pattern appears in pgrep's argv and in diff --git a/sentinel.py b/sentinel.py index 0d31193..decdfa1 100644 --- a/sentinel.py +++ b/sentinel.py @@ -29,11 +29,14 @@ * P dead (or zombie, or PID recycled) -> log it, exit 0 — the normal end of a run. * pool usage (MemTotal-MemAvailable)/MemTotal >= --kill-at -> - re-verify P's identity, SIGTERM P, wait up to --grace s (default 60), - SIGKILL if still alive, write the reason + trainer RSS to the log AND - (atomically, tmp+rename) to the marker file - research/loop_state.json.sentinel_kill, exit 3. --marker overrides - the marker path for SELF-TESTS ONLY; production arms never pass it. + re-verify P's identity, SIGTERM P, write the reason + trainer RSS to + the log AND (atomically, tmp+rename) to the marker file + research/loop_state.json.sentinel_kill, THEN wait up to --grace s + (default 60) and SIGKILL if still alive, exit 3. The marker is written + BEFORE the grace loop on purpose (2026-07-23, see write_kill_marker) + and re-written after it only if the escalation learned something new; + the schema is identical either way. --marker overrides the marker path + for SELF-TESTS ONLY; production arms never pass it. Why --kill-at defaults to 0.80 ------------------------------ @@ -380,6 +383,25 @@ def hottest_soc_c(): return hottest +def write_kill_marker(marker: Path, payload: dict) -> None: + """Atomically (tmp+rename) write the §C6 kill marker. Phase 3 never sees torn JSON. + + Called BEFORE the SIGTERM->SIGKILL grace loop, and again after it only if the + escalation changed anything. 2026-07-23: the marker used to be written only after + the grace loop, and the arch-ladder driver kills its sentinel ~1 s after the trainer + exits (run_arch_ladder.sh: `wait "$tpid"; ... kill "$spid"`). That afternoon's two + thermal kills therefore left ONE marker: the 14:16:46Z kill recorded, the 14:21:37Z + kill lost (its log ends at the KILL line with no "marker written"), so anything + counting thermal events from the marker undercounts them. The kill decision is + already final when SIGTERM goes out, so the record belongs there — a sentinel that + is itself killed mid-grace has still told the truth about what it did. + """ + marker.parent.mkdir(parents=True, exist_ok=True) + tmp = marker.with_name(marker.name + ".tmp") + tmp.write_text(json.dumps(payload, indent=2) + "\n") + os.replace(tmp, marker) + + def watch(pid, kill_at, log_path, interval, grace, marker_path=None) -> int: marker = Path(marker_path) if marker_path else MARKER logf = open(log_path, "a", buffering=1) if log_path else None @@ -448,6 +470,33 @@ def log(msg): except PermissionError as e: kill_failed = f"SIGTERM denied: {e}" log(f"ERROR: {kill_failed}") + kill_time = utcnow() # pinned once: a re-write must not move the timestamp + + def payload(): + return { + "time": kill_time, + "killed_pid": pid, + "trigger": trigger, + "reason": reason, + "gpu_temp_c": gpu_t, + "soc_temp_c": soc_t, + "gpu_throttling": throttling, + "pool_usage": round(usage, 4), + "pool_total_gb": round(total_kib / 2**20, 1), + "trainer_rss_gb": ( + round(rss_gib, 2) if rss_gib is not None else None + ), + "kill_at": kill_at, + "kill_failed": kill_failed, # null on the normal path + "log": str(log_path) if log_path else None, + } + + # Record the kill NOW, before the grace loop — a caller that reaps us the + # moment the trainer dies must not be able to erase the event (see + # write_kill_marker). + write_kill_marker(marker, payload()) + log(f"marker written: {marker}") + marker_kill_failed = kill_failed deadline = time.monotonic() + grace while time.monotonic() < deadline and watched_alive(pid, start_ticks): time.sleep(1) @@ -462,28 +511,9 @@ def log(msg): log(f"ERROR: {kill_failed}") else: log(f"pid {pid} exited within grace period") - payload = { - "time": utcnow(), - "killed_pid": pid, - "trigger": trigger, - "reason": reason, - "gpu_temp_c": gpu_t, - "soc_temp_c": soc_t, - "gpu_throttling": throttling, - "pool_usage": round(usage, 4), - "pool_total_gb": round(total_kib / 2**20, 1), - "trainer_rss_gb": ( - round(rss_gib, 2) if rss_gib is not None else None - ), - "kill_at": kill_at, - "kill_failed": kill_failed, # null on the normal path - "log": str(log_path) if log_path else None, - } - marker.parent.mkdir(parents=True, exist_ok=True) - tmp = marker.with_name(marker.name + ".tmp") - tmp.write_text(json.dumps(payload, indent=2) + "\n") - os.replace(tmp, marker) # atomic: Phase 3 never sees torn JSON - log(f"marker written: {marker}") + if kill_failed != marker_kill_failed: # escalation learned something new + write_kill_marker(marker, payload()) + log(f"marker re-written ({kill_failed}): {marker}") return 3 samples += 1 if samples % 20 == 0: # heartbeat every ~10 min at default interval From 512dfd2429f84f07fd2dc051e2cbb1e78ba05d63 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 20:30:55 +0100 Subject: [PATCH 20/35] NorMuon-at-scale (#9): launch-ready extension package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the launch-ready package for upgrade-plan #9 — the flagship "does NorMuon's win persist or converge with budget?" question — WITHOUT launching (GPU spend stays human-triggered). - run_ladder_scale_ext.sh: extension driver adding the 420M 3rd seed (takes the top rung from n=2 -> n=3 paired seeds, earning a real across-seed CI) + a 840M seed pair (a higher-budget trend point). Ships the FIXED cool-down gate ported from the arch-ladder recovery (sustained-cool dwell + DEFER + honoured return, COOL_C 70->58) so it cannot re-trigger the thermal thrash that preceded today's hard-lock. Reuses train_ablation.py + the CORE .done markers (done cells skipped). - c5_evidence_scale_ext.json: structured §C5 evidence, PASSES c5_validate 7/7. ETA/probe from the real measured 420M cell (6,685 tok/s, 51.5 GB, 17.5 h/cell). - §C5.0 smoke run + verified (exit 0, model built, loss moved, checkpoint saved). Carries an explicit thermal precondition: the box hard-locked today under a lighter load and the durable firmware fix is still pending — recommend the 420M pair first, firmware/kdump fix before the 840M rung. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../c5_evidence_scale_ext.json | 81 +++++++++ .../run_ladder_scale_ext.sh | 169 ++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json create mode 100644 Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json new file mode 100644 index 0000000..a573c96 --- /dev/null +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json @@ -0,0 +1,81 @@ +{ + "run_id": "2026-07-23_qwen3-0.6b_normuon-at-scale", + "model_dir": "Qwen3-0.6B", + "lifecycle_stage": "scaling", + "objective": "pretrain-ablation", + "framework": "pytorch", + "technique_slug": "normuon-optimizer", + "parent_run_id": "2026-07-05_qwen3-0.6b_scaling-persistence", + + "purpose": "Upgrade-plan #9. Extend the CORE scaling-persistence ladder (42M+168M+420M, COMPLETE) to resolve the flagship open question: does NorMuon's +0.474 BPB win over AdamW (2D weights, fixed N=596M) PERSIST or converge as the token budget grows? The 420M rung is currently n=2 paired seeds, which the memory + IMU-1 RESULT.md flag caps the verdict at directional (§C17 wants >=3). This adds the 3rd 420M seed to earn a real across-seed CI at the top rung, plus a 840M point (n=1) to extend the gap-vs-budget trend.", + + "arm_plan": { + "trainer": "train_ablation.py (the exact IMU-1 single-variable trainer, fixed N=596M): only the 196 2D-weight updates differ between arms; embeddings/1D are AdamW wd=0 in BOTH; data split seed 0 fixed. Vary ONLY --steps and --seed.", + "arms": ["adamw @ peak_lr 2.4e-3", "normuon @ 0.011"], + "new_cells": [ + "6409 adamw 2 (420M, 3rd seed)", + "6409 normuon 2 (420M, 3rd seed)", + "12818 adamw 0 (840M, 1st seed)", + "12818 normuon 0 (840M, 1st seed)" + ], + "order": "420M-s2 pair FIRST (completes the n=3 CI at the current top rung, ~35 h), then the 840M s0 pair (~70 h). The CI-completing half is a natural stop point if the box needs the firmware fix before the longer rung.", + "reused": "42M/168M/420M(s0,s1) cells are already .done and are skipped instantly — this run adds only the 4 genuinely new cells (§C13 control-reuse)." + }, + + "iso_flop": { + "note": "This is a deliberate BUDGET SWEEP at fixed N=596M, not an iso-FLOP pair. Within a budget rung the two arms are iso-config/iso-token (only the optimizer on 2D weights differs), so the per-rung gap is a clean single-variable comparison. Across rungs the token budget is the independent variable by design.", + "open_caveat_carried": "AdamW peak_lr is held fixed across budgets (IMU-1 RESULT.md Limitation #1). A per-horizon LR check at the 840M rung would de-confound the largest point; until then the 840M gap is directional and LR-confounded." + }, + + "c5_0_smoke": { + "result": "pass", + "detail": "train_ablation.py --optimizer adamw --seed 0 --steps 1 --no_compile --tag smoke_scale_ext, run 2026-07-23 20:22-20:23 under sentinel watch on the cold idle box. safe_cuda capped CUDA at 85% (109 GB); model built; param split 196 2D->adamw | 114 rest->AdamW; 1 step moved val PPL 184184.11 -> 121008.11 (finite, trending); DONE in 0.5 min; checkpoint saved (1.19 GB) then removed; exit 0.", + "log_path": "smoke_scale_ext.log", + "src": "log" + }, + + "c5_2_budget": { + "tokens_total": 2519040000, + "breakdown": "420M-s2 pair = 2 x 420M = 840M tok; 840M-s0 pair = 2 x 840M = 1680M tok.", + "gpu_hours_est": 105.0, + "gpu_days_est": 4.4, + "source": "measured per-cell wall-time (420M normuon_s0 completed in 1047.5 min = 17.46 h at 6,685 tok/s); 840M ~= 2x.", + "src": "derived" + }, + + "c5_3_probe": { + "tokens_per_sec": 6685, + "peak_mem_gb": 51.5, + "fits": true, + "detail": "Measured from the completed 420M normuon_s0 CORE cell (6409 steps, 51.5 GB peak, 6,685 tok/s, DONE in 1047.5 min) — a far stronger probe than a few steps. Peak mem 51.5 GB is well under the 109 GB safe_cuda cap; a 840M cell is the same model at 2x steps, so memory is unchanged.", + "src": "derived" + }, + + "c5_4_eta_hours": 105.0, + + "c5_5_resume": { + "result": "pass", + "detail": "train_ablation.py checkpoints every --resume_every 200 steps (atomic + fsync) and a fresh run resumes from the checkpoint (crash-survivable upgrade, 2026-07-08; kill-9 verified). The driver re-attempts failed cells on the next pass so a thermally-killed cell resumes rather than restarting. .done markers give cell-level idempotence across reboots.", + "src": "log" + }, + + "c5_6_sentinel": { + "result": "armed per cell", + "detail": "run_ladder_scale_ext.sh arms `sentinel.py watch --kill-at 0.80` on each cell's trainer pid (0.80 < safe_cuda 0.85 < OOM cliff) plus the thermal kill-switch, and preflights before the first launch. The cool-down gate is the FIXED 2026-07-23 version: requires 6 sustained sub-58C samples (3 min dwell), DEFERS on the bounded fall-through (never 'launch anyway'), and honours its return value so the hot-spell backoff engages — the CORE run_ladder.sh gate did none of these and let a cell relaunch onto a heat-soaked box 60s after a kill.", + "src": "log" + }, + + "c5_7_guards": { + "result": "verified", + "detail": "PyTorch script: train_ablation.py imports safe_cuda before torch and calls guard(0.85) — confirmed live in the 2026-07-23 smoke (capped CUDA at 109 GB). Chunked cross-entropy for the 151,936 vocab (never materializes the (N, vocab) fp32 logits that crashed the box 2026-06-08). One GPU job at a time (§C4.5): the driver defers a pass if any python trainer is alive.", + "src": "log" + }, + + "verdict_metric": "Per cell: eval-harness text-lm-v2 BPB on wikitext-2 + a code corpus with the model's own Qwen3 tokenizer, suite_version stamped. Per rung: the AdamW-minus-NorMuon BPB gap with an across-seed CI (420M now n=3 paired seeds -> a real CI; 840M n=1 -> directional), then scaling_ladder.fit_gap_trend() over 168M/420M/840M to ask whether the gap trends to 0. A measured CONVERGENCE (the win compressing with budget) is a PASSING result, not a failure.", + + "not_claimed": "No result is claimed at queue time. The 840M point is n=1 and LR-confounded (fixed AdamW LR across horizons) -> directional at best. Achieved tok/s is reported, never %-of-peak MFU (GB10 peak estimated, §C24).", + + "thermal_precondition": "The box hard-locked 2026-07-23 15:24 BST under an 85M thermal load; NorMuon-at-scale is a heavier 596M/51.5 GB sustained load. The durable firmware fix (DGX OS OTA 7.5.0 + BIOS/BSP + crashkernel, human/sudo) is STILL pending. Recommendation: run the 420M-s2 pair first; land the firmware fix before the 840M rung.", + + "evidence_path": "Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json" +} diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh new file mode 100644 index 0000000..efefa76 --- /dev/null +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# NorMuon-at-scale extension driver — upgrade-plan #9 (research/LOOP_UPGRADE_PLAN_2026-07-22.md). +# +# Extends the CORE scaling-persistence ladder (run_ladder.sh, CORE = 42M+168M+420M, COMPLETE) with +# the two rungs that resolve the flagship open question "does NorMuon's +0.474 BPB win over AdamW +# PERSIST or converge with token budget?": +# - 420M 3rd seed (s2 pair): takes the current top rung from n=2 -> n=3 paired seeds, so the 420M +# gap finally carries a real across-seed CI (§C17) instead of directional. +# - 840M 1st seed (s0 pair): a new higher-budget point to extend the gap-vs-budget trend. +# Cells are ordered 420M-s2 FIRST so the CI-completing half lands before the (much longer) 840M rung. +# +# Same fixed N=596M trainer (train_ablation.py), same results dir, same .done markers as CORE, so any +# already-complete cell is skipped instantly. train_ablation.py has mid-run resume (--resume_every), so +# a thermally-killed cell resumes from its checkpoint on the next pass (the CORE header's "no mid-run +# resume" note is stale — the crash-survivable upgrade landed 2026-07-08). +# +# SAFETY (2026-07-23): the cool-down gate is the FIXED version. The CORE run_ladder.sh (and the +# HybridSSM arch ladder modelled on it) shipped a cool-down that accepted ONE sub-threshold sample, +# LAUNCHED ANYWAY after 30 min, and discarded its own return value — which let a cell relaunch onto a +# heat-soaked box ~60s after a thermal kill, reheat past 90C, and re-kill. That thrash preceded the +# 2026-07-23 15:24 BST hard-lock. This driver requires SUSTAINED cool (dwell), DEFERS on the bounded +# fall-through, and honours the return value so the hot-spell backoff actually engages. +# One GPU job at a time (§C4.5). Smoke-first (§C5.0). sentinel watch --kill-at 0.80 per cell (§C6). +set -u +ROOT=/home/yashb98/Downloads/BuildFromScratch +IMU1=$ROOT/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw +LDIR=$ROOT/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence +TRAIN=$IMU1/train_ablation.py +RESULTS=$IMU1/results +LOG=$LDIR/run_ladder_scale_ext.log +PY=python3 + +# COOL_C derivation (2026-07-23): sentinel kills at TEMP_KILL_C=90C on the hottest of GPU die + SoC +# zones; the measured idle->load transient is ~+31C. Starting below 90-31-1 = 58C keeps even the +# first-minute transient under the kill line. 70C (the CORE default) sat AT the box's own loaded idle +# floor (64-69C), so it gated nothing. Overridable, but do not raise it toward the kill line. +COOL_C="${LADDER_COOL_C:-58}" +COOL_DWELL="${LADDER_COOL_DWELL:-6}" # consecutive sub-COOL_C samples required (6 x 30s = 3 min sustained) +COOL_MAX="${LADDER_COOL_MAX:-30}" # bounded wait: 30 samples = 15 min, then DEFER (never "launch anyway") + +# Hottest of the GPU die + all ACPI SoC zones, whole deg C (empty if unreadable). No sudo. +hottest_c () { + local g z zc max="" + g=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -dc '0-9') + [ -n "$g" ] && max=$g + for z in /sys/class/thermal/thermal_zone*/temp; do + [ -r "$z" ] || continue + zc=$(( $(cat "$z" 2>/dev/null || echo 0) / 1000 )) + { [ -z "$max" ] || [ "$zc" -gt "$max" ]; } && max=$zc + done + echo "$max" +} + +# SUSTAINED-cool gate. Requires COOL_DWELL consecutive samples strictly below COOL_C (the streak resets +# on any hot sample), so a box that is merely dipping between load spikes cannot pass. Returns 0 only on +# a real sustained-cool window; returns 1 if still hot after the bounded wait (the caller DEFERS the cell +# to the next pass, where the hot-spell backoff engages). Unreadable temp => proceed (never fabricate heat). +cool_down () { + local tag=$1 h streak=0 i=0 + while [ "$i" -lt "$COOL_MAX" ]; do + h=$(hottest_c) + [ -z "$h" ] && { echo "[$(date '+%T')] [cooldown] $tag: temp unreadable, proceeding"; return 0; } + if [ "$h" -lt "$COOL_C" ]; then + streak=$(( streak + 1 )) + [ "$streak" -ge "$COOL_DWELL" ] && { echo "[$(date '+%T')] [cooldown] $tag: ${h}C < ${COOL_C}C sustained ${streak}x, launching"; return 0; } + else + [ "$streak" -ne 0 ] && echo "[$(date '+%T')] [cooldown] $tag: ${h}C >= ${COOL_C}C, streak reset" + streak=0 + fi + sleep 30; i=$(( i + 1 )) + done + echo "[$(date '+%T')] [cooldown] $tag: box did not hold < ${COOL_C}C for ${COOL_DWELL} samples within $(( COOL_MAX / 2 )) min — DEFER to next pass" + return 1 +} + +# EXTENSION cells: " ". 6409 steps = 420M tok (~17.5 h/cell measured), 12818 = 840M (~35 h). +# 420M-s2 pair first (completes the n=3 CI at the current top rung), then the 840M s0 pair (n=1 trend point). +CELLS=( + "6409 adamw 2" "6409 normuon 2" + "12818 adamw 0" "12818 normuon 0" +) + +run_cell () { + local steps=$1 arm=$2 seed=$3 + local budgetM=$(( steps * 65536 / 1000000 )) + local tag=persist_${budgetM}M_${arm}_s${seed} + [ -f "$LDIR/${tag}.done" ] && { echo "[$(date '+%T')] [skip] $tag"; return 0; } + # §C4.5: never two trainers — a foreign trainer means someone else owns the GPU, wait it out. + if pgrep -f 'train_[A-Za-z0-9_]*\.py' | while read -r p; do exe=$(tr '\0' '\n' < "/proc/$p/cmdline" 2>/dev/null | head -1); case "$(basename "${exe:-none}")" in python*) [ "$p" != "$$" ] && exit 0;; esac; done; [ $? -eq 0 ]; then + echo "[$(date '+%T')] [wait] $tag: another python trainer is alive, deferring this pass"; return 1 + fi + if ! cool_down "$tag"; then echo "[$(date '+%T')] [defer] $tag: box too hot"; return 1; fi + # pool headroom (unified memory shared; wait for >=60 GB free — a 596M cell runs at ~51.5 GB) + for _ in $(seq 1 90); do a=$(free -g | awk '/Mem:/{print $7}'); [ "${a:-0}" -ge 60 ] && break; sleep 10; done + echo "[$(date '+%F %T')] START $tag steps=$steps (~${budgetM}M tok)" + $PY "$TRAIN" --optimizer "$arm" --seed "$seed" --steps "$steps" --tag "$tag" --resume_every 200 \ + >> "$RESULTS/${tag}.out" 2>&1 & + local tpid=$! + $PY "$ROOT/sentinel.py" watch --pid "$tpid" --kill-at 0.80 --log "$LDIR/sentinel_${tag}.log" \ + >/dev/null 2>&1 & + local spid=$! + wait "$tpid"; local rc=$? + # Give sentinel a beat to write its kill marker before we reap it (marker-before-grace fix, but be safe). + kill "$spid" 2>/dev/null + if [ $rc -eq 0 ] && [ -f "$RESULTS/checkpoint_${tag}.pt" ]; then + touch "$LDIR/${tag}.done"; echo "[$(date '+%T')] [done] $tag"; return 0 + else + echo "[$(date '+%T')] [FAIL rc=$rc] $tag — will retry on next pass (resumes from checkpoint)"; return 1 + fi +} + +{ +echo "===== $(date '+%F %T') NorMuon-at-scale extension driver start (pid $$) =====" +# 1) no GPU trainer live +for _ in $(seq 1 360); do pgrep -f "train_grpo.py|run_phase1_passk.py|train_ablation.py|train_hybrid.py" >/dev/null 2>&1 || break; sleep 20; done +sleep 5 +# 2) preflight (§C6) +$PY "$ROOT/sentinel.py" preflight || { echo "preflight FAIL — abort"; exit 1; } +# 3) SMOKE (§C5.0): 1 step, no compile — confirms model build + data + step + checkpoint save. +echo "[$(date '+%T')] smoke: train_ablation.py --steps 1" +$PY "$TRAIN" --optimizer adamw --seed 0 --steps 1 --no_compile --tag smoke_scale_ext >> "$LDIR/smoke_scale_ext.log" 2>&1 & +smpid=$! +$PY "$ROOT/sentinel.py" watch --pid "$smpid" --kill-at 0.80 --log "$LDIR/sentinel_smoke_scale_ext.log" >/dev/null 2>&1 & +smwatch=$! +wait "$smpid"; smrc=$? +kill "$smwatch" 2>/dev/null +if [ $smrc -ne 0 ] || [ ! -f "$RESULTS/checkpoint_smoke_scale_ext.pt" ]; then + echo "SMOKE FAILED — abort (see $LDIR/smoke_scale_ext.log)"; exit 1 +fi +rm -f "$RESULTS/checkpoint_smoke_scale_ext.pt"; echo "[$(date '+%T')] smoke OK" +# 4) train the extension cells, one at a time; loop passes so a thermally-killed cell is re-attempted +# (resuming from checkpoint) instead of the driver quitting; hot-spell backoff for peak-heat windows. +pass=0; MAXPASS=100; hot_backoff=0; prev_missing=999 +while : ; do + fails=0 + for c in "${CELLS[@]}"; do run_cell $c || fails=$((fails+1)); done + missing=0 + for c in "${CELLS[@]}"; do + set -- $c; s=$1; arm=$2; seed=$3; bM=$(( s * 65536 / 1000000 )) + [ -f "$LDIR/persist_${bM}M_${arm}_s${seed}.done" ] || missing=$((missing+1)) + done + [ "$missing" -eq 0 ] && break + pass=$((pass+1)) + [ "$pass" -ge "$MAXPASS" ] && { echo "[$(date '+%T')] MAXPASS=$MAXPASS reached, $missing cells incomplete — stopping (resume ckpts preserved; re-run to continue)"; break; } + if [ "$fails" -gt 0 ] && [ "$missing" -ge "$prev_missing" ]; then + hot_backoff=$(( hot_backoff + 1 )); wait_s=$(( hot_backoff * 300 )); [ "$wait_s" -gt 1800 ] && wait_s=1800 + echo "[$(date '+%F %T')] pass $pass: $missing/${#CELLS[@]} incomplete, no progress ($fails failed — box likely too hot) — backing off ${wait_s}s" + sleep "$wait_s" + else + hot_backoff=0 + echo "[$(date '+%F %T')] pass $pass: $missing/${#CELLS[@]} cells incomplete ($fails failed) — re-attempting" + fi + prev_missing=$missing +done +# 5) completion gate: only claim done when EVERY extension cell has its marker +missing=0 +for c in "${CELLS[@]}"; do + set -- $c; s=$1; arm=$2; seed=$3; bM=$(( s * 65536 / 1000000 )) + [ -f "$LDIR/persist_${bM}M_${arm}_s${seed}.done" ] || { missing=$((missing+1)); echo " incomplete: persist_${bM}M_${arm}_s${seed}"; } +done +if [ "$missing" -eq 0 ]; then + touch "$LDIR/ladder_scale_ext.done" + echo "===== $(date '+%F %T') NORMUON-AT-SCALE EXTENSION COMPLETE — $LDIR/ladder_scale_ext.done =====" + echo "[$(date '+%T')] next: eval-harness BPB-score the new cells + scaling_ladder.fit_gap_trend over 168M/420M(n=3)/840M" +else + echo "===== $(date '+%F %T') EXTENSION INCOMPLETE — $missing/${#CELLS[@]} cells missing .done =====" + exit 1 +fi +} >> "$LOG" 2>&1 From a36e5c768aa617c8374c0e29972ee0329e7faa19 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 20:56:13 +0100 Subject: [PATCH 21/35] NorMuon-at-scale: scope this launch to the 420M-s2 pair (840M deferred behind firmware fix) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index efefa76..97ed90f 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -74,10 +74,13 @@ cool_down () { } # EXTENSION cells: " ". 6409 steps = 420M tok (~17.5 h/cell measured), 12818 = 840M (~35 h). -# 420M-s2 pair first (completes the n=3 CI at the current top rung), then the 840M s0 pair (n=1 trend point). +# 2026-07-23 LAUNCH SCOPE = the 420M-s2 pair ONLY (user-authorized): completes the n=3 across-seed CI at +# the current top rung (~35 h), then STOPS. The 840M s0 pair is DEFERRED behind the firmware/kdump fix — +# the box hard-locked today under a lighter load, so the heaviest rung waits for the durable fix + a reboot. +# To run the 840M rung later, uncomment its line below and re-launch (done cells are skipped). CELLS=( "6409 adamw 2" "6409 normuon 2" - "12818 adamw 0" "12818 normuon 0" + # "12818 adamw 0" "12818 normuon 0" # 840M — DEFERRED (enable after enable_kdump_gb10.sh + OTA 7.5.0 + reboot) ) run_cell () { From 5a9fd6c8beff0abd42ad40531da538eef7500a4a Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 21:00:23 +0100 Subject: [PATCH 22/35] Fix run_ladder_scale_ext trainer_alive: piped-while returned 0 on empty pgrep The inline check deferred every cell forever (an empty `pgrep | while` exits 0, so it read 'trainer alive' on an idle box). Replaced with the proper argv[0]-is-python function form from run_arch_ladder.sh. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index 97ed90f..68663a3 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -83,13 +83,26 @@ CELLS=( # "12818 adamw 0" "12818 normuon 0" # 840M — DEFERRED (enable after enable_kdump_gb10.sh + OTA 7.5.0 + reboot) ) +# Is a REAL foreign python trainer alive? Require argv[0] to be a python interpreter (a bare +# `pgrep -f train_*.py` self-matches greps/monitors/this driver). Never count ourselves. Returns +# 0 (true) only if some OTHER process is a genuine python trainer. +trainer_alive () { + local p exe + for p in $(pgrep -f 'train_[A-Za-z0-9_]*\.py' 2>/dev/null); do + [ "$p" = "$$" ] && continue + exe=$(tr '\0' '\n' < "/proc/$p/cmdline" 2>/dev/null | head -1) + case "$(basename "${exe:-none}")" in python*) return 0 ;; esac + done + return 1 +} + run_cell () { local steps=$1 arm=$2 seed=$3 local budgetM=$(( steps * 65536 / 1000000 )) local tag=persist_${budgetM}M_${arm}_s${seed} [ -f "$LDIR/${tag}.done" ] && { echo "[$(date '+%T')] [skip] $tag"; return 0; } # §C4.5: never two trainers — a foreign trainer means someone else owns the GPU, wait it out. - if pgrep -f 'train_[A-Za-z0-9_]*\.py' | while read -r p; do exe=$(tr '\0' '\n' < "/proc/$p/cmdline" 2>/dev/null | head -1); case "$(basename "${exe:-none}")" in python*) [ "$p" != "$$" ] && exit 0;; esac; done; [ $? -eq 0 ]; then + if trainer_alive; then echo "[$(date '+%T')] [wait] $tag: another python trainer is alive, deferring this pass"; return 1 fi if ! cool_down "$tag"; then echo "[$(date '+%T')] [defer] $tag: box too hot"; return 1; fi From 78bacf89532642d8727416af89985ea4380cba0a Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 21:08:54 +0100 Subject: [PATCH 23/35] Fix cooldown COOL_C 58->72: 58C was below the SoC idle floor (deadlocked launches) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cool_down compares to hottest_c = max(GPU die, ALL ACPI SoC zones); the SoC zones idle at ~64-68C, so COOL_C=58 could never be satisfied and deferred every cell forever. The +31C transient that argued for 58 was measured on a heat-soaked box mid-cooldown, not a cool one — which is exactly what the dwell rejects. 72C is above the reachable idle floor, 10C under WARN(82), 18C under KILL(90); proven CORE ran at 70C. Fixes both run_ladder_scale_ext.sh and run_arch_ladder.sh (same A-lane value). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_arch_ladder.sh | 17 +++++++++-------- .../c5_evidence_scale_ext.json | 4 ++-- .../run_ladder_scale_ext.sh | 14 +++++++++----- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh index 37ec53a..2ed0d06 100755 --- a/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh +++ b/HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh @@ -19,14 +19,15 @@ DATA="$ROOT/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/t LOG="$LDIR/run_arch_ladder.log" PY=python3 # Cool-down gate (see cool_down() for the 2026-07-23 post-mortem that set these). -# COOL_C=58 is derived from sentinel.py's own constants, not guessed: TEMP_KILL_C=90 minus -# the +31C idle->load transient measured on 2026-07-23 (66C at the 15:17:47 BST launch -> -# soc 97C 50s later, sentinel_attn1to3_85M_s0.log 14:18:37Z), minus 1C. Launching at <=58C -# is therefore the hottest start that keeps the first-minute transient under the kill line; -# the old 70C sat ABOVE the box's own recently-loaded idle floor (64-69C), so it gated -# nothing. A cold box reads 43-44C here, so 58C is attainable — just not one minute after -# a thermal kill, which is exactly the launch this blocks. -COOL_C="${LADDER_COOL_C:-58}" # don't launch onto a box hotter than this +# COOL_C=72 (CORRECTED 2026-07-23 evening): cool_down compares to hottest_c = max(GPU die, ALL ACPI +# SoC zones), and the SoC zones idle at ~64-68C on this box — so an earlier 58C try was BELOW the idle +# floor and deadlocked every launch (the dwell could never be satisfied). sentinel WARNs at 82C, KILLs +# at 90C (3 consecutive). 72C is just above the idle floor (dwell reachable), 10C under WARN, 18C under +# KILL; the proven CORE run_ladder.sh ran the 420M ladder at 70C. The +31C "transient" that argued for +# 58C was measured on a HEAT-SOAKED box mid-cooldown after a kill (66C dipping from 90C), not a cool +# one — which is exactly what the DWELL (6 sustained samples) rejects. The dwell + DEFER is the real +# fix for the thrash; the threshold just has to be reachable AND under WARN. +COOL_C="${LADDER_COOL_C:-72}" # don't launch onto a box hotter than this (sustained, via dwell) COOL_DWELL="${LADDER_COOL_DWELL:-6}" # consecutive cool samples required (6 x 30s = 3 min) COOL_MAX="${LADDER_COOL_MAX:-30}" # bound: 30 x 30s = 15 min, then DEFER (never launch) MAXPASS="${LADDER_MAXPASS:-100}" diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json index a573c96..1fbe503 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json @@ -61,7 +61,7 @@ "c5_6_sentinel": { "result": "armed per cell", - "detail": "run_ladder_scale_ext.sh arms `sentinel.py watch --kill-at 0.80` on each cell's trainer pid (0.80 < safe_cuda 0.85 < OOM cliff) plus the thermal kill-switch, and preflights before the first launch. The cool-down gate is the FIXED 2026-07-23 version: requires 6 sustained sub-58C samples (3 min dwell), DEFERS on the bounded fall-through (never 'launch anyway'), and honours its return value so the hot-spell backoff engages — the CORE run_ladder.sh gate did none of these and let a cell relaunch onto a heat-soaked box 60s after a kill.", + "detail": "run_ladder_scale_ext.sh arms `sentinel.py watch --kill-at 0.80` on each cell's trainer pid (0.80 < safe_cuda 0.85 < OOM cliff) plus the thermal kill-switch, and preflights before the first launch. The cool-down gate is the FIXED 2026-07-23 version: requires 6 sustained sub-COOL_C samples (3 min dwell), DEFERS on the bounded fall-through (never 'launch anyway'), and honours its return value so the hot-spell backoff engages — the CORE run_ladder.sh gate did none of these and let a cell relaunch onto a heat-soaked box 60s after a kill. COOL_C=72C: hottest_c = max(GPU die, ALL SoC zones) idles at ~64-68C, so the threshold sits just above the reachable idle floor, 10C under WARN(82) and 18C under KILL(90); the proven CORE ran the 420M ladder at 70C. The dwell (not a low threshold) is what rejects a heat-soaked box dipping through the threshold mid-cooldown.", "src": "log" }, @@ -75,7 +75,7 @@ "not_claimed": "No result is claimed at queue time. The 840M point is n=1 and LR-confounded (fixed AdamW LR across horizons) -> directional at best. Achieved tok/s is reported, never %-of-peak MFU (GB10 peak estimated, §C24).", - "thermal_precondition": "The box hard-locked 2026-07-23 15:24 BST under an 85M thermal load; NorMuon-at-scale is a heavier 596M/51.5 GB sustained load. The durable firmware fix (DGX OS OTA 7.5.0 + BIOS/BSP + crashkernel, human/sudo) is STILL pending. Recommendation: run the 420M-s2 pair first; land the firmware fix before the 840M rung.", + "thermal_precondition": "The box hard-locked 2026-07-23 15:24 BST under an 85M thermal load; NorMuon-at-scale is a heavier 596M/51.5 GB sustained load. The durable firmware fix (DGX OS OTA 7.5.0 + BIOS/BSP + crashkernel, human/sudo) is STILL pending. Recommendation: run the 420M-s2 pair first; land the firmware fix before the 840M rung. NOTE: this launch scope is the 420M-s2 pair only; the 840M pair is commented out in the driver, deferred behind the firmware fix.", "evidence_path": "Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/c5_evidence_scale_ext.json" } diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index 68663a3..60da5ee 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -30,11 +30,15 @@ RESULTS=$IMU1/results LOG=$LDIR/run_ladder_scale_ext.log PY=python3 -# COOL_C derivation (2026-07-23): sentinel kills at TEMP_KILL_C=90C on the hottest of GPU die + SoC -# zones; the measured idle->load transient is ~+31C. Starting below 90-31-1 = 58C keeps even the -# first-minute transient under the kill line. 70C (the CORE default) sat AT the box's own loaded idle -# floor (64-69C), so it gated nothing. Overridable, but do not raise it toward the kill line. -COOL_C="${LADDER_COOL_C:-58}" +# COOL_C (2026-07-23, corrected): cool_down compares COOL_C to hottest_c = max(GPU die, ALL ACPI SoC +# zones). The SoC zones idle at ~64-68C on this box, so a threshold BELOW that floor (an earlier 58C +# try) can never be satisfied and defers every cell forever. sentinel WARNs at 82C and KILLs at 90C +# (3 consecutive). 72C sits just above the ~64-68C idle floor (so a sustained-cool dwell is reachable), +# 10C under WARN and 18C under KILL. The proven CORE run_ladder.sh ran the whole 420M ladder at 70C; +# 72C matches that with dwell headroom. The REAL fix for the 2026-07-23 thrash is the sustained dwell + +# DEFER below (a heat-soaked box dipping through 72C mid-cooldown can't satisfy 6 consecutive samples), +# NOT a low threshold. Overridable, but keep it above the idle floor and well under WARN. +COOL_C="${LADDER_COOL_C:-72}" COOL_DWELL="${LADDER_COOL_DWELL:-6}" # consecutive sub-COOL_C samples required (6 x 30s = 3 min sustained) COOL_MAX="${LADDER_COOL_MAX:-30}" # bounded wait: 30 samples = 15 min, then DEFER (never "launch anyway") From e01114683f23f5b5752cafb49e3392000b082e3c Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 22:33:20 +0100 Subject: [PATCH 24/35] Pause NorMuon-at-scale post-launch: 596M overheats the box (90C in 28min) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authorized 420M pair launched and trained, but sentinel thermal-killed the first cell at SoC 90-91C after ~28 min (pool only 64% — pure heat). Same thrash pattern that preceded today's 15:24 hard-lock, on a heavier load. Stopped it for a thermal decision; user chose firmware/kdump fix first. - resume_every 200->100 so forward progress survives each ~28-min thermal kill. - kdump enablement scripted separately (scratchpad/enable_kdump_gb10.sh): root cause is USE_KDUMP=0 -> crashkernel=1G-:0M (0 reserved) in the kdump-tools grub drop-in. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index 60da5ee..dc73d00 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -113,7 +113,10 @@ run_cell () { # pool headroom (unified memory shared; wait for >=60 GB free — a 596M cell runs at ~51.5 GB) for _ in $(seq 1 90); do a=$(free -g | awk '/Mem:/{print $7}'); [ "${a:-0}" -ge 60 ] && break; sleep 10; done echo "[$(date '+%F %T')] START $tag steps=$steps (~${budgetM}M tok)" - $PY "$TRAIN" --optimizer "$arm" --seed "$seed" --steps "$steps" --tag "$tag" --resume_every 200 \ + # --resume_every 100 (not 200): the 596M load hits sentinel's 90C thermal kill in ~28 min on this box + # (2026-07-23), close to the step-200 checkpoint interval — so checkpoint every 100 steps (~17 min) to + # guarantee forward progress survives each thermal kill, even if the box still runs warm post-firmware-fix. + $PY "$TRAIN" --optimizer "$arm" --seed "$seed" --steps "$steps" --tag "$tag" --resume_every 100 \ >> "$RESULTS/${tag}.out" 2>&1 & local tpid=$! $PY "$ROOT/sentinel.py" watch --pid "$tpid" --kill-at 0.80 --log "$LDIR/sentinel_${tag}.log" \ From 47cd68517e12ce33c872aa45cf8dd107aa99ebd7 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 22:47:28 +0100 Subject: [PATCH 25/35] Add a proactive thermal governor (SIGSTOP/SIGCONT) below sentinel's hard kill User wants to run on-box without the reboot. New governor watches hottest_c and SIGSTOPs the trainer at >=85C (GPU idles, box cools, ZERO lost progress), rechecks every 3 min, SIGCONTs below 75C. Keeps the box out of the 90C zone that preceded today's hard-lock while making continuous progress; sentinel stays the memory + 90C last-resort guard, checkpoints (resume_every 100) the hard-lock backstop. SIGSTOP/SIGCONT decision logic unit-tested (pause->T, resume->R, never left stopped). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index dc73d00..ac40eb3 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -77,6 +77,42 @@ cool_down () { return 1 } +# ---- Proactive thermal GOVERNOR (2026-07-23, user-requested) ----------------------------------------- +# A SOFT layer BELOW sentinel's 90C hard kill. While a cell trains, this watches hottest_c (GPU die + all +# SoC zones). At/above PAUSE_C it SIGSTOPs the trainer — the process freezes, the GPU goes idle and the box +# cools, with ZERO lost progress (in-memory state is preserved; no checkpoint/restart). It then rechecks +# every GOV_PAUSE_INTERVAL (3 min, per the user) and SIGCONTs once the box drops below RESUME_C. This keeps +# the box out of the 90C zone that preceded the 2026-07-23 hard-lock while making continuous forward +# progress. Checkpoints (--resume_every 100) stay as the backstop for an actual hard-lock. +PAUSE_C="${LADDER_PAUSE_C:-85}" # SIGSTOP the trainer at/above this hottest_c +RESUME_C="${LADDER_RESUME_C:-75}" # SIGCONT once it cools below this (10C hysteresis, no flapping) +GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-30}" # sample this often while running (catch PAUSE_C promptly) +GOV_PAUSE_INTERVAL="${LADDER_GOV_PAUSE:-180}" # check every 3 min while paused/cooling (user spec) + +thermal_governor () { + local pid=$1 tag=$2 paused=0 h + while kill -0 "$pid" 2>/dev/null; do + h=$(hottest_c) + if [ -n "$h" ]; then + if [ "$paused" -eq 0 ]; then + if [ "$h" -ge "$PAUSE_C" ] && kill -STOP "$pid" 2>/dev/null; then + paused=1 + echo "[$(date '+%T')] [gov] $tag: ${h}C >= ${PAUSE_C}C -> PAUSED (SIGSTOP) to cool" + fi + else + if [ "$h" -lt "$RESUME_C" ] && kill -CONT "$pid" 2>/dev/null; then + paused=0 + echo "[$(date '+%T')] [gov] $tag: ${h}C < ${RESUME_C}C -> RESUMED (SIGCONT)" + else + echo "[$(date '+%T')] [gov] $tag: ${h}C >= ${RESUME_C}C, still cooling (recheck in $((GOV_PAUSE_INTERVAL/60))min)" + fi + fi + fi + if [ "$paused" -eq 1 ]; then sleep "$GOV_PAUSE_INTERVAL"; else sleep "$GOV_RUN_INTERVAL"; fi + done + kill -CONT "$pid" 2>/dev/null # never leave a now-exited trainer in a stopped state +} + # EXTENSION cells: " ". 6409 steps = 420M tok (~17.5 h/cell measured), 12818 = 840M (~35 h). # 2026-07-23 LAUNCH SCOPE = the 420M-s2 pair ONLY (user-authorized): completes the n=3 across-seed CI at # the current top rung (~35 h), then STOPS. The 840M s0 pair is DEFERRED behind the firmware/kdump fix — @@ -122,9 +158,14 @@ run_cell () { $PY "$ROOT/sentinel.py" watch --pid "$tpid" --kill-at 0.80 --log "$LDIR/sentinel_${tag}.log" \ >/dev/null 2>&1 & local spid=$! + # Proactive thermal governor: SIGSTOP/SIGCONT the trainer to hold it under PAUSE_C (soft, below the + # sentinel 90C hard kill). Runs beside sentinel; sentinel remains the memory + last-resort thermal guard. + thermal_governor "$tpid" "$tag" >> "$LDIR/gov_${tag}.log" 2>&1 & + local gpid=$! wait "$tpid"; local rc=$? - # Give sentinel a beat to write its kill marker before we reap it (marker-before-grace fix, but be safe). - kill "$spid" 2>/dev/null + kill -CONT "$tpid" 2>/dev/null # if the trainer exited while SIGSTOP'd, don't leave it stopped + kill "$gpid" 2>/dev/null # stop the governor + kill "$spid" 2>/dev/null # stop the sentinel watcher if [ $rc -eq 0 ] && [ -f "$RESULTS/checkpoint_${tag}.pt" ]; then touch "$LDIR/${tag}.done"; echo "[$(date '+%T')] [done] $tag"; return 0 else From 6420c18320534d8d3f6f1ca0ea211ad0ee97dc98 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Thu, 23 Jul 2026 23:02:17 +0100 Subject: [PATCH 26/35] Tighten thermal governor: pause 85->80C, sample 30s->10s (live test overshot to 90) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live 2026-07-23 test: 596M + concurrent GPU work heats the box ~21C/min, so an 85C/30s governor overshot to 90-91C (the hard-lock line) before pausing. 80C/10s catches it at ~82-83C — a real margin below sentinel's 90C kill. Validated live that SIGSTOP cools 90->66C in 3 min and sentinel did NOT kill (governor got there first). ~25% duty at this thermal load; checkpoints (resume_persist_*.pt) preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index ac40eb3..abc9ca9 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -84,9 +84,14 @@ cool_down () { # every GOV_PAUSE_INTERVAL (3 min, per the user) and SIGCONTs once the box drops below RESUME_C. This keeps # the box out of the 90C zone that preceded the 2026-07-23 hard-lock while making continuous forward # progress. Checkpoints (--resume_every 100) stay as the backstop for an actual hard-lock. -PAUSE_C="${LADDER_PAUSE_C:-85}" # SIGSTOP the trainer at/above this hottest_c -RESUME_C="${LADDER_RESUME_C:-75}" # SIGCONT once it cools below this (10C hysteresis, no flapping) -GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-30}" # sample this often while running (catch PAUSE_C promptly) +# PAUSE_C=80 (not the user's literal 85): live 2026-07-23 test showed the 596M load + concurrent GPU work +# heats the box ~21C/min, so with a 30s sample an 85C threshold OVERSHOT to 90-91C — right at sentinel's +# 90C hard-kill / the hard-lock line. 80C + a 10s sample catches it at ~82-83C, keeping a real ~7C margin +# below 90. RESUME_C=75 with the 3-min cool check (user spec) still deep-cools to ~58C each pause, so the +# run window stays ~60s (measured ~25% duty). Raise LADDER_PAUSE_C back to 85 only if the box runs cooler. +PAUSE_C="${LADDER_PAUSE_C:-80}" # SIGSTOP the trainer at/above this hottest_c +RESUME_C="${LADDER_RESUME_C:-75}" # SIGCONT once it cools below this +GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-10}" # sample every 10s while running (heating is ~21C/min — catch it fast) GOV_PAUSE_INTERVAL="${LADDER_GOV_PAUSE:-180}" # check every 3 min while paused/cooling (user spec) thermal_governor () { From f90b06c2f37d971705b4ef54c1743e00793f53de Mon Sep 17 00:00:00 2001 From: yashb98 Date: Fri, 24 Jul 2026 08:58:32 +0100 Subject: [PATCH 27/35] Retune governor to responsive duty-cycle (3-min check was wasting 85% idle) User keeps heavy concurrent load permanently; the die barely cools, so the 3-min cooldown recheck left the trainer idle ~85-90% even after the die was cool. Switch to a tight 78-80C band with 5s rechecks: resume the instant it dips to 78C instead of waiting 3 min. Holds the die just under WARN(82), 10C under the 90C kill, and runs as much as the box's cooling allows (~3x duty). Safety unchanged (PAUSE_C + sentinel backstop). Overrides the earlier 3-min spec, which was the idle cause. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index abc9ca9..9137dc3 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -89,10 +89,17 @@ cool_down () { # 90C hard-kill / the hard-lock line. 80C + a 10s sample catches it at ~82-83C, keeping a real ~7C margin # below 90. RESUME_C=75 with the 3-min cool check (user spec) still deep-cools to ~58C each pause, so the # run window stays ~60s (measured ~25% duty). Raise LADDER_PAUSE_C back to 85 only if the box runs cooler. +# 2026-07-24 RETUNE for "make it work under permanent heavy concurrent load": the user keeps a dozen +# CPU/GPU processes running, so the shared die barely cools; the old 3-min cooldown check left the trainer +# idle 85-90% of the time (it resumed long after the die was already cool enough). This is now a RESPONSIVE +# duty-cycle governor: pause at 80C, resume the moment it dips to 78C, rechecking every 5s — so it holds the +# die in a tight 78-80C band and runs as much of the time as the box's cooling allows (~3x the old duty), +# while staying 10C under sentinel's 90C kill. Safety is unchanged (PAUSE_C + sentinel backstop); only the +# resume latency shrank. Widen LADDER_RESUME_C down / LADDER_GOV_PAUSE up to trade duty for a cooler die. PAUSE_C="${LADDER_PAUSE_C:-80}" # SIGSTOP the trainer at/above this hottest_c -RESUME_C="${LADDER_RESUME_C:-75}" # SIGCONT once it cools below this -GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-10}" # sample every 10s while running (heating is ~21C/min — catch it fast) -GOV_PAUSE_INTERVAL="${LADDER_GOV_PAUSE:-180}" # check every 3 min while paused/cooling (user spec) +RESUME_C="${LADDER_RESUME_C:-78}" # SIGCONT the moment it dips to this (tight 2C band = high duty) +GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-5}" # sample every 5s while running (heating is ~21C/min — catch 80C fast) +GOV_PAUSE_INTERVAL="${LADDER_GOV_PAUSE:-5}" # recheck every 5s while cooling — resume promptly, don't waste cooldown thermal_governor () { local pid=$1 tag=$2 paused=0 h From 3da9063ca6c12556342dcc060d0f1e07dbd75be1 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Fri, 24 Jul 2026 09:10:52 +0100 Subject: [PATCH 28/35] Lower governor band 80/78->76/72: 88C peak was too close to the 90C kill Measured the 80/78 retune: 50% duty (great) but peaked 88C (2C from the hard-lock line) due to ~8C thermal-inertia overshoot. Pulled to 76/72 + 3s run-sampling -> peak ~82-84C (~7C margin), still ~40% duty since the die cools fast to ~50C idle. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../run_ladder_scale_ext.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh index 9137dc3..08ebf58 100644 --- a/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh +++ b/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh @@ -96,10 +96,14 @@ cool_down () { # die in a tight 78-80C band and runs as much of the time as the box's cooling allows (~3x the old duty), # while staying 10C under sentinel's 90C kill. Safety is unchanged (PAUSE_C + sentinel backstop); only the # resume latency shrank. Widen LADDER_RESUME_C down / LADDER_GOV_PAUSE up to trade duty for a cooler die. -PAUSE_C="${LADDER_PAUSE_C:-80}" # SIGSTOP the trainer at/above this hottest_c -RESUME_C="${LADDER_RESUME_C:-78}" # SIGCONT the moment it dips to this (tight 2C band = high duty) -GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-5}" # sample every 5s while running (heating is ~21C/min — catch 80C fast) -GOV_PAUSE_INTERVAL="${LADDER_GOV_PAUSE:-5}" # recheck every 5s while cooling — resume promptly, don't waste cooldown +# 2026-07-24 measured: an 80/78 band gave 50% duty but PEAKED 88C (only 2C under the 90C kill) — the +# ~8C overshoot from thermal inertia + concurrent-load spikes is too much on a box that hard-locked at +# 92C today. Pulled the band down to 76/72 and sped run-sampling to 3s: peak now ~82-84C (a ~7C margin), +# still ~40-45% duty because the die cools fast to ~50C when idle so a 4C band cycles quickly. +PAUSE_C="${LADDER_PAUSE_C:-76}" # SIGSTOP at/above this — leaves room for the ~6-8C overshoot +RESUME_C="${LADDER_RESUME_C:-72}" # SIGCONT at this (4C band; cooling is fast so windows stay long) +GOV_RUN_INTERVAL="${LADDER_GOV_RUN:-3}" # sample every 3s while running — catch 76C before it overshoots far +GOV_PAUSE_INTERVAL="${LADDER_GOV_PAUSE:-5}" # recheck every 5s while cooling — resume promptly thermal_governor () { local pid=$1 tag=$2 paused=0 h From 97040c727f24980faa9d4c2ea787ebff06db2df6 Mon Sep 17 00:00:00 2001 From: yashb98 Date: Tue, 4 Aug 2026 20:16:18 +0100 Subject: [PATCH 29/35] Add model-card provenance fact sheet (every published number -> its eval) Traces each number a HuggingFace model card would carry back to the dataset id, config, split, seq len, stride, tokenizer and results file that produced it, plus Reproduce / Training-details / checkpoint-inventory / loader-API sections. Produced by an 8-dimension parallel file audit with an adversarial verification pass per dimension; three load-bearing findings were re-verified by hand. Three findings contradict what the repo currently asserts: - Qwen3-0.6B/README.md:35-37 claims all four Qwen3 perplexities use "the identical 300k-token FineWeb-Edu val slice". They do not. 13.40 and 46.31 come from tokcache_133072000_300000.pt (hardcoded at eval_original_vs_repro.py:22); 28.65 and 23.52 come from tokcache_1191478400_300000.pt. So 28.65/13.40 and 23.52/13.40 are cross-slice ratios, not like-for-like gaps. - The -0.474 bpb NorMuon result is still advertised as a "significant win" in four places, but the scaling-persistence ladder closed 2026-07-28 with trend_verdict=CONVERGES / ledger_verdict=null: the gap converges away with budget (0.474 -> 0.126 -> 0.072). It holds at a 42M-token budget only. - The "max error 0.0 / bit-exact" claim is CPU-fp32-only on a single 5-token prompt. The repo's own GPU per-layer delta is 1.95e-03, which trips its own 1e-3 gate. 13.40 is confirmed as our own measurement of the released Qwen3-0.6B-Base on this box (2026-06-09), not a figure copied from the tech report; only the "36T tok" label is borrowed. Also records the live environment stamp (python 3.12.11, torch 2.11.0+cu130, CUDA 13.0, cuDNN 91900, GB10 driver 580.142) since the repo pins versions in one place only and bit-exactness is version-sensitive, and the honest gap list: no commit hash stamped in any results file, no dataset revision pinned for fineweb-edu, and zero determinism flags repo-wide. Co-Authored-By: Claude Opus 5 (1M context) --- MODEL_CARD_FACTS.md | 682 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 682 insertions(+) create mode 100644 MODEL_CARD_FACTS.md diff --git a/MODEL_CARD_FACTS.md b/MODEL_CARD_FACTS.md new file mode 100644 index 0000000..aceec00 --- /dev/null +++ b/MODEL_CARD_FACTS.md @@ -0,0 +1,682 @@ +# Model-card fact sheet — provenance for every published number + +**Purpose.** Before any BuildFromScratch result is published as a HuggingFace model card, +every number on that card must be attached to the eval that produced it: dataset id, config, +split, sequence length, stride, tokenizer, and the file on disk that recorded it. This document +is that attachment. It answers a reviewer's questions directly and, where the repo cannot answer, +says so instead of guessing. + +**Status: this is a provenance audit, not a model card.** It is deliberately unflattering. +§9 lists 20 things that would embarrass the author if the current READMEs were published as-is — +including two claims in the repo's own READMEs that are demonstrably false. Fix those before +writing the card, not after a reviewer finds them. + +**Produced** 2026-08-04 by an 8-dimension parallel file audit (one agent per dimension) followed +by an adversarial verification pass over every extracted fact (a second agent per dimension whose +instructions were to *refute*, defaulting to WRONG/NEEDS_QUALIFIER under uncertainty), then a +synthesis pass. 17 agents, 715 tool calls. Facts the adversarial pass overturned are marked +**[CORRECTED]**; facts needing a caveat are marked **[QUALIFIER]**. + +**Scope limit.** Every claim here is traced to a file:line that was read, but this is an audit of +*what the repo records*, not a re-execution of the experiments. Where a recorded number could not +be re-derived from disk, §8 says so. + +--- + +## Independent spot-check of the three load-bearing findings + +The three findings below were re-verified by hand, outside the agent pipeline, because each one +contradicts something the repo currently asserts. All three reproduce. + +**1. The Qwen3 perplexities are NOT on a common val slice — the README's comparability claim is false.** + +`Qwen3-0.6B/README.md:35-37` states: *"All perplexities use **identical eval code on the identical +300k-token FineWeb-Edu val slice** … so every row is directly comparable."* Verified false: + +| Run | Number | Val cache actually used | +|---|---|---| +| `eval_original_vs_repro.py` (released model + LR sweep) | **13.40**, **46.31** | `tokcache_133072000_300000.pt` — hardcoded at `eval_original_vs_repro.py:22` | +| `qwen3_baseline2tpp` (faithful, 1.19B tok) | **28.65** | `tokcache_1191478400_300000.pt` — streamed at `qwen3_baseline2tpp_train.log:3,6` | +| `qwen3_imu1_2tpp` (modernized, 1.19B tok) | **23.52** | `tokcache_1191478400_300000.pt` — `qwen3_imu1_2tpp_train.log` | + +Both caches exist on disk with different sizes (1,066,978,101 B vs 9,534,229,373 B). So +**28.65/13.40 and 23.52/13.40 are cross-slice ratios** and must not be published as like-for-like. +Only 46.31/13.40 is same-slice. + +**2. `13.40` is our own measurement, not a borrowed figure.** + +`Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/original_vs_repro.txt`, verbatim: + +``` +[2026-06-09 16:51:36] Original vs reproduction — val=300,000 tokens, 50 windows x 4096 +ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = 13.400 (204,800 tok, 21s) +``` + +The `21s` wall-clock and the `[safe_cuda] capped CUDA at 85% of 129 GB unified pool` banner in +`results/original_eval_run2.log:2` are execution evidence: the released checkpoint was downloaded +and scored on this box by `eval_original_vs_repro.py`, using this repo's own `eval_ppl`. The only +borrowed element in that string is the `36T tok` label, transcribed from the Qwen3 tech report. +**State this split explicitly on the card** — a reviewer cares a great deal which it is. + +**3. The `−0.474 bpb` NorMuon win is advertised as a `significant win` but has been nulled at scale.** + +`Qwen3-0.6B/README.md:52` still reads *"**NorMuon > AdamW** | wikitext −0.474 bpb [0.444, 0.505] · +code −0.502 [0.456, 0.547] | **significant win**"*. The scaling-persistence ladder that closed +2026-07-28 says otherwise — +`Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json`: + +``` +trend_verdict CONVERGES +ledger_verdict null +significance_verdict null +rationale gap shrinks toward 0 with scale and falls within the noise floor at the + largest budget — an early-training speedup that converges away (no + advantage at scale) +``` + +−0.474 is real *at a 42M-token budget*. It is not a standing result. Four published surfaces still +sell it as a win (§3). + +--- + +## Live environment stamp — measured 2026-08-04 on the box + +Recorded here because the repo stamps versions in exactly one place and bit-exactness is +version-sensitive. This is the current box, which is **not** necessarily the box that produced the +2026-05/06 artifacts (§4.3 covers that gap). + +| | Value | Source | +|---|---|---| +| Python | `3.12.11` (conda-forge, GCC 13.3.0) | `sys.version` | +| torch | `2.11.0+cu130` | `torch.__version__` | +| CUDA (torch) | `13.0` | `torch.version.cuda` | +| CUDA (nvcc) | `13.0, V13.0.88` | `nvcc --version` | +| cuDNN | `91900` | `torch.backends.cudnn.version()` | +| transformers | `5.8.0` | `importlib.metadata` | +| datasets | `4.8.5` | `importlib.metadata` | +| safetensors / accelerate | `0.7.0` / `1.13.0` | `importlib.metadata` | +| numpy | `2.5.1` — **pyproject pins `2.4.4`; live env drifts** | `importlib.metadata` | +| tokenizers | `0.22.2` | `importlib.metadata` | +| jax / flax | `0.11.0` / `0.12.7` | `importlib.metadata` | +| GPU | `NVIDIA GB10`, driver `580.142` | `nvidia-smi` | +| OS / arch | Ubuntu 24.04.4 LTS, `aarch64` | `/etc/os-release`, `uname -m` | +| Repo commit | `3da9063`, branch `harden-research-loop`, **zero git tags** | `git rev-parse HEAD` | + +torch/transformers/datasets/safetensors/accelerate match `SmolLM2-134(base)/pyproject.toml` +field-for-field, so **SmolLM2 parity is re-checkable today**. numpy has drifted. cuDNN `91900` and +driver `580.142` are recorded nowhere in the repo. Qwen3 has no environment file at all. + +--- + +## Quick answers to the questions that prompted this audit + +| Question | Answer | Detail | +|---|---|---| +| Which dataset/config/split gave **15.371**? | `Salesforce/wikitext` / `wikitext-2-raw-v1` / `validation`, seq 1024, stride 512 (overlapping), SmolLM2 tokenizer | §1 | +| Which gave **6.89 → 3.79**? | `roneneldan/TinyStories` / no config / `validation`, seq 1024, stride 1024 (non-overlapping) | §1, §5.3 | +| Which gave **28.65 / 46.31 / 23.52**? | `HuggingFaceFW/fineweb-edu` / `sample-10BT`, private val tail, seq 4096, stride 4096 — but on **two different tails** | §1 | +| Which wikitext for **−0.474 bpb**? | `wikitext-2-raw-v1` (rev `b08601e0…`), *not* wikitext-103 | §3 | +| Is **13.40** ours or borrowed? | **Ours** — measured on this box, 2026-06-09 | §2 | +| Commit hash for the results? | **None stamped.** Best anchors `e791875` / `84a96c0` both *postdate* the artifacts | §4.2 | +| CPU-only or GPU parity? | **CPU fp32 only** for the `max error 0.0` claim. GPU deltas exist and one **trips the repo's own 1e-3 gate** | §4.4 | +| Determinism flags? | **None, repo-wide** — verified negative for all six flags | §4.5 | +| Checkpoint formats? | 107 files, 270.68 GiB, **all `.pt`/`.pkl` pickles — zero safetensors**, no `config.json`, no tokenizer files | §6 | +| `modeling_*.py` or raw weights? | **Export SmolLM2 to stock `LlamaForCausalLM` safetensors** (an exporter exists). `trust_remote_code` only for IMU-1 / partial-RoPE / HybridSSM | §6.3 | +| Real loader API? | No `from_pretrained`. `Qwen3ForCausalLM(Qwen3Config())` + `load_official_weights_into_ours()` from `verify.py` | §7 | + +**Two traps any published snippet must avoid:** passing `attention_mask` **disables causal +masking** in both models (`is_causal=(attention_mask is None)`), and `attention_dropout` is dead +config with no read site. + +--- + +# Full fact sheet + +## 1. Model-index provenance + +**Read this first:** the eight numbers below come from **five mutually incomparable eval recipes** on **four different corpora** with **three different windowing schemes**. No two rows are like-for-like unless explicitly stated. + +| Metric value | Model / run | HF dataset id | Config | Split | Seq len | Stride | Tokenizer | n eval tokens | Source file:line | +|---|---|---|---|---|---|---|---|---|---| +| **15.371** (ours 15.370989092449635 / HF 15.370989964425396, Δ +8.72e-07) | **NOT a repo-trained model.** Official `HuggingFaceTB/SmolLM2-135M` safetensors loaded into this repo's `SmolLM2ForCausalLM` via `load_official_weights_into_ours` **[QUALIFIER]** | `Salesforce/wikitext` (no `revision=` pin) | `wikitext-2-raw-v1` (IS the `-raw-` variant) | `validation` | 1024 | 512 — **overlapping, no `-100` masking** | `HuggingFaceTB/SmolLM2-135M` (own tokenizer; no BOS) | **62,403 scored targets = 31,743 distinct positions (1.99× double-count); first ~11.8% of the 268,140-token split** | `SmolLM2-134(base)/results/perplexity.json:2-3`; recipe `SmolLM2-134(base)/_build_notebook.py:233-257`; blank-row filter `:234` | +| **6.8945** (full precision 6.894546783281595) | Baseline "BEFORE" eval — again the **official** SmolLM2-135M weights in our class, bf16 on cuda, scored before any optimizer step | `roneneldan/TinyStories` (resolved rev `f54c09fd23315a6f9c86f9dc80f725de7d8f9c64`) **[CORRECTED — rev IS recoverable]** | none passed | `validation` | 1024 | **1024 — non-overlapping** **[CORRECTED: not 512]** | `HuggingFaceTB/SmolLM2-135M` | **199,485** = 195 windows × 1023, over the **first 1,040 non-empty stories** (200,068 packed tokens) **[CORRECTED — count recovered by re-running the packer]** | `SmolLM2-134(base)/results/tinystories_before.txt:2`; `results/tinystories_train.log:10`; eval fn `train_tinystories.py:62-78`, called `:196` | +| **3.7900** (full precision 3.7899503859716885) | `SmolLM2-134(base)/checkpoint_tinystories.pt` — continued pretrain from official weights, step 24,414 / 99,999,744 tokens | `roneneldan/TinyStories` | none | `validation` | 1024 | 1024 | `HuggingFaceTB/SmolLM2-135M` | 199,485 (identical `val_tokens` tensor as the 6.8945 row → strictly paired) | `SmolLM2-134(base)/results/tinystories_after.txt:2`; `results/tinystories_train.log:508`; eval call `train_tinystories.py:362` | +| **28.65** | `checkpoint_qwen3_baseline2tpp.pt`, faithful build, **final step 18,150** (1,189,478,400 tok) | `HuggingFaceFW/fineweb-edu` (no `revision=`) | `sample-10BT` | `train` (streaming) → private 300k-token val tail in `tokcache_1191478400_300000.pt` | 4096 | 4096 — non-overlapping | `Qwen/Qwen3-0.6B-Base` | 204,800 (50 × 4096) | `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log:396` | +| **46.31** | `checkpoint_qwen3_lr24.pt` — a **2,000-step / 131,072,000-token LR-selection run**, not a headline model (siblings lr17 46.89, lr30 49.28) | `HuggingFaceFW/fineweb-edu` | `sample-10BT` | `train` (streaming) → val tail in **`tokcache_133072000_300000.pt`** | 4096 | 4096 | `Qwen/Qwen3-0.6B-Base` | 204,800 | `.../results/qwen3_lr24_train.log:226`; also `qwen3_lr24_after.txt:2` | +| **23.52** | `checkpoint_imu1_2tpp_step18000.pt` — modernized/IMU-1 arm, **in-loop eval at step 18,000, NOT the 18,150 endpoint** (no AFTER eval exists) **[QUALIFIER]** | `HuggingFaceFW/fineweb-edu` | `sample-10BT` | val tail in `tokcache_1191478400_300000.pt` | 4096 | 4096 | `Qwen/Qwen3-0.6B-Base` | 204,800 | `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log:381` | +| **13.40** (13.400) | **The released `Qwen/Qwen3-0.6B-Base` HF checkpoint, scored by us** (see §2) | `HuggingFaceFW/fineweb-edu` | `sample-10BT` | val tail in **`tokcache_133072000_300000.pt`** | 4096 | 4096 | `Qwen/Qwen3-0.6B-Base` | 204,800 | `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/original_vs_repro.txt:2` (stdout `results/original_eval_run2.log:6`, 2026-06-09 16:51:36) | +| **−0.474 bpb** (stored as `improvement_bpb` = +0.47432550192416323; AdamW 2.1098171365956357 − NorMuon 1.6354916346714725; CI95 **[0.4434844613250229, 0.5051665425233036]**) | NorMuon vs AdamW, **596,049,920-param** Qwen3, **42M tokens/cell (640 steps × 65,536)**, **n=3 seeds/arm** | `Salesforce/wikitext` **rev `b08601e04326c79dfdd32d625aee71d232d685c3`** | `wikitext-2-raw-v1` | `validation` | 1024 | 512 — overlapping | `Qwen/Qwen3-0.6B-Base` | 204,600 scored = **102,911 distinct** (MAX_WINDOWS=200 cap; 869,710 bytes denominator) | `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/verdict.json:19-30`; corpus `score_cohort.py:54-56`; windowing `score_cohort.py:24` | +| **−0.502 bpb (companion code corpus)** (+0.5015586517902451; AdamW 3.3846985755955523 vs NorMuon 2.8831399238053073; CI95 [0.4559911731303807, 0.5471261304501094]) | same 6 cells | `codeparrot/codeparrot-clean-valid` **rev `4db92d2ec0c1b4c41eeb439cfae16854511d9dcd`** | — | `train` (streaming, whole docs until >500,000 chars) | 1024 | 512 | `Qwen/Qwen3-0.6B-Base` | 204,600 tokens / 843,643 bytes | `Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json:80-84`; corpus `score_cohort.py:57-63` | + +**Cross-row warnings that must ship with any model index:** + +- **[CORRECTED — the repo's own README is FALSE here]** `Qwen3-0.6B/README.md:35-37` claims all four Qwen3 PPLs use "identical eval code on the identical 300k-token FineWeb-Edu val slice". They do not. **13.40 and 46.31** are on `tokcache_133072000_300000.pt` (val sha1 `8ad9e246b0bf63bd`, first ids `[10879, 5547, 481, …]`); **28.65 and 23.52** are on `tokcache_1191478400_300000.pt` (val sha1 `ad3513719d0f81e4`, first ids `[38131, 6022, 369, …]`). Therefore **28.65/13.40 = "2.14×" and 23.52/13.40 = "1.76×" are CROSS-SLICE ratios and must not be published as like-for-like gaps.** Only 46.31/13.40 (3.456×) is same-slice. The same false claim is repeated at `Qwen3-0.6B/results_overview/plots/README.md:50`. +- **23.52 vs 28.65 is not step-matched.** The like-for-like pair is **28.66 @ step 18,000** (`qwen3_baseline2tpp_train.log:389`) vs 23.52 → −17.94%. The headline survives, but as printed the arms differ by 0.83% of budget. +- **Slice sensitivity is ~14%.** The same `checkpoint_qwen3_baseline2tpp.pt` that scores 28.65 on its own slice scores **24.5514** on the dataset-forge held-out FineWeb-Edu split under `text-lm-v2` windowing — recorded with an explicit delta field `"base_ppl_measured_vs_claim_delta": -4.09860353498452` in `Qwen3-0.6B/experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/eval/brief_probes_results.json`. +- **None of 28.65 / 23.52 / 46.31 / 13.40 is decontaminated.** All four used caches built by the pre-audit splitter, which the repo's own current code calls leak-suspect: *"the old code cut the stream by token count — val was the sequential continuation of train, leak-suspect"* (`train_qwen3.py:130-136`). Proof independent of filenames: neither cache contains the `decontam` key that the post-fix splitter writes (`train_qwen3.py:190`). The fix landed in commit `86e79f3` (2026-06-16 21:57 UTC), **after** all four Phase-B runs finished. +- **The repo's own governance bans these as headlines.** `research/eval/base_eval_verdict.md:59`: *"It is **n=1 FineWeb val-PPL — the founding-mistake metric, banned as a sole/headline signal by §C25.7.3**"*. + +**The §C10-comparable, suite-stamped alternative** (safe for a card) — `Qwen3-0.6B/experiments/2026-06-16_qwen3-faithful_eval-first/eval/suite_results.json`, `suite_version: "text-lm-v2"`, target `checkpoint_qwen3_baseline2tpp.pt`, dated 2026-06-16 22:18:01, SEQ/STRIDE/MAX_WINDOWS = 1024/512/200, revision-pinned corpora: + +| Corpus | PPL | BPB | n_tokens | n_bytes | +|---|---|---|---|---| +| `wikitext2_raw_v1_val` | 37.010055463333096 | 1.2256204566076285 | 204,600 | 869,710 | +| `codeparrot_clean_valid` | 438.67295146042875 | 2.128595386220801 | 204,600 | 843,643 | + +Downstream (`text-lm-v3`, 2026-06-24, n=500/task, Wilson CIs) — `research/eval/downstream_v3/*.json`: faithful LAMBADA **0.170** [0.1396, 0.2054], mean BPB-gold 1.18827; IMU-1 **0.212** [0.1784, 0.2500], 1.14232; pRoPE-0.25 **0.166** [0.1360, 0.2011], 1.20234. **[CORRECTED]** Do not write "MC tasks are at chance": `arc_easy` acc_norm 0.454 and `hellaswag` acc_norm 0.348 both carry `"signal": true`; only `winogrande` (0.500 vs chance 0.500) is `"signal": false`. The repo's own careful phrasing is `research/eval/base_eval_verdict.md:63` — *"flagged `signal: true` but only marginally above chance with wide CIs; not headline-bearing."* + +--- + +## 2. Is 13.40 ours or borrowed? + +**13.40 is OURS. It is a measurement this repo performed on this box.** It is not copied from the Qwen3 tech report, the HF model card, or any blog. + +Evidence: + +- `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py:47-51` downloads the released checkpoint and scores it with our own loop: + ``` + hf = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.bfloat16).to(device) + ppl_orig, n = eval_ppl(hf, val, device) + lines.append(f"ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = {ppl_orig:8.3f} ({n:,} tok, {time.time()-t0:.0f}s)") + ``` + `REPO = "Qwen/Qwen3-0.6B-Base"` imported from `train_qwen3.py:59` via `eval_original_vs_repro.py:19`. Scoring uses the repo's own `eval_ppl` (`:26-36`), not any external harness. +- Live-run corroboration with a real safe_cuda banner: `results/original_eval_run2.log:2` *"[safe_cuda] capped CUDA at 85% of 129 GB unified pool (~109 GB)"*, `:3` *"Loading weights: 100%|██████████| 310/310"*, `:6` the 13.400 line, timestamped 2026-06-09 16:51:36. +- An earlier same-day attempt (`results/original_eval_wrapper.log`, 11:19) crashed on `UnpicklingError` before printing any PPL — so there is no conflicting earlier value. + +**What IS borrowed, in the same string:** the label `(36T tok)` is a hardcoded f-string literal at `eval_original_vs_repro.py:51`. The 36T figure is transcribed from the Qwen3 tech report (`training_plan.md:17-19`: *"Per the Qwen3 tech report (verbatim summary): - **Corpus:** 36T tokens across 119 languages"*, cited as arXiv 2505.09388 at `Qwen3-0.6B/README.md:29`). **13.40 = ours (measured); 36T = theirs (copied).** Keep that split explicit. + +**Wording hazard:** `Qwen3-0.6B/PLOTS_INDEX.md:37` and `Qwen3-0.6B/results_overview/plots/README.md:49` both use the word "published" next to 13.40. In context that means *"the published (released) model"*, not *"a published number"* — the generator `make_overview_plots.py:18-19,51` correctly traces `ORIGINAL_PPL = 13.40` to our `original_vs_repro.txt`. **Do not describe 13.40 on a model card as a reported/published figure.** + +**No suite-comparable counterpart exists.** I enumerated all 9 `suite_results*.json` in the repo: every `target_ckpt` is a local `.pt`/`.pkl`; the `text-lm-v2` suite was **never** run against the released Qwen3-0.6B-Base. So there is no BPB/wikitext number for the released model to pair with our 37.01. + +--- + +## 3. −0.474 bpb: which wikitext, and does the claim still stand? + +**Which wikitext:** `wikitext-2-raw-v1` — *not* wikitext-103. Verbatim at `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/score_cohort.py:54-55`: +``` +wt = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation", + revision=WIKITEXT_REV) +``` +with `WIKITEXT_REV = "b08601e04326c79dfdd32d625aee71d232d685c3"` at `score_cohort.py:25`. Same triple pinned as the suite standard at `.claude/skills/eval-harness/references/suite.md:133`. (For contrast, `SmolLM2-134(base)/train.py:78` uses `wikitext-103-raw-v1` **train** — a different thing entirely, and a training corpus, not this eval.) + +**Sign convention:** the JSON stores **+0.47432550192416323** as `improvement_bpb = adamw_mean − normuon_mean` (`score_ladder.py:85-86`: *"so gap > 0 == NorMuon better"*; `research/eval_stats.py:138`). The literal string `−0.474` appears on disk **only** at `Qwen3-0.6B/README.md:52` and `:243`. Both forms mean "NorMuon 0.474 bpb lower (better)". + +**Does it still stand? NO — it has been nulled at scale.** + +`Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json` (re-scored 2026-07-28, n=3 seeds/arm at **every** rung, fixed N=596M, budget swept): + +| Budget | wikitext-2 gap (bpb) | CI95 | significant | code_py gap | CI95 | +|---|---|---|---|---|---| +| 42M | 0.47432550192416323 (`:27`) | [0.4435, 0.5052] | yes | 0.5015586517902451 (`:80`) | [0.4560, 0.5471] | +| 168M | 0.12590584068581911 (`:44`) | [0.0893, 0.1625] | yes | 0.1757807425441804 (`:97`) | [0.1369, 0.2146] | +| 420M | 0.07169397744785555 (`:61`) | [0.05525251860105098, 0.08813543629466011] (`:62-66`) | **yes** | 0.17708989020863175 (`:114`) | [0.1307, 0.2234] | + +- `trend_verdict: "CONVERGES"` (`:198`), `ledger_verdict: "null"` (`:199`), rationale (`:194`): *"gap shrinks toward 0 with scale and falls within the noise floor at the largest budget — an early-training speedup that converges away (no advantage at scale)."* +- `research/ledger/ledger.json:1554,1565-1566` — run `2026-07-05_qwen3-0.6b_scaling-persistence`, status `done`, **verdict `null`**. + +**Three nuances a reviewer will demand and that must not be lost:** + +1. **It is a BUDGET-scaling null at FIXED model size N=596M.** Nothing on disk says anything about larger N. +2. **The 420M wikitext gap is still nominally SIGNIFICANT as measured** (+0.0717, CI excludes 0). The "falls within the noise floor" phrase refers to the **OLS-fitted** gap at the top rung — `gap_hi_fit`/`edge_at_top` = 0.029726712435672376 (`:140-142`) vs `gap_noise` 0.03675972213287565 (`:146`), `edge_resolved: false` on wikitext, **`true` on code**. +3. **[QUALIFIER — the "CONVERGES" label on the code corpus is weak]** code_py goes 0.50156 → 0.17578 → **0.17709**, i.e. it *increases* between the last two rungs with near-total CI overlap. That is a **plateau, not convergence**; only the 3-point OLS slope (−0.34197431351062624, r² 0.8412726939487719, `:156-158`) is negative, and it is dominated by the 42M rung. Rationale on disk (`:171`) is itself hedged: *"still above noise at the largest measured budget but trending out — the edge is eroding, extend the ladder before claiming it."* +4. The `null` is partly gate-driven: the §C25 `scaling` HARD battery is INCOMPLETE (missing `log_rmse_r2`, `holdout_extrapolation_pctdev`, `bootstrap_forecast_ci`, `:207-214`), so `win` was unreachable regardless — **but** `significance_verdict` was independently `null` from the CONVERGES trend mapping, with `c17_cap_applied: false` (`:203-206`). + +**The published surfaces are stale and over-claim. [CORRECTED — exposure is 4 sites, not 2]:** + +- `Qwen3-0.6B/README.md:52` — *"| **NorMuon > AdamW** | wikitext −0.474 bpb [0.444, 0.505] · code −0.502 [0.456, 0.547] | **significant win** |"* +- `Qwen3-0.6B/README.md:175` — *"AdamW by **+0.474 bpb on wikitext-2 (95% CI [0.444, 0.505])** and +0.502 on code — significant."* +- `Qwen3-0.6B/README.md:243` — same claim, tagged **significant win** +- `Qwen3-0.6B/PLOTS_INDEX.md:73` — *"+0.474 bpb, significant"* + +`grep -in 'scaling-persistence|converge|persist|0\.126|0\.072' Qwen3-0.6B/README.md` → **zero hits**. The file's mtime is 2026-07-06, before the ladder completed. The 42M ledger entry still reads `"verdict": "win"` (`research/ledger/ledger.json:482`) with a `caveats` field at `:512` asserting *"no scaling curve"* — a statement that became false when the ladder completed. The `normuon-optimizer` technique's `run_ids` (`ledger.json:153-154`) omit the ladder run (whose `technique_slug` is `null`, `:1557`), so **a ledger query by technique will not surface the null**. + +**Rounding defect:** the three README sites print the CI lower bound as `0.444`; the on-disk value is `0.4434844613250229`, which rounds to **0.443** (the root `README.md:112` gets this right). + +**To the source run's credit:** `RESULT.md:7` does scope it correctly — *"This is an early-training optimization-speed signal at one architecture and one budget; we do NOT claim it holds at scale"* — and `RESULT.md:45` predicts the fade. The failure is that `Qwen3-0.6B/README.md` dropped those qualifiers. + +**Root `README.md:105-121` is ALSO stale** (mtime 2026-07-23 vs verdict.json 2026-07-28): it says *"420M ×2 seeds"*, *"+0.073 [−0.038, +0.184] at 420M … **not significant** at the top"*, *"code_py: +0.502 → +0.176 → +0.192"*, code slope *"−0.328 (r² 0.81)"*, and *"**Verdict: directional, not a headline** — the 420M rung is n=2 (< 3 seeds, §C17)"*. Current truth: n=3/arm, top rung **significant**, code 0.17709, slope −0.34197 (r² 0.84127), `headline_capped_by_c17_power: false`, ledger verdict `null`. The 3rd 420M seed came from run `2026-07-23_qwen3-0.6b_normuon-at-scale` (`research/ledger/runs/2026-07-23_qwen3-0.6b_normuon-at-scale.md:356-360`). + +**Eval pipeline identity (the numbers ARE mutually comparable):** `score_ladder.py:40` imports the 42M scorer directly (`import score_cohort as sc … reuse score() + load_corpora()`), `:454-455` reuse its corpora/device/dtype, `:456` copies the 42M rung verbatim (`"source": "reused:cohort_bpb.json"`). Same `suite_version: text-lm-v2` on both. + +--- + +## 4. Reproduce + +### 4.1 Literal commands + +**Qwen3-0.6B — architecture parity (CPU, no GPU, no artifact written):** +```bash +cd /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B +python3 verify.py # asserts max|Δlogits| < 1e-3 and argmax equality; stdout only +``` +(`Qwen3-0.6B/verify.py:11`; `Qwen3-0.6B/README.md:498`: *"python verify.py # parity gate — runs on CPU, no GPU needed"*. The `cd` is load-bearing — `verify.py:16` does a flat `from model import …` with no sys.path handling.) + +**Qwen3-0.6B — parity with a machine-readable artifact:** +```bash +cd /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b +python3 verify_run.py # writes results/verify.json; exit 0 pass / 1 fail +``` +(`verify_run.py:9,31,93,99,102,106`; this is the first command in the paper appendix, `research/papers/qwen3-imu1-matched-compute/sections/reproducibility.tex:15`.) + +**Qwen3-0.6B — recompute 13.40 / 46.31:** +```bash +cd /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b +python3 eval_original_vs_repro.py # writes results/original_vs_repro.txt +``` +**Not runnable from a fresh clone.** `eval_original_vs_repro.py:41` hardcodes `device = torch.device("cuda")` with no CPU fallback, and it needs two gitignored artifacts: `results/tokcache_133072000_300000.pt` (1,066,978,101 B, 2026-06-08) and `checkpoint_qwen3_lr{17,24,30}.pt` (3,576,719,229 B each). `git check-ignore -v` → `.gitignore:20:*.pt`. It also reads the **pre-decontamination** cache (§1). + +**SmolLM2 — parity:** +```bash +cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" +pytest tests/ -v # broader gate +# or the script form: +python3 verify.py # produces the committed results/parity.log +``` +(`README.md:159-162`; `SmolLM2-134(base)/verify.py:11`; `tests/test_parity.py:8`.) **[QUALIFIER]** The pytest form is broader (param count `test_parity.py:53`, tied-embedding pointer `:61`, 512-token long context `:89-101`, all 30 per-layer hidden states `:104-132`) **but it is not unconditionally stronger: `tests/test_parity.py:34-41` calls `pytest.skip(…)` on ImportError or model-load failure ("can't load {REPO} (no internet or HF cache miss?)"), so with no network a green run proves nothing.** It also never prints the Δ value. + +**SmolLM2 — recompute 15.371 (there is no one-command path):** +```bash +cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" +python3 _build_notebook.py # writes results.ipynb (28 cells, no outputs) +jupyter nbconvert --to notebook --execute results.ipynb \ + --output results.ipynb --ExecutePreprocessor.timeout=2400 +``` +(`SmolLM2-134(base)/results/README.md:66-74`.) Three hazards: (1) `_build_notebook.py:561-562` **unconditionally overwrites** `results.ipynb`, wiping the executed outputs that are currently the only record; (2) all 28 cells re-execute, including a 150-step training cell that **overwrites `../checkpoint.pt`** (`_build_notebook.py:480-483`); (3) the notebook **never imports `safe_cuda`** (grep → zero hits) despite `CLAUDE.md` §C1 mandating it. **[CORRECTED]** A standalone wikitext-2 PPL script *does* exist — `SmolLM2-134(base)/eval_after_vs_base.py:50,74` — but it uses `max_windows=200` and bf16 (`:29`), so it produces a **different** number, it needs `checkpoint_tinystories.pt`, its declared outputs `results/tinystories_vs_base.{md,json}` are **absent from disk**, and its fallback at `:91` does `open("model.py")` on a file that does not exist. + +**SmolLM2 — reproduce the continued-pretrain run (best reconstruction; exact CLI never recorded):** +```bash +cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" +python3 train_tinystories.py --token_budget 100_000_000 +``` +(nearest documented form: `README.md:170-174`). It was **not** a `--resume` run (the log contains the baseline eval, which `train_tinystories.py:194` skips on resume; the CSV carries a header, written only when resume is None). **`grep -n "safe_cuda\|sentinel" train_tinystories.py` → zero hits: re-running as-is violates `CLAUDE.md` §C1 and §C6.** + +**Qwen3 Phase-B training (the 1.19B runs) — the real launch record:** `Qwen3-0.6B/builds/phase_b_driver.sh`, verified verbatim: +``` +:14 S=18150; W=900; COMMON="--eval_every 2000 --ckpt_every 2000 --log_every 50" +:19 cd "$FAITHFUL" && python train_qwen3.py --steps $S --peak_lr 2.4e-3 --end_lr 3.2e-4 \ +:20 --warmup_steps $W $COMMON --run_name baseline2tpp +:24 cd "$MOD" && python train_imu1.py --steps $S --warmup_steps $W $COMMON --run_name imu1_2tpp +:28-29, :33-34 partial-RoPE 0.25 / 0.10 +``` + +### 4.2 Commit hash + +**There is no provenance stamp.** No results file for either model carries a commit/`git_sha` field. `verify.json`'s complete key set is `repo/prompt/dtype/tolerance/max_abs_error/relative_error/hf_next_token_id/our_next_token_id/hf_next_token_text/our_next_token_text/argmax_match/passed/input_shape/total_seconds`. `perplexity.json`'s is `ours_ppl/hf_ppl/tokens/dataset/seq_len/stride`. `parity.log` is raw stdout. + +The only anchors are the commits that *added* the artifacts — and both **postdate** the artifact mtimes, so they bound from above rather than identify the tree that ran: + +| Artifact | mtime | Adding commit | +|---|---|---| +| `Qwen3-0.6B/builds/.../results/verify.json` | 2026-06-08 14:36 | `e791875` "Add Qwen3-0.6B from-scratch reproduction + three-build experiment" (2026-06-10) | +| `SmolLM2-134(base)/results/parity.log` | 2026-05-13 22:20 | `84a96c0` "Initial commit: SmolLM2-135M from-scratch reproduction + harness" (2026-05-20) | + +`git tag -l` → **empty**; HEAD is `3da9063` (2026-07-24). The ledger *does* auto-capture HEAD (`research/ledger/ledger.py:639`: `r["lineage"]["git_commit"] = git_head_commit()`) and all 29 runs carry one — but the earliest is `2026-06-16_qwen3-faithful_eval-first` (`86e79f3`), and **neither reproduction has a ledger run entry at all**. `lineage.env` is null for 28 of 29 runs. + +### 4.3 Software versions + +**Pinned in exactly two files, both under `SmolLM2-134(base)`, and they contradict each other:** + +| | `pyproject.toml` | `requirements.txt` | +|---|---|---| +| python | `requires-python = ">=3.10"` (`:17`) | — | +| torch | `torch==2.11.0` (`:21`) | `torch>=2.4` (`:1`) | +| transformers | `transformers==5.8.0` (`:22`) | `transformers>=4.40` (`:2`) | +| datasets | `datasets==4.8.5` (`:23`) | unpinned | +| safetensors / accelerate / numpy | `0.7.0` / `1.13.0` / `2.4.4` (`:24-26`) | unpinned | +| pytest | `9.0.3` (`:31`) | — | + +Root `README.md:157` offers them as equivalent (`pip install -e . # or: pip install -r requirements.txt`) directly under `:156` *"Install pinned dependencies that produced the 0.0 logit-diff result."* — following the `requirements.txt` branch can install torch 2.4 / transformers 4.x, which is **not** the pinned-environment claim. `pyproject.toml:4-7` admits no lockfile exists and gives the `uv pip compile` command that was never run. **`Qwen3-0.6B/` has NO `requirements.txt`, NO `pyproject.toml`, NO lockfile** — its only install instruction is the unpinned one-liner `pip install torch transformers datasets safetensors accelerate` (`Qwen3-0.6B/README.md:497`). + +**Stamped from an actual execution in exactly ONE place** — the executed `SmolLM2-134(base)/results.ipynb`: cell 1 output `Torch: 2.11.0+cu130` / `Device: cuda | NVIDIA GB10`; `metadata.language_info.version = "3.12.11"` (generator `_build_notebook.py:50-51`). A repo-wide grep for `torch.__version__|torch.version.cuda|platform.python_version|sys.version` over `*.py` returns only that line plus one unrelated HybridSSM file. **No Qwen3 script and neither `verify.py` stamps any version.** + +**Live box (re-measured for this fact sheet):** python `3.12.11`, torch `2.11.0+cu130`, `torch.version.cuda` `13.0`, `torch.backends.cudnn.version()` `91900`, transformers `5.8.0`, datasets `4.8.5`; `nvidia-smi` → driver `580.142`, `NVIDIA GB10`. **These match `pyproject.toml` and the notebook stamp field-for-field** — SmolLM2 bit-exactness is currently re-checkable. Two gaps remain: cuDNN `91900` and driver `580.142` appear **nowhere** in the repo (grep over all `*.md/*.json/*.toml/*.txt/*.log` for `cuDNN|cudnn_version|580.14|CUDA 13|CUDA 12` returns one unrelated prose hit at `jax_vs_pytorch_tradeoffs.md:44`), and `torch==2.11.0` carries no `+cu130` local tag, so a CPU or CUDA-12 build satisfies the pin. + +**For Qwen3 the versions behind `verify.json` (2026-06-08) and `original_vs_repro.txt` (2026-06-09) are UNDETERMINABLE.** The only signal is the `torch_dtype` deprecation banner at `results/original_eval_run2.log:1`, which bounds transformers from below but names no version. + +**The paper appendix records hardware, dtype, seed, batch/step config and the commands — but no software versions.** `research/papers/qwen3-imu1-matched-compute/sections/reproducibility.tex:4-10`: *"a single NVIDIA GB10 (Grace Blackwell, unified ≈119 GB CPU+GPU memory) in bfloat16 with seed 0 … effective batch is 4 × 4 accumulation = 65,536 tokens, and both arms train for 18,150 steps."* No version block anywhere in the file. + +### 4.4 CPU-vs-GPU parity scope — **read carefully, the headline is CPU-only** + +| Claim | Device | dtype | Value | Source | +|---|---|---|---|---| +| Qwen3 `max_abs_error` | **CPU** | fp32 | **0.0** (relative 0.0; argmax `" Paris"`, id 12095, `argmax_match: true`, `passed: true`) | `Qwen3-0.6B/builds/.../results/verify.json` | +| SmolLM2 `max\|Δlogits\|` | **CPU** | fp32 | **0.000e+00** (relative 0.000e+00; argmax `" the"`, id 260) | `SmolLM2-134(base)/results/parity.log:6-9` | +| SmolLM2 final-logits parity | **GPU** | — | **4.72e-05** | `SmolLM2-134(base)/results/comparison_with_hf.md:10` | +| SmolLM2 per-layer (30 layers) | **GPU** | — | **1.95e-03 at layer 14** — **EXCEEDS the repo's own 1e-3 gate** | `comparison_with_hf.md:11` | +| SmolLM2 long-context RoPE | GPU | — | 4.01e-05 (labelled "401-token"; the code truncates at `max_length=512`, `compare_with_hf.py:180-181`) | `comparison_with_hf.md:14` | + +**[CORRECTED — the original fact-finding said "no GPU parity number is stamped anywhere". That is wrong: the GPU numbers exist, in prose.]** They are `prose-only` — the machine-written `results/comparison_with_hf.json` that `compare_with_hf.py:259-261` would produce is **absent from disk**, and `comparison_with_hf.md`'s mtime (2026-05-13 22:07:53) *predates* the notebook run (22:19–22:20), so it was not produced by that run. `results/README.md:3-4` claims *"Every file here is produced live … No values are typed in by hand"* — that blanket claim is **not supported** for `comparison_with_hf.md`. + +**So: the "max error 0.0 / bit-exact" claim is CPU-fp32-only. On GPU the reproduction is close but NOT bit-exact, and one per-layer delta trips the 1e-3 gate.** The repo itself explains this at `comparison_with_hf.md:22-42` (SDPA backend dispatch: HF passes an explicit mask, we pass `is_causal=True`) and devotes a section at `:49` to *"What the earlier ✗ at '1.953e-3' meant — and didn't mean."* `comparison_with_hf.md:51` also notes *"The threshold in `compare_with_hf.py` was `1e-3`, picked for bf16 tolerance"* — i.e. a loose gate for an fp32 claim. + +**Qwen3 has NO GPU parity check at all**, and no long-context or per-layer parity check anywhere. + +**Parity tolerance and input:** `assert max_abs < 1e-3` and `assert hf_next == our_next` — identical in five implementations: `Qwen3-0.6B/verify.py:74,81`; `SmolLM2-134(base)/verify.py:76`; `verify_run.py:33` (`TOLERANCE = 1e-3`); `tests/test_parity.py:73,98,132`; `compare_with_hf.py:235,240,249`. Input is a **single 5-token prompt, batch 1**: `"The capital of France is"` → `input_shape: [1, 5]` (`verify.json`). SmolLM2 token ids `[504, 3575, 282, 4649, 314]`. **This is a thin gate for a model card.** Note the two models' argmaxes differ (Qwen3 `" Paris"`, SmolLM2 `" the"`) — do not present one as shared. + +**Stale docstring:** `SmolLM2-134(base)/verify.py:6` says the logits match *"to bf16 numerical tolerance"* while the code and run are fp32; `Qwen3-0.6B/verify.py:6` correctly says fp32. + +### 4.5 Determinism flags + +**NONE. This is a definitive negative finding, verified twice with and without `--include` filters:** + +``` +grep -rn "allow_tf32|use_deterministic_algorithms|cudnn.deterministic|cudnn.benchmark|CUBLAS_WORKSPACE_CONFIG|set_float32_matmul_precision" → no output +``` +Not one of the six flags appears anywhere in the repo. The only `tf32` strings are an error message in `mfu_meter.py:115` and a `peak_fp32_tf32_tflops_assumed` constant in `research/systems/roofline_hybridssm.py`. Neither `verify.py` sets any seed or flag either — their complete setup is two imports plus a module constant (`Qwen3-0.6B/verify.py:13-19`). + +This matters precisely for the GPU deltas above: `comparison_with_hf.md` attributes them to unpinned backend dispatch, which is what a determinism flag would have controlled. TF32 on Blackwell is left at the PyTorch default and never recorded. + +**Seeds that ARE set (for the non-parity numbers):** `_build_notebook.py:45` `torch.manual_seed(0)` (the 15.371 notebook), `:201`/`:447` (`42`/`0`); `compare_with_hf.py:39`; Qwen3 trainer `train_qwen3.py:225-228` (`random.seed`, `np.random.seed`, `torch.manual_seed`, `torch.cuda.manual_seed_all`); SmolLM2 `train.py:95-96`, `train_tinystories.py:99-100`. Paper appendix records seed 0. + +--- + +## 5. Training details + +### 5.1 Qwen3-0.6B Phase B — the four ~1.19B-token runs + +| | Faithful baseline | IMU-1 / NorMuon (modernized) | partial-RoPE 0.25 | partial-RoPE 0.10 | +|---|---|---|---|---| +| Script / run_name | `train_qwen3.py` / `baseline2tpp` | `train_imu1.py` / `imu1_2tpp` | `train_partialrope.py` / `prope25_2tpp` | `prope10_2tpp` | +| Steps × tok/step | 18,150 × 65,536 | 18,150 × 65,536 | 18,150 × 65,536 | **died at step 5,450/18,150** | +| Total tokens | 1,189,478,400 | 1,189,478,400 | 1,189,478,400 | ~357M | +| seq_len / micro_batch / grad_accum | 4096 / 4 / 4 (= 16 seqs = 65,536 tok) | identical | identical | identical | +| Precision | **full bf16** — weights cast at construction; **no `torch.autocast`, no `GradScaler`, no fp32 master weights** (grep → 0 hits in all three trainers) | same | same | same | +| Cross-entropy | chunked, fp32 accumulator, chunk 8192 (`train_qwen3.py:87,91`) | **`train_imu1.py:38-45` does NOT `.float()` its CE chunks — CE accumulated in bf16. A real, unremarked between-arm numerical difference.** | fp32 chunked | fp32 chunked | +| Optimizer | AdamW, betas (0.9, 0.95), eps 1e-8 (`train_qwen3.py:302`) | **hybrid**: 224 2D non-embed → `NorMuon(lr=0.011, wd=0.1, beta1=0.95, beta2=0.95)`; 198 embed/1D → `AdamW(lr=0.006, betas=(0.9,0.95), eps=1e-8, wd=0.0)` (`train_imu1.py:82-85`; split measured at `qwen3_imu1_2tpp_train.log:2`) | AdamW | AdamW | +| LR schedule | cosine, peak **2.4e-3** → end 3.2e-4 (floor 0.13333), **warmup 900** | **WSD**: linear warmup **900** → stable → linear decay-to-zero over final 20% | cosine, 2.4e-3 → 3.2e-4 | same | +| Weight decay | 0.01, `dim>=2` only (`dim<2` → 0.0) | 0.1 on 2D; 0.0 on 1D | 0.01 | 0.01 | +| Grad clip | 1.0 (`clip_grad_norm_`, `:391`) | 1.0 | 1.0 | 1.0 | +| Extra loss | — | chunked z-loss, weight 1e-4 | — | — | +| Seed | 0 (argparse default; driver never passes `--seed`) | 0 | 0 | 0 | +| torch.compile | on | on | on | on | +| Wall-clock | **2,663.1 min = 44.4 h** (`log:395`) | ~63.9 h (DERIVED — no completion line) | ~46.1 h (DERIVED) | ~14.0 h | +| Throughput | **7,444 tok/s final** (cumulative avg; run span 7,414–7,483) **[CORRECTED — README.md:167's "7,480" is the step-100 reading]** | 5,172 tok/s | 7,168 tok/s | ~7,100 tok/s | +| Peak memory | 52.4 GB | 66.1 GB | 54.3 GB | 54.3 GB | + +**[CORRECTED — arithmetic error propagated from `Qwen3-0.6B/README.md:80`]** NorMuon is **30.5% lower throughput**, which is **+43.9% wall-clock** (63.9 h / 44.4 h = 1.439), **not "~30–31% more wall-clock"**. Do not copy README.md:80's phrasing. + +**[CORRECTED]** The IMU-1 CLI **is** on disk (`phase_b_driver.sh:24`), and it passes **`--warmup_steps 900`, not the script default 50**. Verified against the run's own LR ramp: 0.011 × 50/900 = 6.111e-4 = `log:5`'s `lr 6.11e-04`; 0.011 × 400/900 = 4.889e-3 = `log:12`'s `lr 4.89e-03`. Under warmup=50 the LR would be at peak by step 50. `normuon_lr 0.011 / adam_lr 0.006 / weight_decay 0.1 / decay_frac 0.2 / z_weight 1e-4` were not overridden, so those defaults do hold. + +**Iso-FLOP status: iso-TOKEN only.** No `train_flops` artifact exists for any Phase-B run (grep over `Qwen3-0.6B/builds/` → zero files), so the §C18 ≤5% gate was **never evaluated on disk** for the three-build comparison. `Qwen3-0.6B/README.md:81` asserts params are "iso-FLOP at 1.00043" — that is a parameter-count claim, not a FLOP artifact. + +**IMU-1 confound (5 variables at once, violating the repo's own one-variable rule):** optimizer + schedule shape (WSD vs cosine) + z-loss + architecture (`vr=True ln=True hg=True` — value residuals, LayerNorm scaling, head gating, `qwen3_imu1_2tpp_train.log:1`) + weight_decay 0.1 vs 0.01. + +**MFU: NOT_FOUND for these four runs.** `mfu_meter.py` exists; **[CORRECTED]** MFU *has* been computed elsewhere in the repo (`research/ledger/ledger.json:503` `"mfu": 0.2909` for the normuon-vs-adamw run; `Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/c5_evidence.json:11-14` `mfu 0.3209, achieved_tflops 40.11`) — but never for the 1.19B runs. The GB10 device peak is `estimated: True` (`mfu_meter.py:63-66`, 125.0 bf16-dense TFLOPs), so per `CLAUDE.md:19` a GB10 MFU must **never** be quoted as exact. + +**Chinchilla:** ~2 tokens/param (1.19B / 596,049,920). `builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md:20`: *"The paper used ~36T tokens; we use 131M (Phase A) to 1.19B (Phase B) … Chinchilla-optimal for 596M is ~12B tokens (20 tok/param); even Phase B is ~10× under-trained."* 36T/1.19B ≈ 30,252× less data (corroborated at `research/brutal_scorecard.md:57`). + +### 5.2 Qwen3 training corpus + tokenization + +- **HF dataset id:** `HuggingFaceFW/fineweb-edu`, config `sample-10BT`, split `train`, `streaming=True` — `train_qwen3.py:151`. **No `revision=` pinned.** The line is unchanged in the pre-`86e79f3` version that actually ran (confirmed via `git diff e791875 86e79f3`). +- **Tokenizer:** `AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base")` (`train_qwen3.py:59,281`). **[CORRECTED]** `151,936` is the **model config's `vocab_size`** (`Qwen3-0.6B/model.py:37`), not the tokenizer's vocabulary size — `len(tokenizer)` is **151,669** (`research/eval/private_heldout_v1/private_prose_v1.txt:456`). Write "model vocab_size 151,936". +- **Packing:** each doc encoded `add_special_tokens=False`, one EOS appended, ids concatenated into one flat stream; contiguous **non-overlapping** 4096-token windows (`PackedTextDataset`, `train_qwen3.py:110-123`); `DataLoader(shuffle=True, drop_last=True)` shuffles window order. **No cross-document attention masking.** 290,888 windows available, 290,400 consumed = **99.83% of exactly one epoch** (single-pass). +- The other three arms **loaded the same cache** the faithful run built (`tokcache_1191478400_300000.pt`, 9,534,229,373 B) — `qwen3_imu1_2tpp_train.log:3`, `qwen3_prope25_2tpp_train.log:2`, `qwen3_prope10_2tpp_train.log:2`. +- **No dataset card exists** for this corpus. `research/datasets/` holds only `data-selection-dclm-edu`, `grpo-math-prompts-v1`, `hybridssm-fineweb-edu`, `math-eval-v1`, `math-reasoning-openr1-math-220k` — none fed the 1.19B runs. + +### 5.3 SmolLM2 continued pretrain (the 6.8945 → 3.7900 run) + +| Field | Value | Source | +|---|---|---| +| Init | **official HF SmolLM2-135M safetensors** loaded into our class — **not from scratch** | `train_tinystories.py:39,145-149` | +| Corpus | `roneneldan/TinyStories` split `train` (2,119,719 stories) / `validation` (21,990) | `train_tinystories.py:154-155`; `results/tinystories_train.log:4` | +| Resolved dataset revision | `f54c09fd23315a6f9c86f9dc80f725de7d8f9c64` (cached 2026-05-13 14:16–14:20, ~7 h before the run) **[CORRECTED — not unrecoverable]** | `HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache`, `hub/datasets--roneneldan--TinyStories/refs/main` | +| Model revision | `93efa2f097d58c2a74874c7e644dbc9b0cee75a2` (cached 2026-05-13 11:44–11:47) **[CORRECTED]** | same HF_HOME | +| Packed train tokens | 102,000,116 (97.9 s) | `log:5` | +| Packed val tokens | 200,068 (first **1,040** non-empty stories, 0 empties skipped — re-derived by re-running the packer today) | `log:6` | +| Steps / budget | 24,414 steps → 99,999,744 of 100,000,000 tokens | `log:8,17`; `ck['step']`, `ck['tok_seen']` | +| tok/step | 4,096 (= 1024 × micro_batch × grad_accum) | `log:8`; formula `train_tinystories.py:189` | +| micro_batch / grad_accum | **PROSE-ONLY** — only the product 4,096 is logged. `micro_batch 4` appears at `results/POST_DATA.md:52` and as the current script's argparse default | — | +| seq_len | 1024 | argparse default | +| LR schedule | **WSD, MEASURED from the per-step trace**: linear warmup 200 → peak **3e-4**; stable through step 19,531; linear decay from 19,532 (2.9993856235920535e-4 = 3e-4·(1−1/4883)) to **0.0** at 24,414 | `results/tinystories_train.csv:2,201,19532,19533,24415`; shape `train.py:45-56` | +| Optimizer / betas / eps / wd / clip / seed | **NOT_FOUND in any run artifact.** AdamW, (0.9, 0.95), 1e-8, 0.01 on `dim>=2`, clip 1.0, seed 0 exist ONLY in prose + the *current* script's argparse defaults | see the drift block below | +| Precision | bf16 (`log:1`), logits `.float()` before CE, `reduction="sum"` | `train_tinystories.py:70` | +| Wall-clock | **116.1 min** loop (process 21:23:28 → 23:21:27 ≈ 118.0 min) | `log:506` | +| Throughput | 14,356 tok/s (**cumulative average**, rose 12,766 → 14,356 over the run) | `log:505` | +| GPU | `Device: cuda` only — **"NVIDIA GB10" is PROSE for this run** (`README.md:62`, `POST_DATA.md:17`); `nvidia-smi` today confirms the box's GPU but that is present-day corroboration | `log:1` | +| Peak memory | not recorded (run-era CSV has no `peak_mem_mb` column) | — | + +**Script drift — the on-disk `train_tinystories.py` is NOT the version that produced this run.** Six independent proofs: (1) the script writes a 6-column CSV header (`:256`) but `results/tinystories_train.csv:1` is `step,loss,lr,tok_seen`; (2) `:139` logs `device=… dtype=… seed=…`, the log reads `Device: cuda dtype: torch.bfloat16`; (3) `:140` logs `args={vars(args)}` — `grep -c "args=" log` → 0; (4) defaults `--eval_every 2000 --ckpt_every 2000` would emit 12 and 7 marker lines — `grep -c` → 0 and 0; (5) `save_ckpt` writes `training_recipe/optim/sched/rng_*`, the actual checkpoint has none of them (`ck.keys() == ['model','config','step','tok_seen','baseline_ppl','trained_ppl']`); (6) `:197` formats `PPL={base_ppl:.3f} ({base_n:,} target tokens)` — a different template from `log:10`. mtimes: script 2026-05-19 23:16, results 2026-05-14 00:21. `git log` on the file → single commit `84a96c0`, whose blob is **identical to the working tree**, i.e. git holds only the later version. **The run's exact source is unrecoverable.** + +**[CORRECTED — important]** `SmolLM2-134(base)/results/training_recipe_resolved.json` DOES list AdamW / betas [0.9,0.95] / eps 1e-8 / weight_decay 0.01 / clip_grad 1.0 — but its own line 2 declares `"source": "https://github.com/huggingface/smollm/blob/main/text/pretraining/smollm2/config_smollm2_135M.yaml"`, `"fetched": "2026-05-13"`, and its values are the **UPSTREAM FROM-SCRATCH nanotron config** (lr 0.003, warmup 2000, seq_len 2048, global_batch 512, tokens/step 1,048,576, 2,000,000 steps, ~2.097T tokens, implied_data_parallel 64), **not this run**. `results/tinystories_summary.md:80-82` likewise sources wd/clip to "nanotron config" and the optimizer to "paper §4.1". **These hyperparameters are COPIED FROM AN EXTERNAL CONFIG and must not go on a card as measured.** + +**Two different TinyStories runs are described in the repo.** `results/tinystories_summary.md` documents an **earlier** run: after-PPL **3.7893** (`:9`), wall clock **137.3 min** (`:14`), mean 12,150 tok/s (`:90`), "max temp 72 °C … 5 users on the box" (`:92`). `POST_DATA.md:165` labels it `(prior run)`. Its recipe table (`:67-82`) and throughput table (`:84-92`) **must not be quoted as the 116.1-min run's**. **[CORRECTED]** All three BEFORE generations are byte-identical to `tinystories_before.txt` and the prompt-1 AFTER sample is byte-identical to `tinystories_after.txt`; only prompts 2 and 3 diverge. + +**Derived-statistic check:** best single-batch loss **0.9087928533554077 @ step 22,353** — CONFIRMED (`csv:22354`). First 1000-step bucket mean **1.5860** — CONFIRMED. **`POST_DATA.md:57`'s "1.316 (last)" is off by one bucket**: (23000,24000] = 1.3162; the true last bucket (24000,24414], 414 rows, = **1.3138**. + +### 5.4 SmolLM2 from-scratch (`checkpoint.pt`) — a 150-step demo, not a reproduction + +`train.py:140` `print("Initializing model from scratch (random init)...")`; corpus `Salesforce/wikitext` / `wikitext-103-raw-v1` / `train` (`train.py:78`); seq_len 2048, micro 2 × accum 8 = 32,768 tok/step; AdamW (0.9,0.95) eps 1e-8, peak lr 3.0e-3, wd 0.01 on `dim>=2`, WSD warmup 20 / 20% decay, clip 1.0, bf16. Only recorded run: **150 steps** (~4.9M tokens, DERIVED from defaults — nothing on disk records the demo's batch shape), final loss **6.288341** from start **11.254480** (`results/loss_curve.csv`, 151 lines; `results/summary.json`). `train.py:13-18` calls it a single-GPU starter, not a reproduction. + +**Naming trap:** the notebook's 150-step demo cell header and surrounding prose say "wikitext-103 slice" while the code at `_build_notebook.py:436` loads **wikitext-2-raw-v1 train**. Do not write "trained on wikitext-103" for the notebook demo. + +--- + +## 6. Weights: what exists on disk + +**Totals (independently re-measured with `os.walk` + `getsize`, symlinks excluded, >1 MB):** **107 weights-bearing files, 290,643,004,043 B = 270.68 GiB = 290.64 GB.** Split by extension: `.pt` n=89 (215.08 GiB), `.pkl` n=17 (52.51 GiB), `.discarded_*` n=1 (3.10 GiB). **Zero `.safetensors`, `.msgpack`, `.npz`, `.pth`, `.ckpt` anywhere in the repo.** Everything the naive `find` pattern matches totals 315,669,745,533 B = 293.99 GiB, because it also catches 26 `tokcache_*.pt` **token caches** (23.92 GB — the largest single `.pt` in the repo, 9,534,229,373 B = 8.88 GiB, is a token cache, not a model) and 10 `research/datasets/**/*.bin` **uint32** token shards (1.11 GB). Volume: 2.5 T free of 3.7 T. + +### 6.1 Inventory — the publishable candidates + +| File | Size | Format | In-file keys | config.json? | tokenizer files? | Publishable? | +|---|---|---|---|---|---|---| +| `SmolLM2-134(base)/checkpoint_tinystories.pt` | 269,144,681 B (256.68 MiB) | torch.save zip, `compression method=store` (a **pickle**) | `model, config, step, tok_seen, baseline_ppl, trained_ppl` | **NO** — config is the in-file `config` dict | **NO** | **YES** — a fine-tune of `HuggingFaceTB/SmolLM2-135M` (Apache-2.0 upstream); card MUST say so | +| `SmolLM2-134(base)/checkpoint.pt` | 538,173,921 B | torch.save zip, fp32 | `model, config, losses, lrs, step` (step=150) | NO | NO | **NO** — 150-step random-init demo | +| `Qwen3-0.6B/builds/.../checkpoint_qwen3_baseline2tpp.pt` | ~1.19 GB class | pickle | `model, config, step, tok_seen, arm, seed, fineweb_val_ppl, baseline_ppl, recipe` | NO | NO | Maybe — but see §9 (leak-suspect val, n=1) | +| `.../checkpoint_imu1_2tpp_step18000.pt` | 1,193,196,711 B | pickle | `model, config, step` (no `tok_seen`) | NO | NO | **Loads into stock HF Qwen3? NO** — 752,091,220 elems, config carries `use_value_residual/use_layernorm_scaling/use_head_gating`, **and its 423 state-dict keys are prefixed `_orig_mod.`** (torch.compile) | +| `.../checkpoint_prope10_2tpp_.pt` / `_prope25_` | 1,192,229,775 B (step 4000) | pickle | `model, config, step` | NO | NO | **[CORRECTED]** `partial_rotary_factor: 0.1` is **NOT supported by transformers 5.8.0 Qwen3** — grep over `site-packages/transformers/models/qwen3/` returns nothing; `Qwen3Config.__init__` has no such parameter; empirically `Qwen3RotaryEmbedding(Qwen3Config(head_dim=128, partial_rotary_factor=0.1))` yields `inv_freq` of length 64, identical to the default = **full RoPE**. Loading it into stock Qwen3 silently runs the wrong architecture. | +| 6× `checkpoint_{adamw,normuon}_seed{0,1,2}.pt` (42M cohort) | 1,192,229,527 / 1,192,230,159 B | pickle, `step=640, tok_seen=41,943,040` | + `arm`, `seed`, `fineweb_val_ppl` | NO | NO | Research artifacts, not a model release | +| 12× `checkpoint_persist_{168M,420M}_{adamw,normuon}_s{0,1,2}.pt` | 1,192,232,687 / 1,192,233,319 B | pickle | same | NO | NO | Ditto. **`168M`/`420M` are TOKEN BUDGETS — all are the same 596,049,920-param model.** | +| 18× HybridSSM `*.pkl` | 55.60 GiB total; **params-only = 18,792,709,255 B = 17.50 GiB** | **Python pickle wrapping flax msgpack**: `{params: bytes, opt_state: bytes (exactly 2× params), step: int, rng: uint32[2] (post-fix only)}` | — | NO | NO | Only `checkpoint_ssm_base_s0.pkl` has a §C10 suite score; 4 are quarantined/void (below) | +| **Nothing** | — | — | — | — | — | **There is NO "parity-verified reproduction" checkpoint.** Both `verify.py` scripts download the official weights at runtime and save nothing. What is publishable is the *code + the parity artifact*, not a weights file. | + +**Format warning for any consumer: every checkpoint in this repo is a `torch.save` / `pickle.dump` file, not safetensors.** The 3.4 GB full-training-state Qwen3 variants (keys add `optim, sched, rng_torch, rng_cuda, rng_numpy, rng_python`) **fail `torch.load(weights_only=True)`** with *"Unsupported global: GLOBAL numpy._core.multiarray._reconstruct was not an allowed global by default"*; they load only under `torch.serialization.safe_globals([...])`. + +**Structure of the Qwen3 state dict:** 311 keys, all `torch.bfloat16`, 751,632,384 tensor elements including the tied `lm_head.weight` duplicate = **596,049,920 unique params**. SmolLM2: 273 keys, bf16, 162,826,560 elements = **134,515,008 unique** (49,152 × 576 = 28,311,552 duplicated). + +**HybridSSM params (measured by deserializing the flax tree):** `ssm_base` 305,818,368 (266 leaves) · `attn1to3` 324,867,840 (290) · `fullattn` 267,719,424 (218) · `swa128` 277,156,608 (194). All fp32, tied embed 151,936 × 768 = 116,686,848, no separate `lm_head`. Config `d_model 768, n_layers 24, n_heads 12, n_kv_heads 4, vocab 151,936` (`HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py:21-26`). **The folder is named "-0.2B" but no arm is 0.2B total; 0.2B ≈ non-embedding (ssm_base 189,131,520). No file on disk states the convention.** + +### 6.2 Must NOT be published + +- **Hard-quarantined:** `HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/checkpoint_swa128_nope_85M_s0.pkl.discarded_rng_confound_20260723` (3,325,901,386 B). +- **Comparability-void (iso-FLOP error, +18.88% extra compute):** `checkpoint_swa128_42M_s0.pkl`, `checkpoint_swa128_nope_42M_s0.pkl`, `checkpoint_swa128_85M_s0.pkl`. `c5_evidence_CORRECTION_2026-07-28.md:326-328`: *"The words \"iso-FLOP\" must not be attached to `swa128` or `swa128_nope` in any artifact until those cells are re-run at 4,929 steps / 40,378,368 tokens."* Replacements: `*_isofix_s0.pkl` (both step 4,929). +- **Undocumented second split (my own measurement, not a repo classification):** 10 of the 18 HybridSSM pickles lack the `rng` key (pre-PRNG-fix), 8 carry it. The RNG-confound rationale that quarantined the 85M file applies structurally to every pre-fix file that was resumed. Only one was actually quarantined. +- **7 smoke-test artifacts, 18.88 GiB of pure overhead:** `smoke_{baseline,baseline_resumed,wsd,zloss,arch}.pt` in `2026-06-18_qwen3-0.6b_imu1-deconfound-p1` (16.66 GiB) + `checkpoint_imu1_smoke_step{500,1000}.pt`. +- **12 symlinks** (§C13 control reuse, zero extra disk) with **absolute** paths under `/home/yashb98/Downloads/BuildFromScratch` — they break on any copy/move. Do not `tar` them without dereferencing. + +### 6.3 Recommendation: `trust_remote_code` modeling file vs raw checkpoint + +**Ship the SmolLM2 TinyStories checkpoint as a standard HF `LlamaForCausalLM` in safetensors. Do NOT ship raw `.pt` pickles, and do NOT reach for `trust_remote_code` for SmolLM2.** + +Rationale, all from disk: +1. **SmolLM2 needs no custom code.** Its architecture is stock Llama — the repo's own exporter round-trips into `LlamaForCausalLM` (`SmolLM2-134(base)/scripts/export_to_hf.py:56-59`) and the config comes straight from `AutoConfig.from_pretrained("HuggingFaceTB/SmolLM2-135M")`. A `trust_remote_code` release would force every downloader to opt into executing our Python for zero architectural benefit, and our `.pt` files are **pickles** — the exact format `safetensors` exists to avoid. +2. **Qwen3 faithful is likewise stock** — it is a bit-exact re-implementation of an architecture `transformers` already ships. Convert, don't vendor. +3. **`trust_remote_code` is only justified for architectures HF cannot express**: the **modernized/IMU-1** arm (`use_value_residual`, `use_layernorm_scaling`, `use_head_gating`), the **partial-RoPE** arms (the kwarg is silently ignored by transformers 5.8.0 — publishing them without custom modeling code would ship a *wrong* model), and **HybridSSM** (a novel JAX/Flax architecture with no HF class at all). For those, either write a `modeling_*.py` or do not publish weights. +4. **No exporter exists for Qwen3 or HybridSSM.** `export_to_hf.py` is the only conversion script in the repo (a repo-wide grep for `save_pretrained|save_file` over `*.py` finds no other), it covers SmolLM2 only, requires network, and **has never been run to completion** — `hf_export/` does not exist on disk. + +**Loader snippet — the export path that actually exists (SmolLM2):** +```bash +cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" +python3 scripts/export_to_hf.py --ckpt checkpoint_tinystories.pt --out hf_export/smollm2-135m-tinystories +``` +Which internally does (`scripts/export_to_hf.py:56-67`): +```python +from transformers import AutoConfig, AutoTokenizer, LlamaForCausalLM +cfg = AutoConfig.from_pretrained(args.repo) # HuggingFaceTB/SmolLM2-135M +hf = LlamaForCausalLM(cfg) +missing, unexpected = hf.load_state_dict(ours_sd, strict=False) +# :60-64 filters the tied lm_head.weight and raises SystemExit on any other mismatch +hf.save_pretrained(out, safe_serialization=True) +tok = AutoTokenizer.from_pretrained(args.repo) +tok.save_pretrained(out) +``` +Then the consumer side is ordinary: +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +model = AutoModelForCausalLM.from_pretrained("/smollm2-135m-tinystories") # no trust_remote_code +tok = AutoTokenizer.from_pretrained("/smollm2-135m-tinystories") +``` + +**If a raw checkpoint must be consumed directly** (the repo's own pattern, `SmolLM2-134(base)/eval_after_vs_base.py:42-45`): +```python +import torch +from model_full import SmolLM2ForCausalLM, SmolLM2Config +m = SmolLM2ForCausalLM(SmolLM2Config()) +ck = torch.load("checkpoint_tinystories.pt", map_location="cpu", weights_only=False) # PICKLE +m.load_state_dict(ck["model"]) +m = m.to(device="cuda", dtype=torch.bfloat16).eval() +``` +For Qwen3 checkpoints add the compile-prefix strip that the eval suite uses (`Qwen3-0.6B/experiments/2026-06-16_qwen3-0.6b_eval-faithful/eval_suite.py:97-114`): +```python +ck = torch.load(path, map_location="cpu", weights_only=False) +sd = ck["model"] if isinstance(ck, dict) and "model" in ck else ck +sd = {k.removeprefix("_orig_mod."): v for k, v in sd.items()} # torch.compile-trained +model.load_state_dict(sd, strict=True) +``` + +**Backup status: none.** Zero checkpoints are tracked in git (`git ls-files | grep -cE '\.(pt|pth|bin|safetensors|ckpt|pkl|msgpack|npz)$'` → **0**; `.gitignore:19-24`, `HybridSSM-0.2B/.gitignore:2`). No HF Hub copy, no `hf_export/`. The 269 MB TinyStories weights are the **sole copy** of the −45% result. (The surrounding evidence is safe — `tinystories_train.{log,csv}`, `tinystories_{before,after}.txt`, `POST_DATA.md`, `training_recipe_resolved.json` are all git-tracked.) + +**No checksums.** **[CORRECTED]** 27 of 29 ledger runs have `lineage.artifact_sha256 = null`; the two exceptions are `2026-06-16_qwen3-faithful_eval-first` (`"checkpoint_qwen3_baseline2tpp.pt@step18150"` — a filename, not a hash) and `2026-07-29_hybrid-ssm-0.2b_fineweb-edu-carding` (`c83b7d608a0ca320ae7b7e41dbee05282f074a004a87a9a90f2f4fd0f5032491`, a dataset-carding run). **No model checkpoint on disk has a verifiable checksum.** + +--- + +## 7. Loader API (real code) + +**There is no HF-style API.** Neither model class has `from_pretrained` / `save_pretrained` (grep for `from_pretrained` in `Qwen3-0.6B/model.py` matches only a comment at `:33`; zero hits in `model_full.py`). Neither folder has an `__init__.py`, so **every snippet must `cd` into the model folder or `sys.path.insert` it**. (Note the SmolLM2 folder name contains parentheses and must be quoted in any shell.) + +| | Qwen3 | SmolLM2 | +|---|---|---| +| Model class | `Qwen3ForCausalLM(nn.Module)`, `__init__(self, cfg: Qwen3Config)` (`model.py:237-246`) | `SmolLM2ForCausalLM(nn.Module)`, `__init__(self, cfg: SmolLM2Config)` (`model_full.py:238-248`) | +| Config | `@dataclass Qwen3Config` — **14 fields**, all defaulted; `head_dim` is a real settable field (`model.py:35-51`) | `@dataclass SmolLM2Config` — **13 fields** **[CORRECTED: not 12]**, all defaulted; **`head_dim` is a read-only `@property`** (576//9 = 64) — `SmolLM2Config(head_dim=64)` raises TypeError (`model_full.py:28-49`) | +| Weight loader | `load_official_weights_into_ours(ours, hf_state_dict)` in **`verify.py:22`** (not model.py). No key remapping — module names mirror HF exactly; `load_state_dict(strict=False)` then assert only `lm_head.weight` missing and nothing unexpected (`verify.py:39-44`) | same function at `SmolLM2-134(base)/verify.py:22`, body `:39-46` | +| forward | `forward(input_ids, labels=None, attention_mask=None) -> {"logits": (B,T,151936), "loss": scalar-or-None}` — a **plain dict**, indexed `model(x)["logits"]` (`model.py:259-274`) | identical contract, vocab 49,152 (`model_full.py:263-279`) | +| generate | `@torch.no_grad() generate(input_ids, max_new_tokens=64, temperature=0.8, top_k=50) -> Tensor` (prompt+continuation). `temperature<=0` → greedy. **No KV cache — recomputes the prefix each step** (`model.py:276-295`) | same defaults (`model_full.py:281-…`) | +| Tokenizer repo | `Qwen/Qwen3-0.6B-Base` (`verify.py:19`) | `HuggingFaceTB/SmolLM2-135M` (`verify.py:19`) | +| `safe_cuda` dependency | **NONE** in the model file or `verify.py` (grep exit 1). It is a caller-side §C1 obligation honoured by the training/eval scripts (`train_qwen3.py:42,269`; `eval_suite.py:43,202`). `safe_cuda.guard(fraction=0.85, device=0)` no-ops without CUDA (`safe_cuda.py:47,51-52`) | **zero `safe_cuda` hits in the entire `SmolLM2-134(base)/` tree** | + +**Two silent traps a card must warn about:** +1. **`attention_mask` disables causal masking.** Both models: `F.scaled_dot_product_attention(q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=(attention_mask is None))` (`Qwen3-0.6B/model.py:162-167`; `model_full.py:156-161`). Passing an HF-style 2-D padding mask **turns off causality**. Published snippets must not pass `attention_mask`. +2. **`attention_dropout` is dead config.** Declared at `Qwen3-0.6B/model.py:50` and `model_full.py:42`; grep finds **no read site** in either file — dropout is hardcoded to 0.0. Setting it has no effect. + +### 7.1 Faithful usage snippet — SmolLM2 + +Copied from `SmolLM2-134(base)/generate.py` (the repo's own 30-line end-to-end script) with per-line provenance: + +```python +# cd "SmolLM2-134(base)" — flat-module imports, no __init__.py +import torch # generate.py:7 +from transformers import AutoTokenizer, AutoModelForCausalLM # generate.py:8 +from model_full import SmolLM2ForCausalLM, SmolLM2Config # generate.py:10 +from verify import load_official_weights_into_ours, REPO # generate.py:11 (REPO = "HuggingFaceTB/SmolLM2-135M", verify.py:19) + +tokenizer = AutoTokenizer.from_pretrained(REPO) # generate.py:15 +hf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) # eval_after_vs_base.py:35 (generate.py:16 uses the deprecated torch_dtype=) +model = SmolLM2ForCausalLM(SmolLM2Config()) # generate.py:18 +load_official_weights_into_ours(model, hf.state_dict()) # generate.py:19 +del hf # generate.py:20 +model.eval() # generate.py:21 + +input_ids = tokenizer("The capital of France is", return_tensors="pt").input_ids # call generate.py:23; prompt verify.py:62 +with torch.no_grad(): # ADDED — the repo runs this under @torch.no_grad() (verify.py:49) + logits = model(input_ids)["logits"] # verify.py:66 + next_id = logits[0, -1].argmax().item() # verify.py:80 +print(tokenizer.decode([next_id])) # simplified from verify.py:82 → " the" + +out = model.generate(input_ids, max_new_tokens=64, temperature=0.8, top_k=50) # generate.py:24 +print(tokenizer.decode(out[0], skip_special_tokens=True)) # generate.py:25 +``` + +### 7.2 Faithful usage snippet — Qwen3 + +**There is no equivalent standalone script for Qwen3** — this is composed from `verify.py` + an experiment `eval_suite.py`, and the composition is flagged: + +```python +# cd "Qwen3-0.6B" +import torch # verify.py:13 +from transformers import AutoModelForCausalLM, AutoTokenizer # verify.py:14 +from model import Qwen3ForCausalLM, Qwen3Config # verify.py:16 +from verify import load_official_weights_into_ours, REPO # COMPOSED (import form from SmolLM2 generate.py:11); both names real at verify.py:22 and :19; verify.py:86 is __main__-guarded so the import is safe + +tokenizer = AutoTokenizer.from_pretrained(REPO) # verify.py:50 (REPO = "Qwen/Qwen3-0.6B-Base") +hf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) # verify.py:51 +ours = Qwen3ForCausalLM(Qwen3Config()) # verify.py:55 +load_official_weights_into_ours(ours, hf_model.state_dict()) # verify.py:56 +ours.eval() # verify.py:57 + +text = "The capital of France is" # verify.py:60 +input_ids = tokenizer(text, return_tensors="pt").input_ids # verify.py:61 +with torch.no_grad(): # ADDED — verify.py:47-48 decorates main() with @torch.no_grad() + our_out = ours(input_ids)["logits"] # verify.py:64 + our_next = our_out[0, -1].argmax().item() # verify.py:78 → 12095 = " Paris" +print(tokenizer.decode([our_next])) # simplified from verify.py:80 + +out = ours.generate(input_ids, max_new_tokens=60, temperature=0.7, top_k=40) # call form eval_suite.py:190-191; literals eval_suite.py:68 +print(tokenizer.decode(out[0], skip_special_tokens=True)) # eval_suite.py:192 +``` +Cost note: this loads the full fp32 HF model plus a second full copy (~2 × 596M × 4 B ≈ 4.8 GB), CPU-only, so no `safe_cuda.guard` is needed — matching `verify.py`, which has none. With no KV cache, 60 new tokens = 60 full fp32 CPU forwards of a 596M model — minutes-slow. + +--- + +## 8. CANNOT ANSWER FROM DISK / open gaps + +1. **No git tag, and no commit hash stamped in any results file.** `git tag -l` empty; `verify.json` / `perplexity.json` / `parity.log` carry no commit, versions, device, or timestamp. Best anchors (`e791875`, `84a96c0`) both postdate the artifacts they added. +2. **No dataset revision pinned for `HuggingFaceFW/fineweb-edu` sample-10BT.** `train_qwen3.py:151` passes no `revision=`. The exact snapshot behind the 1.19B tokens and behind 13.40/28.65/23.52/46.31 is unrecoverable. (A pinned sha `87f09149ef…` exists at `research/eval/private_heldout_v1/private_prose_v1.txt:455` but belongs to a *later* dataset-forge prep.) +3. **No dataset revision pinned for the SmolLM2 wikitext-2 PPL.** `_build_notebook.py:233` has no `revision=`. (For TinyStories and the SmolLM2 model the resolved shas *were* recovered from the active `HF_HOME`; for wikitext they were not.) +4. **The exact `torch`/`transformers` versions behind Qwen3's `verify.json` (2026-06-08) and `original_vs_repro.txt` (2026-06-09) are undeterminable.** Qwen3 has no requirements file, no script stamps a version, no ledger entry covers those runs. Same for SmolLM2's `parity.log` (2026-05-13) — only its same-day sibling notebook carries a stamp. +5. **cuDNN 91900 and driver 580.142 are recorded nowhere in the repo.** Version drift in those two is undetectable from disk. +6. **The optimizer hyperparameters of the SmolLM2 100M-token TinyStories run** (weight_decay, betas, eps, grad_clip, seed) — no `args=` line in the log, no `training_recipe` key in the checkpoint, no `grad_norm` column in the CSV, and git holds only a *later* script. The cited values are that later version's argparse defaults, plus an external nanotron config. +7. **The micro_batch / grad_accum factorization for that run** — only the product (4,096 tok/step) is logged. +8. **The exact CLI invocation of the SmolLM2 TinyStories run** was never recorded anywhere. +9. **The GPU model for that run** — `log:1` says only `Device: cuda`. "NVIDIA GB10" is prose. +10. **Peak GPU memory for that run** — no `peak_mem_mb` column in the run-era CSV. +11. **The exact source code that produced the SmolLM2 runs is gone** (overwritten 2026-05-19; git holds only the later version). +12. **The six GPU-side SmolLM2 parity numbers have no machine-written backing** — `results/comparison_with_hf.json` is absent; `compare_with_hf.py:259-261` would create it on re-run. The "401-token RoPE" label contradicts `max_length=512` in the code. +13. **`results/tinystories_vs_base.{md,json}` do not exist** — so **no OOD / catastrophic-forgetting / downstream number exists for `checkpoint_tinystories.pt`**. `results/lm_eval/` does not exist either: `scripts/run_lm_eval.sh` has never run (its own `mkdir -p` never fired). **No downstream benchmark (HellaSwag/ARC/MMLU/…) was ever measured for SmolLM2 in this repo.** +14. **No suite-comparable counterpart to 13.40** — the `text-lm-v2` suite was never run against the released Qwen3-0.6B-Base. +15. **No same-slice re-measurement of the released model on `tokcache_1191478400_300000.pt`** — so the true same-slice gap between 28.65/23.52 and the released model is **unknown**. +16. **Whether the released Qwen3-0.6B-Base saw these FineWeb-Edu val documents during its 36T pretraining is undeterminable.** No overlap test was or could be run here. +17. **No `train_flops` artifact for any Phase-B run** — the §C18 iso-FLOP gate was never evaluated for the three-build comparison. +18. **No MFU for any of the 1.19B runs**, and the GB10 device peak is `estimated: True` regardless. +19. **The Phase-B training runs have no ledger entries at all** (only their downstream evals do) — no ledger-recorded wall_clock, gpu_hours, git_commit, `c5_evidence.json`, or `verdict.json`. No smoke-test artifact either. +20. **No 840M rung on the scaling ladder.** `SEEDS` in `score_ladder.py:49` still declares 840,000,000 but no `checkpoint_persist_840M_*.pt` exists. The trend fit rests on exactly 3 budgets. +21. **No per-horizon LR re-tuning anywhere on disk.** Both AdamW 2.4e-3 and NorMuon 0.011 were tuned at 42M and held fixed at 168M/420M (`verdict.json:257`: *"Inherited confound: AdamW/NorMuon LRs tuned at 42M, not re-tuned per horizon"*). Part of the observed fade could be an LR artifact; nothing on disk separates the two. +22. **Three §C25 HARD scaling-battery items were never computed** — `log_rmse_r2`, `holdout_extrapolation_pctdev`, `bootstrap_forecast_ci` (`verdict.json:210-214`). A §C26 figure for the ladder is also missing. +23. **No ledger detail doc for the ladder run itself** — `research/ledger/runs/` has no `2026-07-05_qwen3-0.6b_scaling-persistence.md`. +24. **Nothing measures whether the NorMuon convergence holds at model sizes other than N=596M.** +25. **HybridSSM ladder scores cannot be traced to specific `.pkl` files** — `arch_ladder_scores.json` contains zero checkpoint filenames (regex sweep → empty). The only link is the cell id via `run_arch_ladder.sh:395`. +26. **Whether "HybridSSM-0.2B" means 0.2B total or 0.2B non-embedding params** — no file states the convention. +27. **License/provenance for redistributing HybridSSM weights** — novel architecture but Qwen3 tokenizer + FineWeb-Edu training data; no LICENSE or data-license record beside the checkpoints. +28. **Whether the 8 resumed pre-PRNG-fix HybridSSM checkpoints carry the same confound** that quarantined the 85M one — no assessment document exists; the rng-key partition is my measurement, not a repo classification. +29. **No SHA256 for any model checkpoint** (see §6). +30. **`research/ledger/ledger.json` is currently uncommitted-modified** (`git status`), so ledger values quoted here are working-tree state, not committed state. **The scaling-ladder `verdict.json` and `ladder_bpb.json` — the null's entire evidence — are NOT git-tracked**, while the 42M "win" evidence IS. +31. **`model.py` does not exist in `SmolLM2-134(base)/`** (only `model_full.py`), yet `results/POST_DATA.md:20` cites `wc -l model.py` for a "198 lines" figure, `results/README.md:25` says `param_count.log` is the "output of `python3 model.py`", `results/README.md:4` credits `../model.py`, and `eval_after_vs_base.py:91` would `FileNotFoundError` on it. **The "198 lines" claim and the `python3 model.py` reproduce instruction are unbacked at any line number.** (The param count itself IS backed: `results/param_count.log:1-2` and `tests/test_parity.py:53`.) +32. **`Qwen3-0.6B/README.md:39-40` mis-sources the param count** to `verify.json`, which has no params field. 596,049,920 is genuinely measured, but at `Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/run_sft_seed0.log:7`. Similarly `SmolLM2-134(base)/README.md:60` attributes the token ids `[504, 3575, 282, 4649, 314]` to `results/summary.json`, which has no tokenization key — they come from `results.ipynb` cell 6. +33. **No HF model card, `MODEL_CARD.md`, or exported HF repo exists anywhere under the repo root** — there is no existing Reproduce section on disk to diff these commands against. +34. **`Qwen3-0.6B/model.py:44` hardcodes `max_position_embeddings = 40_960` with the comment "config.json: max_position_embeddings", but the cached `Qwen/Qwen3-0.6B-Base` config.json on this box (snapshot `da87bfb608c14b7cf20ba1ce41287e8de496c0cd`, dir mtime 2026-06-08 — the very date `model.py:33` cites) says `32768`.** 13 of 14 defaults match; this one does not. Harmless for weight loading (RoPE buffers are `persistent=False`, so parity still passes) but **the claim "every default matches the Base config.json" must not be repeated, and 40,960 must not be published as the Base context length.** I did not fetch live HF HEAD, so this mismatch is asserted only against the on-disk snapshot. + +--- + +## 9. Reviewer red flags — what would embarrass the author if published as-is + +Ranked by how fast a reviewer finds them. + +1. **`Qwen3-0.6B/README.md:35-37` states a falsehood that one `grep` disproves:** "identical eval code on the identical 300k-token FineWeb-Edu val slice … so every row is directly comparable." Two different caches, different sha1, different leading tokens. The derived "2.14×" and "1.76×" gaps are cross-slice. Duplicated at `results_overview/plots/README.md:50`. +2. **A `null` verdict is being advertised as a `significant win` in four places.** `Qwen3-0.6B/README.md:52, :175, :243` and `PLOTS_INDEX.md:73` all sell −0.474 bpb as a win with zero mention of the ladder, CONVERGES, or the null. The source run's own `RESULT.md:7` scoped it honestly; the README dropped the qualifier. The 42M ledger entry still says `"verdict": "win"` with a now-false caveat `"no scaling curve"` (`ledger.json:512`), and the ladder run is missing from the technique's `run_ids` so **a ledger query by technique will not surface the null**. +3. **The root `README.md:105-121` account of the ladder is stale in five specifics** (n=2, "+0.073 [−0.038, +0.184]", "not significant at the top", "code_py … +0.192", "slope −0.328 (r² 0.81)") and concludes "Verdict: directional, not a headline — the 420M rung is n=2". It is now n=3 and the top rung IS significant. Two of the repo's own top-level docs disagree with each other and with `verdict.json`. +4. **"Bit-exact / max error 0.0" is CPU-only, and the repo's own GPU numbers break its own gate.** `comparison_with_hf.md:11` records **1.95e-03** per-layer at layer 14 on GPU against a 1e-3 assert. Any card saying "bit-exact" without "CPU fp32, 5-token prompt" is misleading — and the six GPU numbers have **no** machine-written backing file, while `results/README.md:3-4` claims "Every file here is produced live … No values are typed in by hand." +5. **The headline PPL recipes are non-standard in ways that make the numbers incomparable to published values.** SmolLM2's 15.371 uses an overlapping 1024/512 window with **no `-100` masking**, so 62,403 "target tokens" are really 31,743 distinct positions double-counted, over **the first ~11.8% of the split**, with blank rows filtered out of the join. The BPB suite likewise caps at MAX_WINDOWS=200 = the first ~103k tokens, ~99% double-counted. Both prose sites (`results/README.md:38-39`, `POST_DATA.md:34-36`) state "62,403 target tokens" with no qualification. +6. **15.371 is not a number about a model this repo trained.** It characterizes the *official* SmolLM2-135M checkpoint under a nonstandard recipe. Attaching it to either local checkpoint (`checkpoint.pt` = 150-step demo; `checkpoint_tinystories.pt` = TinyStories) would be flatly wrong. +7. **Every Qwen3 headline PPL sits on a leak-suspect val split the repo itself indicts.** `train_qwen3.py:131-136` calls the old splitter's val "the sequential continuation of train, leak-suspect"; the fix landed *after* all four runs. `decontam_report.json` (2026-07-07) covers a different, later cache. +8. **All four Qwen3 headline numbers are n=1, single-seed, no CI, in-distribution val PPL — the metric the repo's own contracts ban as a headline** (`research/eval/base_eval_verdict.md:59`, §C25.7.3; `research/eval/per_stage_eval_batteries.md:9`). The −45.0% TinyStories result is likewise single-seed, single-corpus, in-domain, with **no** iso-FLOP control and **no** post-training OOD measurement (the repo's own note: *"We didn't measure wikitext-2 PPL post-training but it almost certainly got worse"*, `tinystories_summary.md:124-127`). +9. **A published TinyStories PPL that does not match the checkpoint.** `results/tinystories_summary.md:9` says 3.7893; `ck['trained_ppl']` is 3.78995 (rounds to 3.7900). Same doc reports 137.3 min and 12,150 tok/s for what is a **different, earlier run** — labelled `(prior run)` only in `POST_DATA.md:165`. +10. **Two arithmetic errors that propagate from the repo's own prose.** `Qwen3-0.6B/README.md:80` says NorMuon cost "~30% more wall-clock"; it is **+43.9%**. `README.md:167` quotes 7,480 tok/s (the step-100 reading) for a run whose final cumulative rate is 7,444. And three README sites round the CI lower bound to 0.444 where the value is 0.4434. +11. **Broken citations in the results docs.** `model.py` is cited three times and does not exist; `Qwen3-0.6B/README.md:39` sources the param count to a JSON that has no params field; `SmolLM2-134(base)/README.md:60` sources token ids to a JSON that has no tokenization key; `research/ledger/runs/2026-06-17_…md:113-114` mis-attributes 28.65 to `eval_original_vs_repro.py` (it came from the in-loop `evaluate()`); `SmolLM2-134(base)/verify.py:6` claims bf16 tolerance for an fp32 gate. +12. **A checkpoint that silently loads wrong.** `checkpoint_prope10_2tpp_.pt` carries `partial_rotary_factor: 0.1`, a key transformers 5.8.0's Qwen3 **never reads** — stock loading runs full RoPE. The modernized checkpoint carries 423 `_orig_mod.`-prefixed keys and three non-HF config flags. +13. **An unmeasured claim written as if it were a result.** `results/comparison_with_hf.md:86-88`: *"any benchmark score will match by construction."* No downstream benchmark was ever run (`results/lm_eval/` does not exist). This must not be transcribed in any form that reads as a measurement. +14. **The documented reproduce path destroys its own evidence.** `python3 _build_notebook.py` overwrites `results.ipynb` (wiping the only record of the 15.371 run) and the nbconvert pass overwrites `../checkpoint.pt`. Neither the notebook nor `train_tinystories.py` imports `safe_cuda` or runs `sentinel.py preflight`, violating the repo's own §C1/§C6 on a box where over-allocation reboots the machine. +15. **Two contradictory dependency files offered as equivalent** (`pyproject.toml` hard pins vs `requirements.txt` `torch>=2.4`), no lockfile, and **no environment spec at all for Qwen3** — under a README line that says "Install pinned dependencies that produced the 0.0 logit-diff result." +16. **Zero determinism flags repo-wide**, on a repo whose central claim is numerical equivalence, and whose own GPU deltas are attributed to unpinned backend dispatch. +17. **The null's evidence is untracked while the win's evidence is committed.** `verdict.json` and `ladder_bpb.json` for the scaling ladder are working-tree-only; `MEMORY.md` records a prior incident where a branch switch destroyed gitignored evidence. The 269 MB TinyStories weights are similarly a single uncommitted copy. +18. **A stale ledger `running` entry** (`2026-07-28_hybrid-ssm-0.2b_arch-ladder-repair`, `eta_hours 30.66`) with no live process — `sentinel.py preflight` reports `trainers=none`. +19. **`c5_evidence_CORRECTION_2026-07-28.md:326-328` forbids the words "iso-FLOP" on three HybridSSM cells** until they are re-run; one HybridSSM checkpoint is hard-quarantined for an RNG confound and 8 more resumed pre-fix files were never assessed for the same defect. +20. **Reproducibility claims that a reviewer can falsify with `env`.** The tokenizer/dataset snapshots are not in `~/.cache/huggingface`; the active `HF_HOME` is `/home/yashb98/projects/qwen-distill/hf_cache`, **outside the repo**. So a card claiming "reproducible from this repo alone" is wrong — and any earlier claim that the caches don't exist is also wrong. \ No newline at end of file From 01aaec6a4c88d83082fa237f786f16cef37ce07c Mon Sep 17 00:00:00 2001 From: yashb98 Date: Tue, 4 Aug 2026 22:57:59 +0100 Subject: [PATCH 30/35] Add complete raw dataset behind the model-card fact sheet MODEL_CARD_FACTS.md is the synthesis; this is everything under it, unabridged: 160 extracted facts (value, evidence path, verbatim source quote, self-assessed confidence, caveat) each paired with the ruling the adversarial refute pass returned, plus all 57 gaps the repo cannot answer. Nothing is dropped. Verifier entries with no 1:1 extracted fact are reproduced per dimension under "Additional verifier findings", so all 166 verdicts appear: 157 attached to a fact, 9 standing alone. The refute pass overturned 9 facts outright and qualified 38 more, so 47 of 166 checks caught something that would have been misleading if published as first extracted -- which is why the raw pass is worth keeping alongside the synthesis. Co-Authored-By: Claude Opus 5 (1M context) --- MODEL_CARD_FACTS.md | 6 + MODEL_CARD_FACTS_RAW.md | 5867 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 5873 insertions(+) create mode 100644 MODEL_CARD_FACTS_RAW.md diff --git a/MODEL_CARD_FACTS.md b/MODEL_CARD_FACTS.md index aceec00..4465630 100644 --- a/MODEL_CARD_FACTS.md +++ b/MODEL_CARD_FACTS.md @@ -21,6 +21,12 @@ synthesis pass. 17 agents, 715 tool calls. Facts the adversarial pass overturned *what the repo records*, not a re-execution of the experiments. Where a recorded number could not be re-derived from disk, §8 says so. +**Underlying data.** This file is the synthesis. The complete raw dataset — all 160 extracted facts +with their verbatim source quotes, all 166 adversarial verdicts, and all 57 gaps, nothing summarised +away — is in [`MODEL_CARD_FACTS_RAW.md`](MODEL_CARD_FACTS_RAW.md). The refute pass overturned 9 facts +outright and attached a qualifier to 38 more, so **47 of 166 checks caught something that would have +been misleading if published as first extracted** — which is the reason that file is worth keeping. + --- ## Independent spot-check of the three load-bearing findings diff --git a/MODEL_CARD_FACTS_RAW.md b/MODEL_CARD_FACTS_RAW.md new file mode 100644 index 0000000..19c92e7 --- /dev/null +++ b/MODEL_CARD_FACTS_RAW.md @@ -0,0 +1,5867 @@ +# Model-card fact sheet — complete raw dataset + +Companion to [`MODEL_CARD_FACTS.md`](MODEL_CARD_FACTS.md). That file is the synthesis; this one is +**every fact the audit extracted, unabridged** — value, evidence path, verbatim source quote, +self-assessed confidence and caveat — each paired with the ruling the adversarial verification pass +returned for it. Nothing is summarised away and nothing is dropped: verifier entries that do not +correspond 1:1 to an extracted fact are reproduced in each dimension's *Additional verifier findings*. + +Generated 2026-08-04 from the audit's structured output: 8 dimensions, each run as extract → refute. +The verifier's standing instruction was to *refute* every fact by opening the cited file, defaulting +to `WRONG` / `NEEDS_QUALIFIER` under uncertainty — so `CONFIRMED` means a second independent pass +opened the file and the claim survived. + +| Verdict | Meaning | +|---|---| +| ✅ CONFIRMED | Cited file opened; claim holds as stated | +| ⚠️ NEEDS QUALIFIER | Value correct but materially incomplete without the attached caveat | +| ❌ WRONG | Claim does not survive; corrected value given | +| ⚠️ UNVERIFIABLE | Could not be settled from disk | + +--- +## Verdict summary + +| # | Dimension | Facts | Verdicts | ✅ | ⚠️ Qual | ❌ Wrong | Gaps | +|---|---|---:|---:|---:|---:|---:|---:| +| 1 | [SmolLM2 eval provenance (15.371)](#1-smollm2-eval-provenance-15-371) | 11 | 11 | 9 | 2 | 0 | 6 | +| 2 | [SmolLM2 continued pretrain (6.8945 → 3.7900)](#2-smollm2-continued-pretrain-6-8945-3-7900) | 23 | 24 | 18 | 5 | 1 | 10 | +| 3 | [Qwen3 eval provenance (28.65 / 46.31 / 23.52 / 13.40)](#3-qwen3-eval-provenance-28-65-46-31-23-52-13-40) | 20 | 22 | 15 | 5 | 2 | 7 | +| 4 | [−0.474 bpb — NorMuon vs AdamW](#4-0-474-bpb-normuon-vs-adamw) | 12 | 12 | 8 | 4 | 0 | 7 | +| 5 | [Reproduce — commands, versions, parity, determinism](#5-reproduce-commands-versions-parity-determinism) | 28 | 28 | 24 | 3 | 1 | 7 | +| 6 | [Training details (the 1.19B-token runs)](#6-training-details-the-1-19b-token-runs) | 22 | 22 | 12 | 8 | 2 | 9 | +| 7 | [Loader API (real code)](#7-loader-api-real-code) | 21 | 23 | 17 | 5 | 1 | 5 | +| 8 | [Checkpoint inventory on disk](#8-checkpoint-inventory-on-disk) | 23 | 24 | 16 | 6 | 2 | 6 | +| | **Total** | **160** | **166** | **119** | **38** | **9** | **57** | + +**160 facts extracted; 166 verdicts returned** (verifiers occasionally split or added a check, so the +counts differ). **9** facts were overturned outright and **38** needed a qualifier — +i.e. **47 of 166** checks found something that would have been misleading if published as first +extracted. The **57 gaps** are things the repo genuinely cannot answer, reproduced verbatim per dimension. + +--- + +## 1. SmolLM2 eval provenance (15.371) + +Audit dimension: SmolLM2-135M reproduction eval provenance (wikitext-2 val PPL 15.371 + HF parity) + +### 1.1 Which HF dataset id, config name, and split produced 15.371? (verify whether it is the -raw- variant) + +**Value** + +``` +dataset id = `Salesforce/wikitext` (the Salesforce-namespaced mirror, not the bare `wikitext`), config = `wikitext-2-raw-v1` (it IS the -raw- variant, confirmed), split = `validation`. Loaded with no `revision=` pin. +``` + +**Evidence** — `SmolLM2-134(base)/_build_notebook.py:233` + +**Source quote** + +``` +ds = load_dataset('Salesforce/wikitext', 'wikitext-2-raw-v1', split='validation') [_build_notebook.py:233] +"ds = load_dataset('Salesforce/wikitext', 'wikitext-2-raw-v1', split='validation')\n", [executed notebook cell, results.ipynb:739] + "dataset": "wikitext-2-raw-v1 validation", [results/perplexity.json:5] +``` + +**Confidence** — measured from code + +**Caveat** — Text preprocessing is NOT the vanilla HF perplexity recipe. The script joins with '\n\n' but FILTERS OUT blank/whitespace-only rows: `text = '\n\n'.join(ex['text'] for ex in ds if ex['text'].strip())` (_build_notebook.py:234 / results.ipynb:740). The standard `"\n\n".join(test["text"])` keeps them. Different token stream => this absolute PPL is NOT directly comparable to published wikitext-2 numbers, only to the co-run HF model. No dataset revision sha is pinned, so the exact snapshot is not recoverable from disk. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED at SmolLM2-134(base)/_build_notebook.py:233 -> `ds = load_dataset('Salesforce/wikitext', 'wikitext-2-raw-v1', split='validation')`. Byte-identical source line survives into the executed notebook at SmolLM2-134(base)/results.ipynb:739. Stamped into the machine-written artifact at SmolLM2-134(base)/results/perplexity.json:5 -> "dataset": "wikitext-2-raw-v1 validation". It IS the -raw- variant and it IS the Salesforce-namespaced mirror; no revision= argument on the line. The blank-row filter is real: _build_notebook.py:234 / results.ipynb:740 -> `text = '\n\n'.join(ex['text'] for ex in ds if ex['text'].strip())`. ONE NIT: the caveat's assertion that the standard recipe is `"\n\n".join(test["text"])` is an external-convention claim, not something on disk in this repo — correct as far as I know but flag it as not repo-verifiable. Bonus corroboration the fact missed: eval_after_vs_base.py:74 independently names the same dataset triple, and train.py:78 names wikitext-103-raw-v1 (train split) — so the repo does distinguish the two. +``` + + +### 1.2 What sequence length and stride, and is it sliding-window or non-overlapping chunks? + +**Value** + +``` +SEQ = 1024, STRIDE = 512. SLIDING WINDOW with 50% overlap, but WITHOUT the standard -100 masking of the overlapped context: every window contributes all 1023 shifted targets to the summed NLL, so overlap-region tokens are scored TWICE (once with short context, once with long). Neither non-overlapping chunks nor the canonical HF strided PPL. +``` + +**Evidence** — `SmolLM2-134(base)/_build_notebook.py:239` + +**Source quote** + +``` +# Slide a 1024-token window with stride 512 over the first 32K tokens. +SEQ = 1024 +STRIDE = 512 +N_TOKENS = min(len(input_ids), 32_000) +def ppl(net): + net = net.to(device).eval() + nlls, n = [], 0 + with torch.no_grad(): + for begin in range(0, N_TOKENS - SEQ, STRIDE): + ids = input_ids[begin:begin+SEQ].unsqueeze(0).to(device) + out = net(ids) + logits = out.logits if hasattr(out, 'logits') else out['logits'] + shift_logits = logits[..., :-1, :].float() + shift_labels = ids[..., 1:] + loss = F.cross_entropy(shift_logits.reshape(-1, shift_logits.size(-1)), + shift_labels.reshape(-1), reduction='sum') + nlls.append(loss.item()) + n += shift_labels.numel() + return math.exp(sum(nlls) / n), n [_build_notebook.py:239-257] + "seq_len": 1024, + "stride": 512 [results/perplexity.json:6-7] +``` + +**Confidence** — measured from code + +**Caveat** — Two scope limits a reviewer will ask about: (1) `N_TOKENS = min(len(input_ids), 32_000)` caps the eval to the FIRST 32,000 tokens of the concatenated validation text; the executed notebook printed `Validation tokens: 268,140` (results.ipynb:721), so only the first 31,744 token positions (~11.8% of the validation set) were ever fed to the model. (2) Because overlapped labels are not masked, the 62,403 scored targets cover only 31,743 DISTINCT positions (indices 1..31,743) — a ~1.97x double-count. I re-derived the window arithmetic: range(0, 32000-1024, 512) = 61 windows, first begin=0, last begin=30720, 1023 targets each, 61*1023 = 62,403, max index touched 31,744. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED. _build_notebook.py:240-241 `SEQ = 1024` / `STRIDE = 512`; :242 `N_TOKENS = min(len(input_ids), 32_000)`; :247 `for begin in range(0, N_TOKENS - SEQ, STRIDE)`; :252 `shift_labels = ids[..., 1:]` (1023 targets/window, no -100 masking anywhere in the cell); :256 `n += shift_labels.numel()`. Echoed in results/perplexity.json:6-7 (seq_len 1024, stride 512). I re-ran the arithmetic independently: range(0, 32000-1024, 512) = 61 windows, first begin 0, last begin 30720, 61*1023 = 62,403 targets, 31,743 DISTINCT label indices (1..31,743), 31,744 positions ever fed, 31744/268140 = 11.84% coverage, duplication factor 1.9659. All match the fact. TWO PRECISION NITS in the fact's own derivation prose: (a) it writes 'max index touched 31,744' — the max index is 31,743; 31,744 is the position COUNT. (b) 'overlap-region tokens are scored TWICE' is true for 30,660 of the 31,743 distinct positions; the remaining 1,083 are scored once (I computed the exact multiplicity histogram: Counter({2: 30660, 1: 1083})). Also worth knowing: `SEQ` is REBOUND to 512 later at _build_notebook.py:442 for the training-demo cell, but perplexity.json is written at :270-273 inside the PPL cell, before the rebind — so the 1024 in the JSON is correct. +``` + + +### 1.3 Which tokenizer (exact HF repo id) was used? + +**Value** + +``` +`HuggingFaceTB/SmolLM2-135M`, via `AutoTokenizer.from_pretrained(REPO)` with REPO imported from verify.py. No revision pin. BPE, vocab 49,152. `add_special_tokens` left at the transformers default; the executed output shows 5 tokens for a 5-word prompt, i.e. no BOS prepended. Same repo id as the model => own-tokenizer PPL. +``` + +**Evidence** — `SmolLM2-134(base)/verify.py:19` + +**Source quote** + +``` +REPO = "HuggingFaceTB/SmolLM2-135M" [verify.py:19] +from verify import load_official_weights_into_ours, REPO [_build_notebook.py:43 / results.ipynb cell 1] +tokenizer = AutoTokenizer.from_pretrained(REPO) [_build_notebook.py:112] +encodings = tokenizer(text, return_tensors='pt') [_build_notebook.py:235] + "Tokens : [504, 3575, 282, 4649, 314]\n", [results.ipynb:269 — 'The capital of France is' -> 5 tokens, no BOS] + "Tokenizer": "BPE, vocab 49,152", [results/summary.json:6] +``` + +**Confidence** — measured from code + +**Caveat** — I could not verify `add_bos_token` from a config on disk: the local HF hub cache (~/.cache/huggingface/hub) contains only `models--Qwen--Qwen3.5-9B`, no SmolLM2-135M snapshot. The no-BOS conclusion is inferred from the executed 5-token/5-piece output, not from a tokenizer_config.json. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Tokenizer id/vocab/no-BOS are all CORRECT and are now CONFIRMED FROM A CONFIG ON DISK. The caveat 'no SmolLM2-135M snapshot exists on disk' is REFUTED: a full snapshot lives at /home/yashb98/projects/qwen-distill/hf_cache/hub/models--HuggingFaceTB--SmolLM2-135M/snapshots/93efa2f097d58c2a74874c7e644dbc9b0cee75a2/ (a second project's HF_HOME, not ~/.cache). That directory also yields the revision sha the 'gaps' section says is unrecoverable: 93efa2f097d58c2a74874c7e644dbc9b0cee75a2. +``` + +**Verifier note** + +``` +VALUE CONFIRMED: verify.py:19 `REPO = "HuggingFaceTB/SmolLM2-135M"`; _build_notebook.py:43 `from verify import load_official_weights_into_ours, REPO`; :112 `tokenizer = AutoTokenizer.from_pretrained(REPO)` (no revision=); :235 `encodings = tokenizer(text, return_tensors='pt')` (add_special_tokens left at default True). results/summary.json:6 "Tokenizer": "BPE, vocab 49,152". CAVEAT REFUTED — from the snapshot above I read: tokenizer_config.json has NO `add_bos_token` key, tokenizer_class = 'GPT2Tokenizer' (byte-level BPE), bos_token = eos_token = '<|endoftext|>', model_max_length 8192; tokenizer.json has `post_processor: null` (so add_special_tokens=True adds NOTHING). That is a config-level proof of the no-BOS behaviour, not an inference. I further decoded the ids from that snapshot's vocab.json: 504='The', 3575='Ġcapital', 282='Ġof', 4649='ĠFrance', 314='Ġis', and 260='Ġthe' — exactly matching results.ipynb:269 and :280-281. vocab.json length = 49152, config.json vocab_size = 49152, tie_word_embeddings=True, 30 layers / 576 hidden / 9Q / 3KV / rope_theta 100000. TWO STANDING QUALIFIERS: (1) that snapshot's blobs are dated May 13 11:44 while results.ipynb ran May 13 ~22:20 (same day) — strongly suggestive but NOT proof it is the snapshot the run used, since the code pins no revision and the cache is outside this repo; (2) it is outside the repo root, so a card claiming 'reproducible from this repo alone' still cannot pin the tokenizer. +``` + + +### 1.4 How many target tokens did the PPL average over? (verify the '62,403 target tokens' prose figure) + +**Value** + +``` +62,403 scored target tokens — VERIFIED three ways: the machine-written JSON, the executed notebook stdout, and independent re-derivation of the loop bounds (61 windows x 1023 targets). +``` + +**Evidence** — `SmolLM2-134(base)/results/perplexity.json:4` + +**Source quote** + +``` +"tokens": 62403, [results/perplexity.json:4] + "Ours : ppl = 15.371 (62,403 target tokens, 4.8s)\n", + "HF : ppl = 15.371 (62,403 target tokens, 9.0s)\n", [results.ipynb:729-730, executed output] +- `perplexity.json` — sliding-window CE perplexity on wikitext-2-raw-v1 + validation, 62,403 target tokens, seq=1024 stride=512. [results/README.md:38-39] +``` + +**Confidence** — results JSON + +**Caveat** — 62,403 is the count of SCORED targets, not distinct tokens. Overlapping windows without label masking mean only 31,743 distinct positions are covered; roughly half the 62,403 are duplicate scorings of the same token under a different context length. results/README.md:38-39 and results/POST_DATA.md:34-36 state '62,403 target tokens' without that qualification. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED four ways. (1) Machine-written: results/perplexity.json:4 `"tokens": 62403`. (2) Executed stdout: results.ipynb:729-730 'Ours : ppl = 15.371 (62,403 target tokens, 4.8s)' / 'HF : ppl = 15.371 (62,403 target tokens, 9.0s)'. (3) Prose: results/README.md:38-39 and results/POST_DATA.md:34-36. (4) My own re-derivation: 61 windows x 1023 = 62,403. The fact's qualifier is correct and material — 62,403 is SCORED targets over only 31,743 distinct positions. Both prose sites (results/README.md:38-39, results/POST_DATA.md:34-36) state the number with no such qualification, so if the figure goes on a model card it MUST be labelled 'scored targets (1.97x double-counted by overlapping windows), 31,743 distinct positions, first 11.8% of the validation split'. +``` + + +### 1.5 Exact values: ours vs HF to full precision, and the delta + +**Value** + +``` +ours_ppl = 15.370989092449635; hf_ppl = 15.370989964425396; delta (hf - ours) = +8.719757609298995e-07 (abs 8.72e-07). BOTH are MEASURED on this box in the same notebook kernel — the 'HF' figure is a live forward pass of HF's LlamaForCausalLM, NOT copied from a model card or paper. fp32 (HF loaded dtype=torch.float32, our model default fp32, logits .float() before CE), run on GPU (Device: cuda | NVIDIA GB10). +``` + +**Evidence** — `SmolLM2-134(base)/results/perplexity.json:2` + +**Source quote** + +``` +"ours_ppl": 15.370989092449635, + "hf_ppl": 15.370989964425396, [results/perplexity.json:2-3] +ours_ppl, n_tok = ppl(model) +... +hf_ppl, _ = ppl(hf_model) [_build_notebook.py:260,263 — both computed live] +hf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) [_build_notebook.py:110] + "Device: cuda | NVIDIA GB10\n" [results.ipynb:45] +``` + +**Confidence** — results JSON + +**Caveat** — Delta recomputed by me from the JSON floats: 15.370989964425396 - 15.370989092449635 = 8.719757609298995e-07. README.md:57-58 rounds to 15.370989 / 15.370990 with 'Δ ≈ 9 × 10⁻⁷' — consistent. Important regime split: the PPL comparison ran on GPU (both models), whereas the max|Δlogits| = 0 'bit-exact' claim ran on CPU; do not merge them into one claim. Also n=1, one corpus slice, no seeds, no CI — an implementation-equivalence check, not a quality benchmark. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Values and delta are exactly right and both ARE measured (not copied). Missing material qualifier for a MODEL CARD: 'ours' is not a model this repo trained — it is the OFFICIAL SmolLM2-135M safetensors loaded into the reimplementation class (_build_notebook.py:114 `load_official_weights_into_ours(model, hf_model.state_dict())`). 15.371 therefore characterizes the official checkpoint under a nonstandard eval recipe, and must NOT be attached to either local checkpoint (checkpoint.pt = the 150-step demo; checkpoint_tinystories.pt = TinyStories continued-pretrain). +``` + +**Verifier note** + +``` +VALUES VERIFIED: results/perplexity.json:2-3 give exactly 15.370989092449635 and 15.370989964425396. I recomputed the delta in Python: 15.370989964425396 - 15.370989092449635 = 8.719757609298995e-07 — identical to the fact, correct sign (HF higher). Consistent with the executed 'Δppl = 0.000001' at results.ipynb:731 and with README.md:57-58 ('15.370989' / '15.370990 (Δ ≈ 9 × 10⁻⁷)'). MEASURED-not-copied CONFIRMED: _build_notebook.py:260 `ours_ppl, n_tok = ppl(model)` and :263 `hf_ppl, _ = ppl(hf_model)` are both live forward passes; the HF model is loaded at :110 `AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32)`. fp32/GPU CONFIRMED: :251 `shift_logits = logits[..., :-1, :].float()`, :244 `net = net.to(device).eval()`, device set at :46, and the run printed 'Device: cuda | NVIDIA GB10' (results.ipynb:45) with 'Torch: 2.11.0+cu130' (results.ipynb:44). The fact's own caveats (GPU-PPL vs CPU-bit-exact regime split; n=1, one slice, no seeds/CI) are correct and must survive to the card. +``` + + +### 1.6 Which script computes 15.371, and what is the exact command to re-run it? + +**Value** + +``` +There is NO standalone perplexity script. It is cell 14 of `results.ipynb`; that cell's source is authored by the notebook generator `_build_notebook.py:230-273`. Documented regeneration = the two-step generate-then-nbconvert pass in results/README.md:66-73. +``` + +**Evidence** — `SmolLM2-134(base)/results/README.md:66` + +**Source quote** + +```` +```bash +# from /home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)/ +python3 _build_notebook.py # writes results.ipynb (28 cells, no outputs) +jupyter nbconvert --to notebook \ + --execute results.ipynb \ + --output results.ipynb \ + --ExecutePreprocessor.timeout=2400 +# total runtime ~3 minutes (perplexity & training are the slow cells) +``` [results/README.md:66-74] +with open(RESULTS / 'perplexity.json', 'w') as f: + json.dump({'ours_ppl': ours_ppl, 'hf_ppl': hf_ppl, 'tokens': n_tok, + 'dataset': 'wikitext-2-raw-v1 validation', + 'seq_len': SEQ, 'stride': STRIDE}, f, indent=2) [_build_notebook.py:270-273] +```` + +**Confidence** — measured from code + +**Caveat** — Three re-run hazards: (1) `python3 _build_notebook.py` OVERWRITES results.ipynb and wipes the executed outputs that are currently the only record of the run; (2) the nbconvert pass re-executes all 28 cells including a 150-step from-scratch training demo (results.ipynb cell 23), so it is not a cheap PPL-only re-run; (3) the notebook never imports safe_cuda / calls safe_cuda.guard() before torch (_build_notebook.py:32-52), which CLAUDE.md §C1 mandates for every PyTorch script on this GB10 box. Provenance timestamps are consistent: results/perplexity.json mtime 2026-05-13 22:19:09, results.ipynb 22:20:20; `git diff HEAD` on both is empty (identical to the single commit 84a96c0). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED. I enumerated the notebook with nbformat: 28 cells total; index 13 = markdown '## 6. Perplexity on wikitext-2-raw-v1 validation', index 14 = code '# %% Perplexity on wikitext-2 validation'. So 'cell 14' is correct 0-indexed (it is section 6 by the notebook's own numbering — say 'cell index 14' on any card to avoid ambiguity). results/README.md:66-74 carries the exact bash block quoted. No standalone PPL script exists — I grepped every .py/.sh in the folder for wikitext/perplex/math.exp; the only other eval-PPL path is eval_after_vs_base.py (fact 10) and train_tinystories.py:64 (TinyStories, non-overlapping). ALL THREE RE-RUN HAZARDS CONFIRMED: (1) _build_notebook.py:561-562 `out = Path('results.ipynb')` / `nbf.write(nb, str(out))` — unconditional overwrite, wiping the executed outputs; (2) 28 cells re-execute, including cell 23 whose body sets `STEPS = 150` (its own header comment says '200 optimizer steps on wikitext-103 slice' and its code at :435-436 actually loads wikitext-2 train — the cell comment is stale twice over, but the fact's '150-step' is the correct figure); (3) grep for 'safe_cuda' in _build_notebook.py returns ZERO hits — the setup cell at :32-52 imports torch at :35 with no guard, violating CLAUDE.md §C1. PROVENANCE CONFIRMED: stat gives results/perplexity.json mtime 2026-05-13 22:19:09 and results.ipynb 22:20:20. MINOR WORDING FIX: 'the single commit 84a96c0' should read 'the only commit that has ever touched these files' — `git rev-list --count HEAD` = 77; `git log -- SmolLM2-134(base)/results/perplexity.json` and `... results.ipynb` each return only 84a96c0, and `git status --porcelain 'SmolLM2-134(base)/'` is empty. +``` + + +### 1.7 What are the parity / bit-exactness numbers (max|Δlogits|, argmax agreement) and which script produces them? + +**Value** + +``` +max|Δlogits| = 0.000e+00 (relative 0.000e+00); argmax agreement YES — HF token 260 -> ' the', ours 260 -> ' the'. Prompt "The capital of France is", fp32, CPU. Produced by `python3 verify.py` (logged to results/parity.log) and independently reproduced by results.ipynb cell 6. Gates: `assert max_abs < 1e-3` (verify.py:76) and `assert hf_next == our_next` (verify.py:83); the same gates are wrapped as pytest in tests/test_parity.py (short-prompt logits, argmax, 512-token long context, all 30 per-layer hidden states, param count 134,515,008, tied-embedding pointer). +``` + +**Evidence** — `SmolLM2-134(base)/results/parity.log:6` + +**Source quote** + +``` +max |Δlogits| = 0.000e+00 +relative = 0.000e+00 +HF next token : ' the' +Ours next : ' the' + +✓ Architecture parity verified. [results/parity.log:6-11] + assert max_abs < 1e-3, f"Outputs diverge: {max_abs}. Architecture mismatch." [verify.py:76] + assert hf_next == our_next, "Next-token disagreement" [verify.py:83] + "max |Δlogits| = 0.000e+00\n", + "HF argmax last : 260 → ' the'\n", + "Ours argmax last : 260 → ' the'\n", [results.ipynb:278,280,281] + "max |Δlogits| vs HF": "0.000e+00", [results/summary.json:7] +``` + +**Confidence** — measured from code + +**Caveat** — The 0.0 is a CPU result on a 5-token prompt: verify.py never calls .to(cuda), and results.ipynb cell 6 runs before any .to(device). The GPU-side numbers quoted in README.md:73-78 / results/comparison_with_hf.md:10-15 are prose-only — see next fact. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED line by line. results/parity.log:6-11 reads exactly 'max |Δlogits| = 0.000e+00' / 'relative = 0.000e+00' / "HF next token : ' the'" / "Ours next : ' the'" / '✓ Architecture parity verified.'. verify.py:76 `assert max_abs < 1e-3, ...` and verify.py:83 `assert hf_next == our_next, "Next-token disagreement"` — both present verbatim. Token id 260 comes from results.ipynb:280-281 ("HF argmax last : 260 → ' the'" / "Ours argmax last : 260 → ' the'"), NOT from parity.log, and the fact attributes it correctly; I independently confirmed 260='Ġthe' from the SmolLM2 vocab.json. results/summary.json:7 "max |Δlogits| vs HF": "0.000e+00". CPU CLAIM CONFIRMED BY CONSTRUCTION: verify.py has no .to('cuda') anywhere, and in _build_notebook.py the first `.to(device)` occurrences are at :244/:248 (inside the PPL helper) and :449/:466 (training demo) — all AFTER the parity cell at :108-134. tests/test_parity.py fully checks out: :53 `assert n == 134_515_008`, :61 tied data_ptr, :73 short-prompt <1e-3, :83 argmax, :93 `max_length=512` long context, :127 `assert len(hf_states) == len(our_states) == 30` with :132 per-layer <1e-3. Extra corroboration: parity.log:1 emits the `torch_dtype` deprecation warning, which matches verify.py:53's `torch_dtype=torch.float32` (the notebook uses the newer `dtype=` at :110) — independent evidence parity.log really is verify.py's output. +``` + + +### 1.8 Are the six GPU-side cross-check parity numbers (4.72e-05, 1.95e-03 @ L14, 5/5 greedy, 5/5 top-10, 4.01e-05 long-context, 0.072 vs 0.080 sampling) backed by a results file? + +**Value** + +``` +NO. The .md write-up exists but the machine-written JSON it should derive from is absent from disk. +``` + +**Evidence** — `SmolLM2-134(base)/compare_with_hf.py:259` + +**Source quote** + +``` +with open(RESULTS / "comparison_with_hf.json", "w") as f: + json.dump(findings, f, indent=2) + print(f"\nSaved {RESULTS}/comparison_with_hf.json") [compare_with_hf.py:259-261] + +$ ls SmolLM2-134(base)/results/ -> attention, comparison_with_hf.md, generations.txt, loss_curve.csv, param_count.log, parity.log, perplexity.json, plots, POST_DATA.md, README.md, summary.json, tinystories_after.txt, tinystories_before.txt, tinystories_summary.md, tinystories_train.csv, tinystories_train.log, topk_predictions.json, training_recipe_resolved.json (no comparison_with_hf.json) +``` + +**Confidence** — PROSE ONLY + +**Caveat** — Concrete unverifiable discrepancy: comparison_with_hf.md:14 and README.md:77 report the long-context check as '401-token RoPE', but compare_with_hf.py:180-181 truncates at `max_length=512`. 401 is presumably the actual tokenized length of the 2000-char probe, but with the JSON missing nothing on disk proves it. Do not put any of these six numbers on a model card as measured without re-running `python3 compare_with_hf.py`. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED. `ls SmolLM2-134(base)/results/comparison_with_hf.json` -> 'No such file or directory' (exit 2); the full results/ listing contains comparison_with_hf.md but no .json. compare_with_hf.py:259-261 is exactly as quoted and is the only writer. All six numbers do exist as prose in results/comparison_with_hf.md:10-15 and are duplicated in README.md:73-78. THE 401-vs-512 DISCREPANCY IS REAL: comparison_with_hf.md:14 and README.md:77 say '401-token RoPE', while compare_with_hf.py:177 banners '5. Long-context (RoPE sanity at 512 tokens)' and :179-181 build `long_text = ("In the field of language modeling, " * 60)[:2000]` then tokenize with `truncation=True, max_length=512`; :251 prints the label '5. Long-context (512 tok)'. tests/test_parity.py:92-93 uses the identical construction. So 401 is plausibly the realized token count of the 2000-char probe, but with the JSON gone nothing on disk proves it. ADDITIONAL FINDING the fact should carry: results/README.md:3-4 asserts 'Every file here is produced live ... No values are typed in by hand' — that blanket claim is NOT supported for comparison_with_hf.md. Its mtime (2026-05-13 22:07:53) also predates the notebook run (22:19-22:20), so it was not produced by that run. +``` + + +### 1.9 Were downstream benchmark numbers (HellaSwag/ARC/MMLU etc.) measured in this repo? + +**Value** + +``` +NOT_FOUND — never run. A harness script exists (scripts/run_lm_eval.sh) but its output directory does not exist. +``` + +**Evidence** — `SmolLM2-134(base)/scripts/run_lm_eval.sh:27` + +**Source quote** + +``` +OUT_DIR="results/lm_eval" [scripts/run_lm_eval.sh:27] +$ ls SmolLM2-134(base)/results/lm_eval -> "ls: cannot access 'results/lm_eval': No such file or directory" + +- **Published downstream benchmarks** (HellaSwag, ARC, MMLU, etc. — model card + reports these). Computing them from scratch would take a few hours per task. [results/comparison_with_hf.md:84-85] +``` + +**Confidence** — NOT FOUND + +**Caveat** — results/comparison_with_hf.md:86-88 then asserts 'any benchmark score will match by construction' — an unmeasured claim; it must not be transcribed to a model card as a result. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED. scripts/run_lm_eval.sh:27 `OUT_DIR="results/lm_eval"`; :23 `TASKS="hellaswag,arc_easy,arc_challenge,piqa,winogrande,commonsense_qa,openbookqa,mmlu"`; :22 BASE_REPO=HuggingFaceTB/SmolLM2-135M; :29 `mkdir -p "$OUT_DIR"` would create it on any run. `ls SmolLM2-134(base)/results/lm_eval` -> 'No such file or directory' (exit 2), and the results/ listing has no lm_eval entry — the script has never completed even its mkdir. results/comparison_with_hf.md:84-85 confirms the omission is deliberate ('Published downstream benchmarks ... Computing them from scratch would take a few hours per task'), and :86-88 does make the unmeasured assertion 'any benchmark score will match by construction'. That sentence is a prediction, not a result; it must not be transcribed onto a card in any form that reads as a measurement. +``` + + +### 1.10 Is there a second, different wikitext-2 PPL path in this folder that could be confused with 15.371? + +**Value** + +``` +YES — eval_after_vs_base.py computes a DIFFERENT wikitext-2 PPL: same dataset/config/split and same seq=1024/stride=512, but capped at max_windows=200 (not 32,000 tokens) and run in bf16 on GPU (not fp32). It serves the TinyStories before/after comparison, not the 15.371 headline. +``` + +**Evidence** — `SmolLM2-134(base)/eval_after_vs_base.py:50` + +**Source quote** + +``` +def ppl(model, text, seq=1024, stride=512, max_windows=200): [eval_after_vs_base.py:50] +wk = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation") [eval_after_vs_base.py:74] +dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32 [eval_after_vs_base.py:29] +# --- 2. wikitext-2 validation (general text, was 15.371 for both before) - [eval_after_vs_base.py:72] +``` + +**Confidence** — measured from code + +**Caveat** — eval_after_vs_base.py:8 says it writes results/tinystories_vs_base.{md,json}; neither file is on disk. Its code-corpus fallback at line 91 does `open("model.py").read()`, and model.py does not exist in this folder — that branch would crash. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED. eval_after_vs_base.py:50 `def ppl(model, text, seq=1024, stride=512, max_windows=200):` with :53-54 `for i, begin in enumerate(...)` / `if i >= max_windows: break`; :74 `wk = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation")`; :29 `dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32` applied at :38 and :45; :72 comment 'was 15.371 for both before'. So it is a WIDER slice (200 windows ~= 204,600 targets vs 61 windows / 62,403) at LOWER precision — genuinely non-comparable to 15.371 in both directions. Its outputs are missing: eval_after_vs_base.py:8 declares results/tinystories_vs_base.md and .json; `ls` on both -> 'No such file or directory'. The :91 fallback `code_text = open("model.py").read() * 30` would crash (model.py absent). TWO MORE wikitext paths the fact did not enumerate, both TRAINING not eval, but confusable on a card: train.py:78 loads wikitext-103-raw-v1 TRAIN, and _build_notebook.py:436 loads wikitext-2-raw-v1 TRAIN for the 150-step demo while the surrounding prose at :402 and the cell header at :428 both say 'wikitext-103 slice' — a code/prose mismatch. Do not write 'trained on wikitext-103' for the notebook demo. +``` + + +### 1.11 Do the prose docs cite any file that does not exist? + +**Value** + +``` +YES — `model.py`. results/POST_DATA.md:20 cites '198 lines | model.py line count | `wc -l model.py`' and results/README.md:25 cites 'param_count.log — output of `python3 model.py`', but the folder contains only model_full.py (a stale __pycache__/model.cpython-312.pyc suggests model.py once existed). +``` + +**Evidence** — `SmolLM2-134(base)/results/POST_DATA.md:20` + +**Source quote** + +``` +| **198 lines** | `model.py` line count for the from-scratch architecture | `wc -l model.py` | [results/POST_DATA.md:20] +- `param_count.log` — output of `python3 model.py` (param count + random-init forward). [results/README.md:25] +$ ls SmolLM2-134(base)/model.py -> "ls: cannot access 'model.py': No such file or directory" +``` + +**Confidence** — measured from code + +**Caveat** — The parameter count itself IS backed: results/param_count.log:1-2 reads 'params: 134,515,008 (target 134,515,008)' / 'tied: True', and tests/test_parity.py:53 asserts n == 134_515_008. Only the '198 lines of model.py' claim and the 'python3 model.py' reproduce instruction are broken. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +VERIFIED. results/POST_DATA.md:20 reads '| **198 lines** | `model.py` line count for the from-scratch architecture | `wc -l model.py` |'. results/README.md:25 reads '- `param_count.log` — output of `python3 model.py` (param count + random-init forward).' `ls SmolLM2-134(base)/model.py` -> 'No such file or directory'; the directory listing shows model_full.py (15,443 bytes) only, and __pycache__/model.cpython-312.pyc is present alongside model_full.cpython-312.pyc, so model.py did once exist. A THIRD broken reference the fact missed: results/README.md:4 also credits '`../verify.py` / `../model.py`' as producers of the results files. PARAM COUNT IS SOUND: results/param_count.log:1-2 reads 'params: 134,515,008 (target 134,515,008)' / 'tied: True', tests/test_parity.py:53 asserts `n == 134_515_008`, and the official config.json in the snapshot I found confirms vocab 49152 / hidden 576 / 30 layers / tie_word_embeddings=True. Only the '198 lines' figure and the 'python3 model.py' reproduce instruction are unbacked — a card must not repeat either. +``` + + +### 1.G Gaps — not determinable from disk + +- No dataset revision/sha pin. `load_dataset('Salesforce/wikitext', 'wikitext-2-raw-v1', split='validation')` (_build_notebook.py:233) carries no `revision=` argument, so the exact dataset snapshot behind 15.371 is not recoverable from disk. Same for tokenizer/model: `AutoTokenizer.from_pretrained(REPO)` and `AutoModelForCausalLM.from_pretrained(REPO, ...)` pin no revision. +- Tokenizer special-token behaviour cannot be confirmed from a config file. The local HF hub cache (~/.cache/huggingface/hub) holds only `models--Qwen--Qwen3.5-9B`; there is no SmolLM2-135M snapshot to read tokenizer_config.json / add_bos_token from. The 'no BOS prepended' conclusion rests only on the executed notebook output (5 tokens for a 5-word prompt, results.ipynb:269). +- results/comparison_with_hf.json is missing, so the six GPU-side cross-check numbers in results/comparison_with_hf.md and README.md §0.2 (4.72e-05 final logits, 1.95e-03 @ layer 14, '401-token' long context, 0.072 vs 0.080 sampling) have no machine-written backing file. compare_with_hf.py:259-260 would create it on re-run. +- eval_after_vs_base.py:8 declares outputs results/tinystories_vs_base.{md,json}; neither exists. The TinyStories before/after PPLs (6.8945 -> 3.7900, 199,485 target tokens) exist only as plain text in results/tinystories_before.txt and results/tinystories_after.txt:2-3, not as structured JSON. +- Environment versions only partially recorded. results.ipynb cell 1 output gives 'Torch: 2.11.0+cu130' and 'Device: cuda | NVIDIA GB10'; no transformers or datasets version is recorded for the run that produced 15.371, and requirements.txt (62 bytes) was not pinned to that run. +- No statistical rigor around 15.371: single run, single corpus slice (first ~11.8% of wikitext-2 validation), no seeds, no confidence interval, no second corpus, no BPB. It was NOT produced by the research/ eval-harness, so it carries no suite_version stamp — by the repo's own §C10/§C17 rules it is an implementation-equivalence check, not a comparable quality number. + +--- + +## 2. SmolLM2 continued pretrain (6.8945 → 3.7900) + +Audit dimension: SmolLM2-135M continued-pretraining run on TinyStories (val PPL 6.8945 -> 3.7900, -45.0%) + +### 2.1 Which HF dataset id / config / split produced 6.8945 and 3.7900? + +**Value** + +``` +hf id `roneneldan/TinyStories`, split="validation", NO config/subset name passed (positional config arg omitted), NO revision pin. Training data was the same id, split="train". +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:155` + +**Source quote** + +``` +val_ds = load_dataset("roneneldan/TinyStories", split="validation") +(line 154:) train_ds = load_dataset("roneneldan/TinyStories", split="train") +``` + +**Confidence** — measured from code + +**Caveat** — Corroborated by the run log: `SmolLM2-134(base)/results/tinystories_train.log:4` -> `train: 2,119,719 stories, val: 21,990`. NOT pinned to a dataset revision/sha, so a re-run is not byte-reproducible. The eval does NOT use the whole validation split: it consumes stories in order until the packed token buffer reaches 200,000 (train_tinystories.py:172-181), landing at 200,068 tokens -> a PREFIX of the 21,990 stories. How many stories that was is not recorded anywhere. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +VALUE IS CORRECT. Two caveat sentences must be replaced: (a) the resolved revision IS recoverable — `roneneldan/TinyStories` @ f54c09fd23315a6f9c86f9dc80f725de7d8f9c64 (cached 2026-05-13 14:16-14:20, i.e. ~7h BEFORE the run started); (b) the eval consumed the FIRST 1,040 non-empty validation stories (I re-ran the packer today and it reproduced 200,068 tokens byte-identically). +``` + +**Verifier note** + +``` +VALUE CONFIRMED verbatim. train_tinystories.py:154 = ` train_ds = load_dataset("roneneldan/TinyStories", split="train")`; :155 = ` val_ds = load_dataset("roneneldan/TinyStories", split="validation")`. No positional config arg, no revision= kwarg — correct. results/tinystories_train.log:4 = `[21:23:34] train: 2,119,719 stories, val: 21,990` — confirmed verbatim. CAVEAT PARTLY REFUTED: the agent concluded "a re-run is not byte-reproducible" and "how many stories that was is not recorded anywhere". Both are overstated. HF_HOME is redirected to /home/yashb98/projects/qwen-distill/hf_cache (env var, not ~/.cache), and it contains hub/datasets--roneneldan--TinyStories with refs/main = f54c09fd23315a6f9c86f9dc80f725de7d8f9c64, blobs dated 2026-05-13 14:16-14:20 — before the run began. I re-executed the exact packing loop (train_tinystories.py:172-181) on CPU today after `python3 sentinel.py preflight` (exit 0) and got: `packed tokens: 200068 stories consumed: 1040 empty skipped: 0` — an exact byte-level reproduction of log:6. So the eval subset IS reconstructible: the first 1,040 stories of the validation split. It remains true that the CODE passes no revision pin. +``` + + +### 2.2 How was the eval text assembled from that split? + +**Value** + +``` +Stories are stripped, empty ones skipped, each encoded with add_special_tokens=False and followed by tokenizer.eos_token_id, concatenated until >= 200,000 tokens. Actual packed length: 200,068 tokens. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:172` + +**Source quote** + +``` +val_buf = [] + for ex in val_ds: + text = ex["text"].strip() + if not text: + continue + val_buf.extend(tokenizer.encode(text, add_special_tokens=False)) + val_buf.append(eos) + if len(val_buf) >= 200_000: + break +``` + +**Confidence** — measured from code + +**Caveat** — Packed length confirmed in the log: `results/tinystories_train.log:6` -> `[21:25:09] packed 200,068 val tokens`. No cross-document attention masking — EOS separator only, attention flows across story boundaries (documented as a known simplification at SmolLM2-134(base)/README.md:792-795). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_tinystories.py:172-181 matches the quote verbatim (I read the file; :172 = ` val_buf = []`, :179 = ` if len(val_buf) >= 200_000:`). results/tinystories_train.log:6 = `[21:25:09] packed 200,068 val tokens` — verbatim. Independently reproduced today: 200,068 tokens, 1,040 stories, 0 empties skipped, eos_token_id = 0 (`<|endoftext|>`). Cross-doc caveat CONFIRMED: SmolLM2-134(base)/README.md:792-795 reads `- *Cross-document attention masking.* Real pretraining packs multiple / documents into one sequence and masks attention so a position in doc A can't / see tokens from doc B. We pack with a simple EOS separator and let attention / flow freely — fine for a tiny demo, sloppy for real runs.` One scoping nuance a reviewer may raise: that passage sits under README.md:785 `### What this script intentionally does *not* do` inside §10 (README.md:751), which is written about train.py — but train_tinystories.py:38 imports PackedTextDataset from train.py, so the simplification does apply to this run. +``` + + +### 2.3 Sequence length and stride used for that eval + +**Value** + +``` +seq_len = 1024, stride = 1024 (NON-OVERLAPPING windows; stride == seq_len). Window count capped at max_windows=200. Targets are the 1023 shifted tokens per window. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:68` + +**Source quote** + +``` +def evaluate(model, val_tokens, device, seq_len: int, max_windows: int = 200): + """Sliding-window CE perplexity over val_tokens (no overlap).""" + ... + for begin in range(0, min(len(val_tokens) - seq_len, max_windows * seq_len), seq_len): + ids = val_tokens[begin:begin + seq_len].unsqueeze(0).to(device) + logits = model(ids)["logits"][..., :-1, :].float() + labels = ids[..., 1:] +``` + +**Confidence** — measured from code + +**Caveat** — IMPORTANT: stride is 1024, NOT 512. The stride-512 setting belongs to a DIFFERENT eval — `SmolLM2-134(base)/eval_after_vs_base.py:50` (`def ppl(model, text, seq=1024, stride=512, max_windows=200)`) and the wikitext-2 parity eval (`results/perplexity.json` -> "seq_len": 1024, "stride": 512). eval_after_vs_base.py never produced output (see separate fact). Any writeup that attaches "stride 512" to the 6.8945/3.7900 pair would be wrong. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +STRONGEST-VERIFIED FACT IN THE SET, and I can add a hard proof the agent did not give. train_tinystories.py:63 and :68 match the quote verbatim (`for begin in range(0, min(len(val_tokens) - seq_len, max_windows * seq_len), seq_len):`). The stride=1024 claim is not merely code-read, it is PROVEN by the recorded token count: with stride 1024 → range(0, min(200068-1024, 204800)=199044, 1024) = 195 windows × 1023 = 199,485 = exactly the logged value. With stride 512 the same buffer would give 200 windows (max_windows binds) × 1023 = 204,600 ≠ 199,485. So stride 512 is arithmetically excluded. Contrast eval CONFIRMED: eval_after_vs_base.py:50 = `def ppl(model, text, seq=1024, stride=512, max_windows=200):` (grep -n, exact line). results/perplexity.json confirmed: {"dataset": "wikitext-2-raw-v1 validation", "seq_len": 1024, "stride": 512}. ONE IMPRECISION: the value says "Window count capped at max_windows=200" — the cap did NOT bind (199,044 < 204,800); the buffer length bound at 195 windows. Phrase it as "max_windows=200 cap present but not reached; 195 windows actually scored." +``` + + +### 2.4 Tokenizer used for that eval + +**Value** + +``` +`AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M")` — the official SmolLM2 BPE, vocab 49,152. Same tokenizer for train packing and eval packing. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:144` + +**Source quote** + +``` +tokenizer = AutoTokenizer.from_pretrained(REPO) +(SmolLM2-134(base)/verify.py:19:) REPO = "HuggingFaceTB/SmolLM2-135M" +``` + +**Confidence** — measured from code + +**Caveat** — vocab_size 49152 independently confirmed from the checkpoint's stored config (read live: ck['config']['vocab_size'] == 49152). No local HF cache entry for HuggingFaceTB/SmolLM2-135M exists today (~/.cache/huggingface/hub contains only CACHEDIR.TAG and models--Qwen--Qwen3.5-9B), so a re-run re-downloads unpinned. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +VALUE IS CORRECT. The caveat sentence "No local HF cache entry for HuggingFaceTB/SmolLM2-135M exists today (~/.cache/huggingface/hub contains only CACHEDIR.TAG and models--Qwen--Qwen3.5-9B)" is FALSE and must be deleted. Correct statement: HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache; that cache holds models--HuggingFaceTB--SmolLM2-135M with refs/main = 93efa2f097d58c2a74874c7e644dbc9b0cee75a2, blobs dated 2026-05-13 11:44-11:47 (before the run). +``` + +**Verifier note** + +``` +VALUE CONFIRMED: train_tinystories.py:144 = ` tokenizer = AutoTokenizer.from_pretrained(REPO)`; verify.py:19 = `REPO = "HuggingFaceTB/SmolLM2-135M"`. Live-loaded the tokenizer today: vocab_size 49152, eos_token_id 0. ck['config']['vocab_size'] == 49152 confirmed by torch.load. CAVEAT REFUTED: the agent checked ~/.cache/huggingface/hub, but `env | grep HF_` shows HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache — the default cache is not the active one. The real cache contains the model snapshot (config.json, model.safetensors, tokenizer.json, merges.txt, vocab.json) at revision 93efa2f097d58c2a74874c7e644dbc9b0cee75a2. This is a materially wrong statement about reproducibility and would embarrass the card if a reviewer ran `env`. +``` + + +### 2.5 Eval numerical precision + +**Value** + +``` +Model held in torch.bfloat16 on cuda; logits upcast to fp32 (.float()) before cross_entropy with reduction="sum"; PPL = exp(sum(nll)/n_target_tokens). +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:70` + +**Source quote** + +``` +logits = model(ids)["logits"][..., :-1, :].float() + labels = ids[..., 1:] + loss = F.cross_entropy(logits.reshape(-1, logits.size(-1)), + labels.reshape(-1), reduction="sum") + nlls.append(loss.item()) + n += labels.numel() + ... + return math.exp(sum(nlls) / n), n +``` + +**Confidence** — measured from code + +**Caveat** — bf16 confirmed at results/tinystories_train.log:1 -> `[21:23:28] Device: cuda dtype: torch.bfloat16`. So 6.8945 is a bf16-forward number, not the fp32 number a parity-grade eval would give. + +**Verdict** — _no 1:1 verifier entry; see Additional verifier findings below._ + + +### 2.6 How many eval target tokens? (prose says 199,485 — verify) + +**Value** + +``` +199,485 — VERIFIED, and reproduced by arithmetic: 200,068 packed val tokens -> range(0, min(200068-1024, 200*1024), 1024) yields 195 windows -> 195 x 1023 target tokens = 199,485. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_before.txt:2` + +**Source quote** + +``` +Validation PPL: 6.8945 (199,485 target tokens) +(results/tinystories_after.txt:2:) Validation PPL: 3.7900 (199,485 target tokens) +(results/tinystories_train.log:10:) [21:25:14] baseline TinyStories-val PPL = 6.895 on 199,485 target tokens +``` + +**Confidence** — results JSON + +**Caveat** — I re-executed the window arithmetic against the on-disk evaluate() and the logged 200,068 val tokens; it lands exactly on 195 windows x 1023 = 199,485. Both before and after used the identical val_tokens tensor in the same process, so the two PPLs are strictly paired on the same tokens. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Three independent on-disk sources verified verbatim: results/tinystories_before.txt:2 = `Validation PPL: 6.8945 (199,485 target tokens)`; results/tinystories_after.txt:2 = `Validation PPL: 3.7900 (199,485 target tokens)`; results/tinystories_train.log:10 = `[21:25:14] baseline TinyStories-val PPL = 6.895 on 199,485 target tokens`. I re-ran the window arithmetic in Python: range(0, min(200068-1024, 200*1024), 1024) → 195 begins → 195*1023 = 199485. Pairing claim CONFIRMED structurally: val_tokens is built once at train_tinystories.py:181 and passed unchanged to evaluate() at :196 and :362. Minor labelling nit only: confidence is tagged "results-json" but the cited artifacts are .txt/.log, not JSON. +``` + + +### 2.7 Was 6.8945 measured by this repo, or copied from a paper/model card? + +**Value** + +``` +MEASURED by this repo. It is the official HuggingFaceTB/SmolLM2-135M safetensors loaded into this repo's own SmolLM2ForCausalLM (via load_official_weights_into_ours), cast to bf16 on cuda, then scored by the same evaluate() before any optimizer step. Not copied from anywhere. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:196` + +**Source quote** + +``` +if args.resume is None: + log("Baseline (BEFORE) eval...", log_path) + base_ppl, base_n = evaluate(model, val_tokens, device, args.seq_len) +(lines 144-149:) tokenizer = AutoTokenizer.from_pretrained(REPO) + hf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) + model = SmolLM2ForCausalLM(SmolLM2Config()) + load_official_weights_into_ours(model, hf.state_dict()) + del hf + model = model.to(device=device, dtype=dtype) +``` + +**Confidence** — measured from code + +**Caveat** — "Base checkpoint" here = the OFFICIAL HF weights, not a repo-trained base. Full-precision value read live out of the checkpoint: ck['baseline_ppl'] = 6.894546783281595. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_tinystories.py:144-149 verified verbatim (AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) → SmolLM2ForCausalLM(SmolLM2Config()) → load_official_weights_into_ours → .to(device, dtype)). Lines 194-197 verified: the baseline evaluate() call at :196 sits BEFORE the AdamW construction at :213, so "before any optimizer step" is structurally guaranteed. Live-read from checkpoint_tinystories.pt via torch.load(map_location='cpu'): ck['baseline_ppl'] = 6.894546783281595 — matches the 4-dp prose exactly. The caveat's clarification that "base checkpoint" = official HF weights, not a repo-trained base, is correct and material — keep it on the card. This is unambiguously a repo measurement, not a copied number. +``` + + +### 2.8 Was 3.7900 measured on the continued-pretrained checkpoint? + +**Value** + +``` +YES. Measured in-process at end of training by the same evaluate() on the same val_tokens. Full-precision value stored in the checkpoint: trained_ppl = 3.7899503859716885 (baseline_ppl = 6.894546783281595). Exact delta = -45.0297%, which rounds to the cited -45.0%. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:362` + +**Source quote** + +``` +trained_ppl, trained_n = evaluate(model, val_tokens, device, args.seq_len) +(results/tinystories_train.log:508:) [23:21:25] AFTER PPL = 3.790 (BEFORE was 6.895; improvement +3.105 = +45.0%) +``` + +**Confidence** — results JSON + +**Caveat** — I read the two floats live out of checkpoint_tinystories.pt (torch.load, map_location='cpu'). This is n=1: one seed, one corpus, no across-seed CI, no iso-FLOP control arm, no downstream evals. Under CLAUDE.md's rigor bar (§C10/§C17/§C18/§C25) this is a directional in-domain result, not a `win`. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_tinystories.py:362 = ` trained_ppl, trained_n = evaluate(model, val_tokens, device, args.seq_len)` — verbatim. results/tinystories_train.log:508 = `[23:21:25] AFTER PPL = 3.790 (BEFORE was 6.895; improvement +3.105 = +45.0%)` — verbatim at the cited line number. Live torch.load gives ck['trained_ppl'] = 3.7899503859716885, ck['baseline_ppl'] = 6.894546783281595; I computed 100*(b-t)/b = 45.029738645593945 → -45.0% correct to the stated precision and sign. The n=1 / no-CI / no-iso-FLOP-control / no-downstream caveat is correct and, per CLAUDE.md §C10/§C17/§C18/§C25, MUST ship with the number — it caps this at `directional`, not `win`. +``` + + +### 2.9 Training config — tokens, steps, seq len, tokens/step (MEASURED) + +**Value** + +``` +token_budget 100,000,000; final tok_seen 99,999,744; total_steps 24,414; seq_len 1024; tokens per optimizer step 4,096; 99,609 packed train windows from 102,000,116 packed train tokens. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_train.log:5` + +**Source quote** + +``` +[21:25:09] packed 102,000,116 train tokens in 97.9s +[21:25:09] packed 200,068 val tokens +[21:25:09] 99,609 train windows of 1024 +[21:25:09] tok/step = 4,096 total_steps = 24,414 +``` + +**Confidence** — measured from code + +**Caveat** — Cross-checked: 100_000_000 // 4096 = 24,414 exactly; results/tinystories_train.csv has 24,414 data rows; last row is `24414,1.3074886798858643,0.0,99999744`; ck['step']=24414, ck['tok_seen']=99999744. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +results/tinystories_train.log:5-8 verified verbatim at the cited line numbers (`packed 102,000,116 train tokens in 97.9s` / `packed 200,068 val tokens` / `99,609 train windows of 1024` / `tok/step = 4,096 total_steps = 24,414`). CSV: 24,415 lines = 1 header + 24,414 data rows (wc -l); last row read directly = `24414,1.3074886798858643,0.0,99999744`. torch.load gives ck['step'] = 24414, ck['tok_seen'] = 99999744. 100_000_000 // 4096 = 24414 exactly. Every element independently reproduces. +``` + + +### 2.10 Micro batch / grad accum / global batch + +**Value** + +``` +Global batch = 4,096 tokens = 4 sequences of 1024 per optimizer step (MEASURED as the product). The claimed split micro_batch=4 x grad_accum=1 is PROSE-ONLY — nothing in the run artifacts records the two factors separately. +``` + +**Evidence** — `SmolLM2-134(base)/results/POST_DATA.md:52` + +**Source quote** + +``` +**0.01** (on 2D params only), grad_clip 1.0, bf16, seq_len 1024, micro_batch 4. +(SmolLM2-134(base)/train_tinystories.py:189:) tok_per_step = args.seq_len * args.micro_batch * args.grad_accum +``` + +**Confidence** — PROSE ONLY + +**Caveat** — Only the product (4,096) is logged. micro_batch=4 / grad_accum=1 are the argparse defaults of the CURRENT on-disk script (train_tinystories.py:106-107), which is NOT the version that produced this run (see the script-drift fact). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +results/POST_DATA.md:52 = ` **0.01** (on 2D params only), grad_clip 1.0, bf16, seq_len 1024, micro_batch 4.` — verbatim at the cited line. train_tinystories.py:189 = ` tok_per_step = args.seq_len * args.micro_batch * args.grad_accum` — verbatim. Only the product is logged (log:8), so the factorization is genuinely unrecoverable from artifacts. Confirming detail the agent missed: results/tinystories_summary.md:74-75 tabulates `| Micro batch | 4 | this run |` and `| Grad accumulation | 1 | this run |` — but summary.md documents the PRIOR run (see the two-runs fact), so it is not evidence for the 116.1-min run either. The prose-only verdict holds and is if anything better supported. +``` + + +### 2.11 LR schedule (MEASURED from the per-step LR trace) + +**Value** + +``` +WSD: linear warmup 200 steps to peak 3e-4; stable 3e-4 through step 19,531; linear decay over the final 20% (decay_start = int(24414*0.8) = 19,531, first decayed LR at step 19,532 = 2.9993856e-4 = 3e-4*(1 - 1/4883)) to exactly 0.0 at step 24,414. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_train.csv:19533` + +**Source quote** + +``` +1,1.9550740718841553,1.4999999999999998e-06,4096 (csv:2 -> 3e-4 * 1/200) +200,1.7322626113891602,0.0003,819200 (csv:201) +19531,1.4177279472351074,0.0003,79998976 (csv:19532) +19532,1.4647554159164429,0.00029993856235920535,80003072 (csv:19533) +24414,1.3074886798858643,0.0,99999744 (csv:24415) +``` + +**Confidence** — measured from code + +**Caveat** — Schedule shape matches make_wsd_scheduler in SmolLM2-134(base)/train.py:45-56 exactly (`decay_start = int(total_steps * (1.0 - decay_frac))`), which pins peak_lr=3e-4, warmup_steps=200, decay_frac=0.20 from the data, not from prose. LR recorded is sched.get_last_lr() after sched.step(), i.e. the LR for the following step. + +**Verdict** — _no 1:1 verifier entry; see Additional verifier findings below._ + + +### 2.12 Optimizer, betas, eps, weight decay, grad clip, seed + +**Value** + +``` +NOT_FOUND in any run artifact. Claimed values (AdamW, betas (0.9, 0.95), eps 1e-8, weight_decay 0.01 on 2D params only, grad_clip 1.0, seed 0) exist ONLY in prose + in the argparse defaults of the current on-disk script. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:213` + +**Source quote** + +``` +optim = AdamW( + [{"params": decay, "weight_decay": args.weight_decay}, + {"params": no_decay, "weight_decay": 0.0}], + lr=args.peak_lr, betas=(0.9, 0.95), eps=1e-8, + ) +(prose, SmolLM2-134(base)/README.md:101-102:) recipe AdamW(0.9, 0.95), peak LR **3e-4** ... wd 0.01, grad-clip 1.0, bf16 +``` + +**Confidence** — PROSE ONLY + +**Caveat** — The run log contains NO `args=` line and NO seed line (grep for 'args=' across results/tinystories_train.log returns nothing; log:1 is only `Device: cuda dtype: torch.bfloat16`). The saved checkpoint has NO 'training_recipe' key (live-read keys: ['model','config','step','tok_seen','baseline_ppl','trained_ppl']). The CSV has no grad_norm column, so gradient clipping left no trace at all. These hyperparameters were never recorded for this run. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +CONFIRMED and STRENGTHENED. Absence verified by direct grep on results/tinystories_train.log: `grep -c "args="` → 0; `grep -in seed` → no matches; `grep -c "\[eval @ step"` → 0; `grep -c "\[ckpt @ step"` → 0. Live torch.load: list(ck.keys()) == ['model','config','step','tok_seen','baseline_ppl','trained_ppl'] — no 'training_recipe'. CSV header is `step,loss,lr,tok_seen` — no grad_norm column, so clipping left no trace. train_tinystories.py:213-217 and README.md:101-102 both quoted verbatim and correct. ADDITIONAL EVIDENCE THE AGENT MISSED, which strengthens the finding: results/training_recipe_resolved.json exists and does list AdamW / betas [0.9,0.95] / eps 1e-8 / weight_decay 0.01 / clip_grad 1.0 — but its own line 2 declares `"source": "https://github.com/huggingface/smollm/blob/main/text/pretraining/smollm2/config_smollm2_135M.yaml"` and its values are the UPSTREAM FROM-SCRATCH config (lr 0.003, warmup 2000, seq_len 2048, 2M steps), not this run. results/tinystories_summary.md:80-82 likewise sources wd/clip to "nanotron config_smollm2_135M.yaml" and the optimizer to "paper §4.1". So these hyperparameters are COPIED FROM AN EXTERNAL CONFIG, not measured here — an even stronger reason to keep them off a model card as measured values. +``` + + +### 2.13 Precision, wall-clock, throughput, GPU + +**Value** + +``` +Precision bf16. Training-loop wall clock 116.1 min; whole process 21:23:28 -> 23:21:27 = ~118.0 min (includes 97.9 s tokenization + before/after evals + generations). Mean throughput 14,356 tok/s (= 99,999,744 / (116.1*60) = 14,355). GPU: the log records only `Device: cuda`. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_train.log:506` + +**Source quote** + +``` +[23:21:17] step 24400/24414 loss 1.4281 lr 8.60e-07 tok/s 14,356 tok 99.9M/100M ETA 0.1 min +[23:21:21] Training complete in 116.1 min. +(log:1:) [21:23:28] Device: cuda dtype: torch.bfloat16 +``` + +**Confidence** — measured from code + +**Caveat** — "NVIDIA GB10" is PROSE-ONLY for this run (README.md:62, results/POST_DATA.md:17) — the log never records a device name. `nvidia-smi` on this box today returns `NVIDIA GB10`, which makes it near-certain but is present-day corroboration, not a run record. The logged tok/s is a cumulative average (tok_seen/elapsed), so POST_DATA.md:55-56's "~14,300 tok/s through the second hour" describes the running mean, not an instantaneous second-hour rate. Peak memory was not recorded (no peak_mem column in the CSV). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +All numbers verified. log:1 = `[21:23:28] Device: cuda dtype: torch.bfloat16`; log:505 = `[23:21:17] step 24400/24414 loss 1.4281 lr 8.60e-07 tok/s 14,356 tok 99.9M/100M ETA 0.1 min`; log:506 = `[23:21:21] Training complete in 116.1 min.` (NOTE: evidence_path says :506 but the first quoted line is :505 — the 116.1-min headline is at :506, so the citation still lands on the load-bearing line). 21:23:28→23:21:27 = 117.98 min ✓. 99,999,744/(116.1*60) = 14,355.4 ✓. GB10-is-prose CONFIRMED: README.md:62 = `| TinyStories run wall-clock (NVIDIA GB10, bf16) | **116.1 min**, 100M tokens, 24,414 steps | ...`; POST_DATA.md:17 same claim; nvidia-smi today returns `NVIDIA GB10`. Cumulative-average claim CONFIRMED twice over: train_tinystories.py:331 computes tps from t0 set once before the loop, AND the log shows a monotone rise (12,766 @ step 50 → 14,302 @ 11,750 → 14,356 @ 24,400) which only a running mean does. So POST_DATA.md:55-56's "~14,300 tok/s through the second hour" is indeed the running mean, not an instantaneous rate. +``` + + +### 2.14 Which training script produced the run? + +**Value** + +``` +SmolLM2-134(base)/train_tinystories.py — but the on-disk copy is NOT the version that produced these results. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:256` + +**Source quote** + +``` +csv_w.writerow(["step", "loss", "lr", "grad_norm", "peak_mem_mb", "tok_seen"]) +(but results/tinystories_train.csv:1 is:) step,loss,lr,tok_seen +``` + +**Confidence** — measured from code + +**Caveat** — Five independent proofs of script drift: (1) script writes a 6-column CSV header, the CSV has 4 columns; (2) script:139 logs `device=... dtype=... seed=...`, log:1 reads `Device: cuda dtype: torch.bfloat16`; (3) script:140 logs `args={vars(args)}` — no such line in the log; (4) script defaults --eval_every 2000 and --ckpt_every 4000 would emit 12 `[eval @ step` and 6 `[ckpt @ step` lines — the log has 0 of each (grep -c returns 0,0); (5) script's save_ckpt writes training_recipe/optim/sched/rng keys, the actual checkpoint has none of them. File mtimes agree: train_tinystories.py 2026-05-19 23:16 vs results 2026-05-14 00:21. Git has a single commit (84a96c0) containing only the LATER version, so the run's exact source is not recoverable. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +All five drift proofs independently verified. (1) train_tinystories.py:256 writes a 6-column header; results/tinystories_train.csv:1 = `step,loss,lr,tok_seen` (4 cols). (2) script:139 emits `device={device} dtype={dtype} seed={args.seed}`; log:1 is `Device: cuda dtype: torch.bfloat16` — different capitalisation, no seed field. (3) script:140 emits `args={vars(args)}`; grep -c "args=" on the log → 0. (4) grep -c for `[eval @ step` and `[ckpt @ step` → 0 and 0, though defaults 2000/4000 over 24,414 steps would force 12 and 7 respectively. (5) script save_ckpt (262-294) writes training_recipe/optim/sched/rng_*; the actual ck has none. I ADD A SIXTH PROOF: log:10 reads `PPL = 6.895 on 199,485 target tokens` while script:197 formats `PPL={base_ppl:.3f} ({base_n:,} target tokens)` — different template. Git verified: `git log -- SmolLM2-134(base)/train_tinystories.py` → single commit 84a96c0; I extracted that blob and diffed it against the working tree — IDENTICAL, i.e. git holds only the later version. mtimes verified: train_tinystories.py 2026-05-19 23:16:42; results 2026-05-14 00:21. Run source is not recoverable. +``` + + +### 2.15 Which eval script produced 6.8945 / 3.7900? + +**Value** + +``` +The SAME training script's internal `evaluate()` (train_tinystories.py:62-78), called at line 196 (before) and line 362 (after). It was NOT eval_after_vs_base.py. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:62` + +**Source quote** + +``` +@torch.no_grad() +def evaluate(model, val_tokens, device, seq_len: int, max_windows: int = 200): + """Sliding-window CE perplexity over val_tokens (no overlap).""" +``` + +**Confidence** — measured from code + +**Caveat** — eval_after_vs_base.py exists and is the in-domain + OOD comparison script, but its declared outputs `results/tinystories_vs_base.json` and `results/tinystories_vs_base.md` DO NOT EXIST on disk (`ls results/tinystories_vs_base.json` -> No such file). It also would crash on its own fallback path: line 91 does `open("model.py")`, and model.py is absent from both disk and git. So no OOD/catastrophic-forgetting number for this checkpoint exists anywhere. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_tinystories.py:62-64 verified verbatim (`@torch.no_grad()` / `def evaluate(...)` / docstring `"""Sliding-window CE perplexity over val_tokens (no overlap)."""`). Absence of the alternative verified: eval_after_vs_base.py:138 = `with open(RESULTS / "tinystories_vs_base.json", "w") as f:` and :148 prints the save path, but `ls results/tinystories_vs_base.json` and `.md` both return No such file. eval_after_vs_base.py:91 = ` code_text = open("model.py").read() * 30` and model.py is absent from disk and from git ls-files — so the fallback path would indeed raise FileNotFoundError. Conclusion that no OOD/forgetting number exists for this checkpoint holds. +``` + + +### 2.16 Exact re-run command + +**Value** + +``` +NOT_FOUND as a recorded invocation. Nearest documented forms are in the root README quickstart. Best reconstruction: `cd "SmolLM2-134(base)" && python3 train_tinystories.py --token_budget 100_000_000` (everything else at script defaults). +``` + +**Evidence** — `README.md:170` + +**Source quote** + +``` +# Continued pretraining on TinyStories from official weights. +python train_tinystories.py --token_budget 10_000_000 # ~10M tokens for a quick run + +# Resume a run that died: +python train_tinystories.py --resume checkpoint_tinystories.pt --token_budget 100_000_000 +``` + +**Confidence** — PROSE ONLY + +**Caveat** — Three caveats. (a) The 100M run was NOT a --resume run: the log contains the baseline eval, which train_tinystories.py:194 skips whenever --resume is set, and the CSV carries a header (written only when resume is None). (b) Re-running the CURRENT script with those defaults would produce extra artifacts the original run does not have (mid-training eval CSV, periodic checkpoints, grad_norm/peak_mem columns) and would save optimizer/scheduler/RNG state. (c) train_tinystories.py does NOT `import safe_cuda` and there is no `sentinel.py preflight` in the path (imports at lines 17-39 are argparse/csv/math/pathlib/random/time/warnings/numpy/torch/datasets/transformers only) — re-running it as-is violates CLAUDE.md §C1 and §C6. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Root README.md:170-174 verified verbatim (`# Continued pretraining on TinyStories from official weights.` at :170, `python train_tinystories.py --token_budget 10_000_000` at :171, resume form at :174). Caveat (a) CONFIRMED: the log contains the baseline eval block, and script:194 (`if args.resume is None:`) skips it on resume; the CSV carries a header, written only at script:255-256 when resume is None. Caveat (b) CONFIRMED by the drift evidence. Caveat (c) CONFIRMED by direct grep: `grep -n "safe_cuda\|sentinel" train_tinystories.py` returns ZERO hits — the script imports only argparse/csv/math/pathlib/random/time/warnings/numpy/torch/datasets/transformers (lines 19-39). Re-running it as-is would violate CLAUDE.md §C1 (safe_cuda.guard before torch) and §C6 (sentinel preflight). That is a real, material warning and should stay attached to any published reproduce recipe. +``` + + +### 2.17 Where is the resulting checkpoint, filename, size, format? + +**Value** + +``` +/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)/checkpoint_tinystories.pt — 269,144,681 bytes (269.1 MB / 256.7 MiB), mtime 2026-05-14 00:21:27 +0100. Format: torch.save zip archive (uncompressed, method=store), a dict with keys ['model','config','step','tok_seen','baseline_ppl','trained_ppl']. +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:294` + +**Source quote** + +``` +torch.save(ck, args.ckpt_path) +(argparse default, line 124:) ap.add_argument("--ckpt_path", default="checkpoint_tinystories.pt") +(shell:) checkpoint_tinystories.pt: Zip archive data, at least v0.0 to extract, compression method=store +(stat:) checkpoint_tinystories.pt 269144681 bytes mtime=2026-05-14 00:21:27.908514546 +0100 +``` + +**Confidence** — measured from code + +**Caveat** — Keys read live via torch.load(map_location='cpu'). The 'model' state_dict holds 273 tensors, ALL torch.bfloat16, totalling 162,826,560 elements = 134,515,008 unique params + 28,311,552 duplicated (lm_head.weight stored separately despite tie_word_embeddings=true). 269,144,681 bytes is consistent with 162,826,560 x 2 bytes + pickle overhead. It carries NO optimizer/scheduler/RNG state, so it is not resumable in the sense the current script's --resume path advertises. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +stat -c '%s' → 269144681 exactly; stat mtime → 2026-05-14 00:21:27.908514546 +0100; `file` → `Zip archive data, at least v0.0 to extract, compression method=store`. torch.load(map_location='cpu') gives exactly those six keys. I independently verified the tensor accounting: 273 tensors, dtype Counter({'torch.bfloat16': 273}), total 162,826,560 elements, and 'lm_head.weight' IS present in the state_dict despite tie_word_embeddings=true — 162,826,560 = 134,515,008 + 28,311,552 (= 49152×576) exactly, so the duplication claim is arithmetically confirmed. 162,826,560 × 2 = 325,653,120 bytes > the 269,144,681 file size, so the fact's line "269,144,681 bytes is consistent with 162,826,560 × 2 bytes + pickle overhead" is ARITHMETICALLY WRONG as a consistency argument — the file is SMALLER than 2 bytes/elem would imply (likely the tied lm_head is stored as a storage alias rather than a second copy in the zip). Drop that sentence; it does not affect the size, format, or key claims, which all verify. The "not resumable" note is correct: script:229-244 expects optim/sched/rng_* keys the file lacks. +``` + + +### 2.18 Is the checkpoint version-controlled or backed up? + +**Value** + +``` +No. It is gitignored (`*.pt`) and untracked — it exists only on this box's local disk. No HF Hub copy and no export dir found (results/lm_eval/ and hf_export/ do not exist). +``` + +**Evidence** — `.gitignore:20` + +**Source quote** + +``` +# Checkpoints (270MB+ each; not for version control — use HF Hub or git-lfs) +*.pt +(git check-ignore -v output:) .gitignore:20:*.pt SmolLM2-134(base)/checkpoint_tinystories.pt +``` + +**Confidence** — measured from code + +**Caveat** — Given MEMORY.md's "Branch switch wipes gitignored evidence" guard, this 269 MB artifact is the sole copy of the -45% result's weights and is exactly the class of file that has been destroyed before on this repo. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +`git check-ignore -v` returns exactly `.gitignore:20:*.pt SmolLM2-134(base)/checkpoint_tinystories.pt`. .gitignore:19 is the comment `# Checkpoints (270MB+ each; not for version control — use HF Hub or git-lfs)` and :20 is `*.pt` (evidence_path :20 is the load-bearing line even though the quote starts at :19). `ls results/lm_eval` and `ls hf_export` → No such file or directory. USEFUL SCOPING THE AGENT DID NOT ADD: the surrounding EVIDENCE is safe — `git ls-files SmolLM2-134\(base\)/results` shows tinystories_train.log, tinystories_train.csv, tinystories_before.txt, tinystories_after.txt, tinystories_summary.md, POST_DATA.md and training_recipe_resolved.json are ALL tracked in git. Only the 269 MB weights are the single-copy artifact. That sharpens the MEMORY.md branch-switch risk to the weights alone. +``` + + +### 2.19 Does the config stored in the checkpoint match SmolLM2-135M? + +**Value** + +``` +Yes: vocab_size 49152, hidden_size 576, intermediate_size 1536, num_hidden_layers 30, num_attention_heads 9, num_key_value_heads 3, max_position_embeddings 8192, rope_theta 100000.0, rms_norm_eps 1e-05, tie_word_embeddings true, attention_bias false. +``` + +**Evidence** — `SmolLM2-134(base)/checkpoint_tinystories.pt:0` + +**Source quote** + +``` +ck['config'] = {"vocab_size": 49152, "hidden_size": 576, "intermediate_size": 1536, "num_hidden_layers": 30, "num_attention_heads": 9, "num_key_value_heads": 3, "max_position_embeddings": 8192, "rope_theta": 100000.0, "rms_norm_eps": 1e-05, "initializer_range": 0.041666666666666664, "tie_word_embeddings": true, "attention_bias": false, "attention_dropout": 0.0} +``` + +**Confidence** — results JSON + +**Caveat** — Binary artifact, so there is no line number — read live with torch.load. Matches the architecture table at SmolLM2-134(base)/results/POST_DATA.md:41-43. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +CONFIRMED and STRENGTHENED beyond what the agent did. It compared only against POST_DATA.md:41-43 (verified verbatim: `30 decoder layers · hidden 576 · intermediate 1536 · 9 Q heads / 3 KV heads (GQA / 3:1) · head_dim 64 · vocab 49,152 · RoPE θ=100,000 (split-halves) · RMSNorm · / SwiGLU · tied embeddings · no biases anywhere.`). I additionally diffed the checkpoint config against the OFFICIAL upstream config.json in the HF cache (/home/yashb98/projects/qwen-distill/hf_cache/hub/models--HuggingFaceTB--SmolLM2-135M/snapshots/93efa2f097d58c2a74874c7e644dbc9b0cee75a2/config.json): all eleven claimed fields match exactly, including initializer_range 0.041666666666666664, attention_dropout 0.0, rms_norm_eps 1e-05, max_position_embeddings 8192, tie_word_embeddings true, attention_bias false. Only cosmetic difference: upstream rope_theta is the int 100000, the checkpoint stores 100000.0. Binary artifact so ":0" as a line number is a placeholder, correctly flagged. +``` + + +### 2.20 Do the derived training-loss statistics in the prose check out? + +**Value** + +``` +Mostly. Best single-batch loss 0.9087928533554077 at step 22,353 -> "0.9088 @ step 22,353" CONFIRMED. First 1000-step bucket mean 1.5860 -> "1.586" CONFIRMED. But "1.316 (last)" is the 23000-24000 bucket (1.3162); the actual LAST bucket 24000-24414 means 1.3138. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_train.csv:22354` + +**Source quote** + +``` +22353,0.9087928533554077,0.00012662297767765718,91557888 +(claim at results/POST_DATA.md:19:) | **0.9088** | Best single-step training loss (step 22,353, deep in WSD decay) | `results/tinystories_train.csv` | +(claim at results/POST_DATA.md:57:) - Bucket-mean training loss (1000-step buckets): 1.586 (first) → **1.316** (last). +``` + +**Confidence** — measured from code + +**Caveat** — I recomputed min-loss and the bucket means directly from the 24,414-row CSV. The "1.316 (last)" label is off by one bucket — cosmetic, but it is a headline-adjacent number. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +I recomputed all of it from the 24,414-row CSV in Python. min loss = 0.9087928533554077 at step 22353 — and csv:22354 reads verbatim `22353,0.9087928533554077,0.00012662297767765718,91557888`. Bucket means: (0,1000] = 1.5859884355068208 → 1.586 ✓; (23000,24000] = 1.3161785732507705 → 1.316; (24000,24414] = 1.3138229582044814 (414 rows) → 1.314 at 3dp. POST_DATA.md:19 and :57 both quoted verbatim and correct as quotes. So the "1.316 (last)" label on POST_DATA.md:57 is genuinely off by one bucket. Worth noting the repo is internally inconsistent here: results/tinystories_summary.md:12 uses 1.313 for the last bucket, which is the (nearly) right value. +``` + + +### 2.21 Are there two different TinyStories runs described in the repo? + +**Value** + +``` +YES. results/tinystories_summary.md describes an EARLIER run: 137.3 min wall clock, after-PPL 3.7893, different generated samples. The 6.8945 -> 3.7900 / 116.1 min pair comes from the LATER run (the one in tinystories_train.log/.csv and the current checkpoint). +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_summary.md:9` + +**Source quote** + +``` +| TinyStories-val perplexity | **6.8945** | **3.7893** | **−45.0%** | +(line 14:) Wall clock: **137.3 minutes** on NVIDIA GB10, bf16. (Original estimate: 135 min — off by 2 min, ~1.5%.) +``` + +**Confidence** — measured from code + +**Caveat** — results/POST_DATA.md:165 does label it `results/tinystories_summary.md long-form write-up (prior run)`, and mtimes agree (summary.md 2026-05-13 22:07, i.e. before the final run's baseline eval at 22:25). But summary.md's recipe table (lines 67-82) and throughput table (lines 84-92, incl. "mean 12,150 tok/s", "max temp 72 C", "5 users on the box") describe the PRIOR run and must not be quoted as the recipe/throughput of the 116.1-min run. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Conclusion (two runs; summary.md = the earlier one) is CORRECT and well-evidenced. But "different generated samples" is only 2/3 true and must be tightened to: "all three BEFORE samples are byte-identical to results/tinystories_before.txt, and the AFTER sample for prompt 1 is byte-identical to results/tinystories_after.txt; only prompts 2 and 3 diverge in the AFTER column." +``` + +**Verifier note** + +``` +Core claim CONFIRMED on three independent axes. (1) results/tinystories_summary.md:9 = `| TinyStories-val perplexity | **6.8945** | **3.7893** | **−45.0%** |` vs after.txt's 3.7900 / ck['trained_ppl'] 3.78995. (2) summary.md:14 = `Wall clock: **137.3 minutes** on NVIDIA GB10, bf16.` vs log:506's 116.1 min. (3) POST_DATA.md:165 = `results/tinystories_summary.md long-form write-up (prior run)`. Cited ranges verified: the recipe table spans summary.md:67-82 and the throughput block 84-92, including `| Mean across full run | **12,150** |` at :90 and the `max temp 72 °C ... 5 users on the box` note at :92 — the warning not to quote these as the 116.1-min run's recipe/throughput is CORRECT and important. TWO CORRECTIONS. (a) I diffed the generations: summary.md:23-24 (prompt 1 AFTER, ending `...He started to climb`) is byte-identical to results/tinystories_after.txt, as are all three BEFORE samples; prompts 2 and 3 AFTER genuinely diverge (`He was very sad because he had no friends` vs `He was very strong and brave`; `a small fairy named Lila` vs `a small, beautiful bird who loved to whistle`). Identical BEFOREs are expected — set_seed(0) then a deterministic baseline eval leaves the same RNG state. (b) The mtime argument needs a stated correction factor: log timestamps run exactly 1 h behind file mtimes (before.txt mtime 2026-05-13 22:25:16 +0100 vs log `[21:25:16] Baseline generations`), so summary.md's 22:07:53 mtime = log-clock 21:07:53, ~16 min BEFORE the run's 21:23:28 start. The agent's "before the baseline eval at 22:25" silently mixes clocks but lands on the right conclusion. Also note summary.md's bucket means (1.586 / 1.339 / 1.324 / 1.312 / 1.313) match the CURRENT CSV to 3-4 digits (I computed 1.5860 / 1.3383 / 1.3240 / 1.3121 / 1.3138) — consistent with both runs using seed 0 and the same shuffle order, which is why POST_DATA could reuse "1.586". +``` + + +### 2.22 Was any out-of-domain / catastrophic-forgetting measurement made on the trained checkpoint? + +**Value** + +``` +NO. No post-training wikitext-2 PPL, no code PPL, no downstream benchmark results exist on disk for checkpoint_tinystories.pt. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_summary.md:125` + +**Source quote** + +``` +2. **Lower peak LR** (1e-4 instead of 3e-4): less catastrophic-forgetting risk + if you also care about preserving general-text quality. We didn't measure + wikitext-2 PPL post-training but it almost certainly got worse — that's the + tradeoff continued pretraining always makes. +``` + +**Confidence** — measured from code + +**Caveat** — Confirmed by absence: `results/tinystories_vs_base.json` (eval_after_vs_base.py's output) does not exist; `results/lm_eval/` (scripts/run_lm_eval.sh's output dir) does not exist; `results/tinystories_eval.csv` (mid-training PPL trace) does not exist. The wikitext-2 15.371 in results/perplexity.json is the BASE-vs-HF parity number, not a post-training number. So the -45.0% is an unbalanced, in-domain-only claim. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Quote verified but the line number is off by one: results/tinystories_summary.md:124 begins `2. **Lower peak LR** (1e-4 instead of 3e-4): less catastrophic-forgetting risk`, so the quoted block spans :124-127, not starting at :125 (:125 is ` if you also care about preserving general-text quality. We didn't measure`). Absence CONFIRMED by direct ls: results/tinystories_vs_base.json, results/tinystories_vs_base.md, results/lm_eval/, results/tinystories_eval.csv and hf_export/ ALL return "No such file or directory". scripts/run_lm_eval.sh:27 sets `OUT_DIR="results/lm_eval"` — never produced. results/perplexity.json verified: {"ours_ppl": 15.370989092449635, "hf_ppl": 15.370989964425396, "tokens": 62403, "dataset": "wikitext-2-raw-v1 validation", ...} — that is the BASE-vs-HF parity number, correctly not a post-training number. The "unbalanced, in-domain-only" framing is right and must ship with the -45.0%. +``` + + +### 2.23 Is `model.py`, cited as a source in the dossier, actually on disk? + +**Value** + +``` +NO — model.py does not exist in SmolLM2-134(base)/ (only model_full.py) and is not tracked in git. A stale __pycache__/model.cpython-312.pyc is the only trace. +``` + +**Evidence** — `SmolLM2-134(base)/results/POST_DATA.md:20` + +**Source quote** + +``` +| **198 lines** | `model.py` line count for the from-scratch architecture | `wc -l model.py` | +(shell: `ls model*.py` -> ) model_full.py +(shell: `git ls-files | grep -i smollm` -> lists model_full.py, no model.py) +``` + +**Confidence** — NOT FOUND + +**Caveat** — Also affects results/README.md:29 ("`param_count.log` — output of `python3 model.py`") and eval_after_vs_base.py:91 (`code_text = open("model.py").read() * 30`), which would raise FileNotFoundError. Peripheral to the PPL claim, but it is a broken citation in the same dossier. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +VALUE IS CORRECT. One citation in the caveat is wrong: the param_count.log/model.py reference is at results/README.md:25, NOT :29. results/README.md:29 is part of an unrelated bullet about training_recipe_resolved.json. results/README.md:4 also references `../model.py`. +``` + +**Verifier note** + +``` +VALUE CONFIRMED: `ls model*.py` → model_full.py only; `git ls-files | grep SmolLM2 | grep .py` lists model_full.py, train.py, train_tinystories.py, verify.py, generate.py, compare_with_hf.py, eval_after_vs_base.py, benchmark_training.py, _build_notebook.py, scripts/export_to_hf.py, tests/test_parity.py — no model.py. __pycache__/model.cpython-312.pyc exists (16,082 bytes, mtime 2026-05-19 10:46). POST_DATA.md:20 verified verbatim (`| **198 lines** | \`model.py\` line count ... | \`wc -l model.py\` |`), eval_after_vs_base.py:91 verified verbatim. CITATION ERROR: grep -n on results/README.md gives `25:- \`param_count.log\` — output of \`python3 model.py\` (param count + random-init forward).` and `4:\`../verify.py\` / \`../model.py\`)`. Line 29 is ` (warmup=2000, weight_decay=0.01, clip_grad=1.0, seq_len=2048, global_batch=512,` — a different bullet entirely. Note also that results/param_count.log IS tracked in git while its stated generator is gone, so the 198-line claim is unverifiable at any line number. +``` + + +### 2.V Additional verifier findings (no 1:1 extracted fact) + +**2.V1 — ✅ CONFIRMED** · LR schedule + +**Checked against** + +``` +WSD: 200-step linear warmup to 3e-4; stable through step 19,531; linear decay from 19,532 (2.9993856e-4) to 0.0 at 24,414. +``` + +**Verifier note** + +``` +All five CSV rows verified verbatim at the exact cited line numbers: csv:2 = `1,1.9550740718841553,1.4999999999999998e-06,4096`; csv:201 = `200,1.7322626113891602,0.0003,819200`; csv:19532 = `19531,1.4177279472351074,0.0003,79998976`; csv:19533 = `19532,1.4647554159164429,0.00029993856235920535,80003072`; csv:24415 = `24414,1.3074886798858643,0.0,99999744`. train.py:45-56 verified: `def make_wsd_scheduler(optimizer, warmup_steps, total_steps, decay_frac=0.2)` with `decay_start = int(total_steps * (1.0 - decay_frac))`. int(24414*0.8) = 19531; 3e-4*(1 - 1/(24414-19531)) = 3e-4*(1-1/4883) = 2.9993856235920535e-4 — matches the logged float to all 17 digits. 3e-4*(1/200) = 1.5e-6 matches csv:2. The "derived from data not prose" framing is fair. +``` + + +**2.V2 — ❌ WRONG** · GAP CHECK: "No dataset revision/sha pin ... and no local HF cache entry for either the dataset or HuggingFaceTB/SmolLM2-135M — a re-run is not byte-reproducible." + +**Checked against** + +``` +No local HF cache entry for either the dataset or the model; re-run not byte-reproducible. +``` + +**Corrected value** + +``` +Both caches EXIST at the active HF_HOME (/home/yashb98/projects/qwen-distill/hf_cache): hub/models--HuggingFaceTB--SmolLM2-135M @ 93efa2f097d58c2a74874c7e644dbc9b0cee75a2 (cached 2026-05-13 11:44-11:47) and hub/datasets--roneneldan--TinyStories @ f54c09fd23315a6f9c86f9dc80f725de7d8f9c64 (cached 2026-05-13 14:16-14:20). Both predate the run's 21:23 start. The surviving true statement is narrower: the CODE passes no revision= pin, so a re-run on a machine without this cache could resolve a different revision. +``` + +**Verifier note** + +``` +This is the one hard refutation in the set. The agent inspected ~/.cache/huggingface/hub, but `env | grep HF_` shows HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache, so the default path was never the active cache. `find /home/yashb98 -iname "*SmolLM2*"` and `-iname "*TinyStories*"` locate both snapshots there, with refs/main pinned to the shas above. I then re-ran the exact val packing loop against those cached artifacts today and reproduced results/tinystories_train.log:6 byte-for-byte (200,068 tokens). Byte-level data reproducibility is therefore DEMONSTRATED, not absent. +``` + + +**2.V3 — ⚠️ NEEDS QUALIFIER** · GAP CHECK: "How many of the 21,990 TinyStories validation stories were consumed to reach the 200,068-token eval buffer is not recorded, so the exact eval subset cannot be reconstructed without re-running the packer." + +**Checked against** + +``` +Not recorded; eval subset cannot be reconstructed without re-running the packer. +``` + +**Corrected value** + +``` +1,040 stories. The eval set is the first 1,040 non-empty stories of roneneldan/TinyStories split="validation" @ f54c09fd23315a6f9c86f9dc80f725de7d8f9c64 (0 empty stories skipped). +``` + +**Verifier note** + +``` +Literally true that no artifact records it, but I closed the gap: after `python3 sentinel.py preflight` (exit 0, mem_available=83%), I re-executed train_tinystories.py:172-181 verbatim on CPU with the cached tokenizer and dataset. Output: `packed tokens: 200068 stories consumed: 1040 empty skipped: 0`. The 200,068 figure matches log:6 exactly, which both closes the gap and independently validates that the run-era packing code was identical to the on-disk version despite the script drift. Restate the gap as "not recorded in any artifact, but deterministically recoverable — measured today as 1,040 stories." +``` + + +### 2.G Gaps — not determinable from disk + +- The actual optimizer hyperparameters of the 100M run (weight_decay, betas, eps, grad_clip, seed) are unrecoverable: the run log has no `args=` line, the checkpoint has no `training_recipe` key, the CSV has no grad_norm column, and git holds only a LATER version of train_tinystories.py (single commit 84a96c0). The cited values are that later version's argparse defaults, restated as prose. +- The micro_batch / grad_accum split cannot be separated — only their product with seq_len (4,096 tokens/step) is logged. micro_batch=4, grad_accum=1 is prose. +- The exact CLI invocation of the run was never recorded anywhere on disk. +- The GPU model is not recorded in any run artifact (log line 1 says only `Device: cuda`). `NVIDIA GB10` for this run is prose; nvidia-smi confirms the box's GPU today but that is present-day corroboration. +- No dataset revision/sha pin for roneneldan/TinyStories, and no local HF cache entry for either the dataset or HuggingFaceTB/SmolLM2-135M — a re-run is not byte-reproducible. +- How many of the 21,990 TinyStories validation stories were consumed to reach the 200,068-token eval buffer is not recorded, so the exact eval subset cannot be reconstructed without re-running the packer. +- Peak GPU memory for the run was not recorded (the run-era CSV has no peak_mem_mb column). +- No OOD / catastrophic-forgetting / downstream-benchmark number exists for checkpoint_tinystories.pt — eval_after_vs_base.py and scripts/run_lm_eval.sh both left no outputs on disk. +- No across-seed CI, no iso-FLOP control arm, no second corpus — the -45.0% is a single-seed, single-corpus, in-domain paired measurement, which under CLAUDE.md §C10/§C17/§C18/§C25 supports at most a `directional` claim. +- The exact source code that produced the run is gone (not in git, overwritten on disk 2026-05-19), so nothing beyond what the log/CSV/checkpoint recorded can be recovered. + +--- + +## 3. Qwen3 eval provenance (28.65 / 46.31 / 23.52 / 13.40) + +Audit dimension: Qwen3-0.6B eval provenance for 28.65 / 46.31 / 23.52 / 13.40 + +### 3.1 CRITICAL: Is 13.40 this repo's OWN measurement of the released Qwen3-0.6B-Base, or a number copied from the Qwen3 tech report / HF card? + +**Value** + +``` +OUR OWN MEASUREMENT. The repo downloaded the released HF checkpoint and evaluated it with its own eval code. Script: Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py, run 2026-06-09 16:51:36, result 13.400. It is NOT copied from any paper or model card. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py:47-51` + +**Source quote** + +``` +from transformers import AutoModelForCausalLM + t0 = time.time() + hf = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.bfloat16).to(device) + ppl_orig, n = eval_ppl(hf, val, device) + lines.append(f"ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = {ppl_orig:8.3f} ({n:,} tok, {time.time()-t0:.0f}s)") +``` + +**Confidence** — measured from code + +**Caveat** — The word 'published' used next to 13.40 in Qwen3-0.6B/PLOTS_INDEX.md:37 and Qwen3-0.6B/results_overview/plots/README.md:49 is MISLEADING wording — it means 'the published (released) model', not 'a published number'. The generator script make_overview_plots.py:18-19,51 correctly traces it to our own original_vs_repro.txt. Do not describe 13.40 on a model card as a reported/published figure. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Evidence quote exists verbatim at Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py:47-51. The script downloads and scores the real HF checkpoint: line 49 `hf = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.bfloat16)`, with REPO="Qwen/Qwen3-0.6B-Base" imported from train_qwen3 (eval_original_vs_repro.py:19; train_qwen3.py:59). Scoring uses the repo's own eval_ppl (eval_original_vs_repro.py:26-36), not an external number. The caveat is also confirmed: Qwen3-0.6B/PLOTS_INDEX.md:37 reads "vs published 13.40 dashed line" and Qwen3-0.6B/results_overview/plots/README.md:49-51 reads "The published Qwen3-0.6B-Base 13.40 is an EXTERNAL reference" — both ambiguous wordings; Qwen3-0.6B/results_overview/make_overview_plots.py:18-19 and :51 correctly trace ORIGINAL_PPL = 13.40 to original_vs_repro.txt. Agree the card must not call 13.40 a reported/published figure. +``` + + +### 3.2 13.40 — which artifact/log records it, and when? + +**Value** + +``` +results/original_vs_repro.txt line 2 = 13.400; stdout captured in results/original_eval_run2.log line 6, timestamped 2026-06-09 16:51:36. No .json exists — the only result files are .txt and .log. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/original_vs_repro.txt:1-2` + +**Source quote** + +``` +[2026-06-09 16:51:36] Original vs reproduction — val=300,000 tokens, 50 windows x 4096 +ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = 13.400 (204,800 tok, 21s) +``` + +**Confidence** — measured from code + +**Caveat** — Both files are git-tracked (verified with git ls-files --error-unmatch). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +original_vs_repro.txt:1-2 matches the quote character-for-character. original_eval_run2.log:6 identical. Both git-tracked (`git ls-files --error-unmatch` exit 0). "No .json" independently verified: repo-wide `grep -rl '13\.400\|13\.40' --include='*.json'` returns only Qwen3-0.6B/experiments/2026-06-24_qwen3-0.6b_data-dclm-vs-fineweb/c5_evidence.json (a methodology file for a different experiment), no result JSON. Minor timing nuance not in the fact: the 16:51:36 stamp is written at eval_original_vs_repro.py:43 at script start; 13.400 was measured ~21 s later (the `21s` field). An earlier attempt the same day (results/original_eval_wrapper.log, 11:19) crashed on an UnpicklingError at line 62 BEFORE printing any PPL — so there is no conflicting earlier value. +``` + + +### 3.3 13.40 — what metric exactly, on which HF dataset id / config / split? + +**Value** + +``` +FineWeb-Edu validation perplexity (NOT wikitext, NOT any standard benchmark). Corpus = HuggingFaceFW/fineweb-edu, config 'sample-10BT', split 'train' (streaming), from which a 300,000-token 'val' tail was carved. exp(mean token NLL) over 204,800 scored tokens. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:151` + +**Source quote** + +``` +ds = load_dataset("HuggingFaceFW/fineweb-edu", "sample-10BT", split="train", streaming=True) +``` + +**Confidence** — measured from code + +**Caveat** — NO dataset revision is pinned in the load_dataset call, and NO model revision is pinned in from_pretrained(REPO). Neither the fineweb-edu snapshot sha nor the Qwen3-0.6B-Base commit sha used on 2026-06-09 is recorded anywhere on disk. This number is therefore not exactly reproducible. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Dataset id/config/split CONFIRMED at the cited line. The '300,000-token val TAIL' clause is NOT supported by the cited path in its current state — it is only true of the historical splitter (git show e791875:Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:123-148). +``` + +**Verifier note** + +``` +train_qwen3.py:151 is verbatim: `ds = load_dataset("HuggingFaceFW/fineweb-edu", "sample-10BT", split="train", streaming=True)`. BUT the surrounding function at that path TODAY (train_qwen3.py:126-191) is the doc-disjoint seeded-hash splitter, which does NOT carve a tail; the tail behaviour is in commit e791875 line 141 `(buf if len(buf) < n_train else val).extend(ids + [eos])`. Citing the current file for a 'tail' is a citation mismatch (the fact list does disclose this in fact 17, but the card must not cite train_qwen3.py:151 as evidence for the tail). I independently corroborated that the 2026-06-09 caches came from the OLD splitter: torch.load on both caches shows keys == ['train','val'] with no 'decontam' key, whereas the new splitter saves {'train','val','decontam'} (train_qwen3.py:190). 'No revision pinned' also CONFIRMED — line 151 has no revision arg, eval_original_vs_repro.py:49 has no revision arg. 204,800 = 50x4096 confirmed by the loop bound at eval_original_vs_repro.py:30. +``` + + +### 3.4 13.40 — sequence length, stride, tokenizer, tokens evaluated? + +**Value** + +``` +seq_len 4096, stride 4096 (NON-overlapping windows, no sliding window), 50 windows = 204,800 scored tokens out of the 300,000-token val slice. Tokenizer = Qwen/Qwen3-0.6B-Base (the model's own tokenizer, vocab 151,936). Model loaded in bfloat16. Loss via chunked_cross_entropy (fp32 accumulation, chunk 8192). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py:21-22,30-36` + +**Source quote** + +``` +SEQ_LEN, MAX_WINDOWS = 4096, 50 +CACHE = HERE / "results" / "tokcache_133072000_300000.pt" +... + for begin in range(0, min(len(val) - SEQ_LEN, MAX_WINDOWS * SEQ_LEN), SEQ_LEN): + ids = val[begin:begin + SEQ_LEN + 1].unsqueeze(0).to(device) + out = model(input_ids=ids[:, :-1]) + logits = out.logits if hasattr(out, "logits") else out["logits"] + loss = chunked_cross_entropy(logits, ids[:, 1:]) * (ids.size(1) - 1) + nlls += loss.item(); n += ids.size(1) - 1 + return math.exp(nlls / max(1, n)), n +``` + +**Confidence** — measured from code + +**Caveat** — Tokenizer identity confirmed at train_qwen3.py:59 (REPO = "Qwen/Qwen3-0.6B-Base") and :281 (AutoTokenizer.from_pretrained(REPO)); eval_original_vs_repro.py imports REPO from train_qwen3 (line 19). + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +All correct EXCEPT the vocab attribution: 151,936 is the MODEL config's vocab_size (Qwen3-0.6B/model.py:37 `vocab_size: int = 151_936 # config.json: vocab_size`), not the tokenizer's vocabulary size. len(tokenizer) is 151,669 (research/eval/private_heldout_v1/private_prose_v1.txt:456). +``` + +**Verifier note** + +``` +eval_original_vs_repro.py:21-22 and :30-36 match the quote verbatim. Stride == SEQ_LEN confirmed by `range(..., SEQ_LEN)` at line 30 → non-overlapping. bf16 at line 49. chunked_cross_entropy fp32 accumulation with chunk=8192 confirmed at train_qwen3.py:81-93 (`total = flat.new_zeros((), dtype=torch.float32)`, `flat[i:i+chunk].float()`). Tokenizer identity confirmed at train_qwen3.py:59 and :281. Do not write 'tokenizer vocab 151,936' on a card — say 'model vocab_size 151,936'. +``` + + +### 3.5 13.40 — how many training tokens for that arm? + +**Value** + +``` +36T tokens — but this is a COPIED figure, not measured here. It comes from the Qwen3 tech report summary transcribed into the build's training_plan.md. The '36T' annotation printed inside our own eval log is a hardcoded f-string label, not a measurement. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/training_plan.md:17-19` + +**Source quote** + +``` +Per the Qwen3 tech report (verbatim summary): + +- **Corpus:** 36T tokens across 119 languages +``` + +**Confidence** — PROSE ONLY + +**Caveat** — So 13.40 is OURS (measured) but the '36T' it is paired with is THEIRS (copied from arxiv.org/abs/2505.09388, cited as [qwen3paper] at Qwen3-0.6B/README.md:29). Keep that split explicit on a model card. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +training_plan.md:17-19 matches the quote verbatim ("Per the Qwen3 tech report (verbatim summary):" / "- **Corpus:** 36T tokens across 119 languages"). The '36T tok' string in our own log is indeed a hardcoded f-string literal at eval_original_vs_repro.py:51, not a measurement. Citation [qwen3paper]: https://arxiv.org/abs/2505.09388 confirmed at Qwen3-0.6B/README.md:29. The measured/copied split is correctly drawn. +``` + + +### 3.6 28.65 — which run/build produced it? + +**Value** + +``` +Build 1, faithful reproduction: Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b, run_name 'baseline2tpp', checkpoint_qwen3_baseline2tpp.pt. It is the post-training AFTER eval at the final step 18,150. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log:396` + +**Source quote** + +``` +[18:05:19] AFTER val PPL=28.65 (BEFORE 185810.49; improvement +185781.84 = +100.0%) +``` + +**Confidence** — measured from code + +**Caveat** — Also written to results/qwen3_baseline2tpp_after.txt:2 ("val PPL: 185810.49 -> 28.65"). Single run, single seed (seed 0), no CI. Fresh run, not resumed (log:12 'Training to 18,150 steps (starting at 0)'). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +qwen3_baseline2tpp_train.log:396 matches verbatim: `[18:05:19] AFTER val PPL=28.65 (BEFORE 185810.49; improvement +185781.84 = +100.0%)`. Corroborated at qwen3_baseline2tpp_after.txt:2 `val PPL: 185810.49 -> 28.65`. Fresh (not resumed) confirmed at log:12 `Training to 18,150 steps (starting at 0)` and log:2 `'resume': None`. Single seed 0 confirmed at log:1 and log:2. +``` + + +### 3.7 28.65 — metric, dataset, seq len, stride, tokenizer, tokens evaluated? + +**Value** + +``` +FineWeb-Edu val PPL, same eval function as 13.40 (train_qwen3.evaluate): seq_len 4096, stride 4096 non-overlapping, max_windows 50 = 204,800 scored tokens, tokenizer Qwen/Qwen3-0.6B-Base, bf16, chunked fp32 CE. Dataset HuggingFaceFW/fineweb-edu / sample-10BT / split=train (streaming). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:195,199-207` + +**Source quote** + +``` +def evaluate(model, val_tokens, device, seq_len: int, max_windows: int = 50): +... + for begin in range(0, min(len(val_tokens) - seq_len, max_windows * seq_len), seq_len): + ids = val_tokens[begin:begin + seq_len + 1].unsqueeze(0).to(device) + logits = model(ids[:, :-1])["logits"] + loss = chunked_cross_entropy(logits, ids[:, 1:]) * (ids.size(1) - 1) + nlls += loss.item() + n += ids.size(1) - 1 + ... + return math.exp(nlls / max(1, n)), n +``` + +**Confidence** — measured from code + +**Caveat** — CRITICAL: 28.65 was measured on a DIFFERENT 300k val slice than 13.40 — see the dedicated finding below. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_qwen3.py:195 and :199-207 match the quote verbatim. max_windows defaults to 50 and is never overridden — all three call sites pass only (model, val_tokens, device, args.seq_len): train_qwen3.py:341, :412, :426. stride == seq_len via `range(..., seq_len)` at :199. The cross-slice caveat is the correct one to carry forward. +``` + + +### 3.8 28.65 — training tokens / steps / recipe? + +**Value** + +``` +18,150 optimizer steps x 65,536 tok/step = 1,189,478,400 tokens (1.19B, ~2 tokens-per-parameter). AdamW betas (0.9,0.95) eps 1e-8, cosine peak_lr 2.4e-3 -> end_lr 3.2e-4, warmup 900, weight_decay 0.01, grad_clip 1.0, bf16, seq_len 4096, micro_batch 4 x grad_accum 4, seed 0, torch.compile on. ~2663 min wall (~44 h) at ~7,480 tok/s. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log:2-3` + +**Source quote** + +``` +[21:14:18] args={'steps': 18150, 'seq_len': 4096, 'micro_batch': 4, 'grad_accum': 4, 'peak_lr': 0.0024, 'end_lr': 0.00032, 'warmup_steps': 900, 'weight_decay': 0.01, 'grad_clip': 1.0, 'mem_fraction': 0.85, 'seed': 0, 'dtype': 'bfloat16', 'log_every': 50, 'eval_every': 2000, 'ckpt_every': 2000, 'no_compile': False, 'run_name': 'baseline2tpp', ...} +[21:14:18] tok/step=65,536 steps=18,150 token_budget=1,189,478,400 +``` + +**Confidence** — measured from code + +**Caveat** — Wall-clock/throughput figures from builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md:167 and log:395 ('Training complete in 2663.1 min'). + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Everything confirmed EXCEPT the throughput: the run-average is ~7,444 tok/s, not ~7,480. 1,189,478,400 tok / (2663.1 min x 60) = 7,444 tok/s, and the log's own final counter reads 7,444 (qwen3_baseline2tpp_train.log:393). 7,480 is the step-100 reading (log:14) that README.md:167 generalised. +``` + +**Verifier note** + +``` +Args dict at log:2 matches the quote verbatim; log:3 gives tok/step=65,536, steps=18,150, token_budget=1,189,478,400. betas=(0.9,0.95) eps=1e-8 confirmed at train_qwen3.py:302 (`lr=args.peak_lr, betas=(0.9, 0.95), eps=1e-8`) — these are NOT in the log's args dict, so they come from source defaults, which is fine but worth stating. 'Training complete in 2663.1 min.' confirmed at log:395. If the card quotes throughput, quote 7,444 tok/s (measured) not README.md:167's 7,480. +``` + + +### 3.9 23.52 — which run/build produced it, and at which step? + +**Value** + +``` +Build 2, modernized 'IMU-1' bundle: Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b, run_name 'imu1_2tpp'. It is the IN-LOOP eval at STEP 18,000 — NOT the final step 18,150. train_imu1.py has no post-training AFTER eval, so no end-of-run number exists. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log:381` + +**Source quote** + +``` +[09:27:03] [eval @ 18000] val PPL=23.52 +``` + +**Confidence** — measured from code + +**Caveat** — MATERIAL ASYMMETRY: 23.52 is at 18,000 steps = 1,179,648,000 tokens, while 28.65 is at 18,150 steps = 1,189,478,400 tokens (0.83% more). The like-for-like same-step comparison is baseline 28.66 @ step 18000 (qwen3_baseline2tpp_train.log:389) vs IMU-1 23.52, so the -17.9% headline survives — but 23.52 vs 28.65 is not step-matched as printed. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +qwen3_imu1_2tpp_train.log:381 matches verbatim: `[09:27:03] [eval @ 18000] val PPL=23.52`. The log has exactly 386 lines and ends `[09:58:25] step 18150/18150 ...` / `[09:58:25] DONE` — no AFTER eval, confirming no end-of-run number exists. The step-asymmetry caveat is also confirmed: baseline 28.66 @ step 18000 at qwen3_baseline2tpp_train.log:389. Arithmetic checks: 23.52/28.66 = -17.94%, 23.52/28.65 = -17.91% — the -17.9% headline does survive the step-matching. This asymmetry is material and correctly flagged. +``` + + +### 3.10 23.52 — metric, dataset, seq len, stride, tokenizer, val slice? + +**Value** + +``` +Identical harness to 28.65: train_imu1.py imports evaluate + stream_tokens + PackedTextDataset directly from train_qwen3, and loaded the SAME token cache (tokcache_1191478400_300000.pt). seq 4096, stride 4096, 50 windows = 204,800 scored tokens, tokenizer Qwen/Qwen3-0.6B-Base, FineWeb-Edu sample-10BT. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log:3` + +**Source quote** + +``` +[18:05:40] loaded cached tokens from tokcache_1191478400_300000.pt (1,191,478,400 train + 300,000 val) +``` + +**Confidence** — measured from code + +**Caveat** — Cross-check: train_imu1.py:155-158 ('from train_qwen3 import stream_tokens, evaluate, PackedTextDataset' ... 'stream_tokens(tokenizer, token_budget + 2_000_000, 300_000, log_path)') and :156 AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base"). 23.52 and 28.65 ARE on the same val slice and are mutually comparable. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +qwen3_imu1_2tpp_train.log:3 matches verbatim. train_imu1.py:155 (`from train_qwen3 import stream_tokens, evaluate, PackedTextDataset`), :156 (`AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base")`), :158 (`stream_tokens(tokenizer, token_budget + 2_000_000, 300_000, log_path)`) all confirmed. train_imu1.py:202 calls `evaluate(model, val_tokens, device, seq)` with max_windows defaulted to 50. The faithful run built that same cache (baseline log:6 `streamed 1,191,478,748 train + 300,012 val tokens`, cache key n_train=18150*65536+2,000,000=1,191,478,400 per train_qwen3.py:283-285). Same-slice claim for this PAIR is sound. +``` + + +### 3.11 23.52 — training tokens / recipe? + +**Value** + +``` +1,179,648,000 tokens at the 23.52 eval point (18,000 x 65,536); full budget 18,150 steps = 1,189,478,400. Recipe: value-residual + layernorm-scaling + head-gating architecture (vr=ln=hg=True), NorMuon on 224 2-D matrices + AdamW on 198 1-D/embedding params, WSD schedule (normuon_lr 0.011 / adam_lr 0.006 defaults, 20% linear decay tail), chunked z-loss 1e-4, seq 4096, micro_batch 4 x grad_accum 4, seed 0. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log:1-2` + +**Source quote** + +``` +[18:05:35] device=cuda bundle: vr=True ln=True hg=True steps=18150 +[18:05:35] param split: 224 NorMuon (2D), 198 AdamW (1D/embed) tok/step=65,536 +``` + +**Confidence** — measured from code + +**Caveat** — The log does NOT echo the LR/z-weight values, so normuon_lr=0.011 / adam_lr=0.006 / z_weight=1e-4 / weight_decay=0.1 / warmup=50 / decay_frac=0.2 are the argparse DEFAULTS at train_imu1.py:96-107, not confirmed from the run record. If the launcher passed overrides they are not on disk in this log. + +**Verdict — ❌ WRONG** + +**Corrected value** + +``` +warmup_steps was 900, NOT 50. And the exact CLI IS on disk: Qwen3-0.6B/builds/phase_b_driver.sh:24 records `cd "$MOD" && python train_imu1.py --steps $S --warmup_steps $W $COMMON --run_name imu1_2tpp` with S=18150, W=900, COMMON="--eval_every 2000 --ckpt_every 2000 --log_every 50" (phase_b_driver.sh:14). normuon_lr 0.011 / adam_lr 0.006 / weight_decay 0.1 / decay_frac 0.2 / z_weight 1e-4 were NOT overridden, so those defaults do hold. +``` + +**Verifier note** + +``` +I confirmed warmup=900 empirically against the run's own LR ramp, which the defaults cannot produce: 0.011 x 50/900 = 6.111e-4 == log:5 `lr 6.11e-04`; 0.011 x 100/900 = 1.222e-3 == log:6 `lr 1.22e-03`; 0.011 x 400/900 = 4.889e-3 == log:12 `lr 4.89e-03`. Under warmup=50 the LR would already be at peak 0.011 by step 50. decay_frac=0.2 also confirmed empirically: decay tail starts at step 14,520; at step 17,750 the predicted lr = 0.011 x 400/3630 = 1.212e-3 == log:375 `lr 1.21e-03`. Token count 18,000 x 65,536 = 1,179,648,000 confirmed. Both the '50' value and the 'no CLI on disk' gap claim must be corrected before publication. +``` + + +### 3.12 46.31 — which run produced it, and what does it actually measure? + +**Value** + +``` +Phase-A LR sweep arm 'lr24' (peak_lr 2.4e-3) in the faithful build. AFTER val PPL = 46.31 on FineWeb-Edu val — the same in-loop metric as 28.65, NOT a wikitext or benchmark number. Sweep siblings: lr17 (1.7e-3) = 46.89, lr30 (3.0e-3) = 49.28; lr24 best. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_lr24_train.log:226` + +**Source quote** + +``` +[06:15:14] AFTER val PPL=46.31 (BEFORE 183922.14; improvement +183875.83 = +100.0%) +``` + +**Confidence** — measured from code + +**Caveat** — Also in results/qwen3_lr24_after.txt:2 ('val PPL: 183922.14 -> 46.31') and independently re-scored as 46.310 by eval_original_vs_repro.py (original_vs_repro.txt:4). Siblings from qwen3_lr17_after.txt:2 (46.89) and qwen3_lr30_after.txt:2 (49.28). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +qwen3_lr24_train.log:226 matches verbatim: `[06:15:14] AFTER val PPL=46.31 (BEFORE 183922.14; improvement +183875.83 = +100.0%)`. qwen3_lr24_after.txt:2 = `val PPL: 183922.14 -> 46.31`. Siblings: qwen3_lr17_after.txt:2 = 46.89, qwen3_lr30_after.txt:2 = 49.28. Re-score confirmed at original_vs_repro.txt:3-5 (46.892 / 46.310 / 49.276). One framing nit: the original_vs_repro re-score is the SAME eval code on the SAME cache, so it is a consistency check, not an independent measurement — do not present it on a card as corroboration by a second method. +``` + + +### 3.13 46.31 — training tokens / steps / recipe? + +**Value** + +``` +2,000 steps x 65,536 tok/step = 131,072,000 tokens (131M). AdamW, cosine peak_lr 2.4e-3 -> end_lr 3.2e-4, warmup 150, weight_decay 0.01, grad_clip 1.0, bf16, seq 4096, micro_batch 4 x grad_accum 4, seed 0, torch.compile on. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_lr24_train.log:2-3` + +**Source quote** + +``` +[01:21:12] args={'steps': 2000, 'seq_len': 4096, 'micro_batch': 4, 'grad_accum': 4, 'peak_lr': 0.0024, 'end_lr': 0.00032, 'warmup_steps': 150, ... 'run_name': 'lr24', ...} +[01:21:12] tok/step=65,536 steps=2,000 token_budget=131,072,000 +``` + +**Confidence** — measured from code + +**Caveat** — Confirms the orchestrator's framing: 46.31 is a SHORT 2k-step / 131M-token LR-selection run, not a headline model. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +qwen3_lr24_train.log:2 args dict and :3 (`tok/step=65,536 steps=2,000 token_budget=131,072,000`) match the quote verbatim, including warmup_steps: 150 and no_compile: False. The 'short LR-selection run, not a headline model' framing is correct — log:225 shows the run took only 293.4 min vs the baseline's 2663.1. +``` + + +### 3.14 46.31 — seq len, stride, tokenizer, tokens evaluated, val slice? + +**Value** + +``` +seq 4096, stride 4096 non-overlapping, 50 windows = 204,800 scored tokens, tokenizer Qwen/Qwen3-0.6B-Base, val slice tokcache_133072000_300000.pt — the SAME slice used for 13.40. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_lr24_train.log:6` + +**Source quote** + +``` +[01:21:13] loaded cached tokens from tokcache_133072000_300000.pt (133,072,000 train + 300,000 val) +``` + +**Confidence** — measured from code + +**Caveat** — GOOD NEWS: 46.31 vs 13.40 IS an apples-to-apples same-slice comparison (both from tokcache_133072000_300000.pt, both via 50x4096 non-overlapping windows). The '3.5x' gap at 131M tokens is internally consistent. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +qwen3_lr24_train.log:6 matches verbatim: `loaded cached tokens from tokcache_133072000_300000.pt (133,072,000 train + 300,000 val)`. That is the exact file hardcoded at eval_original_vs_repro.py:22 (`CACHE = HERE / "results" / "tokcache_133072000_300000.pt"`), so 46.31 and 13.40 really are same-slice, same-windowing. The 3.5x ratio is internally consistent (46.310/13.400 = 3.456). +``` + + +### 3.15 BLOCKER: Is the README's claim that all four numbers share one val slice true? + +**Value** + +``` +NO — IT IS FALSE. 13.40 and 46.31 were measured on tokcache_133072000_300000.pt; 28.65 and 23.52 on tokcache_1191478400_300000.pt. These are two DIFFERENT 300,000-token FineWeb-Edu slices. I verified this empirically: their val tensors have different sha1 (8ad9e246b0bf63bd vs ad3513719d0f81e4) and different leading tokens ([10879, 5547, 481, ...] vs [38131, 6022, 369, ...]). +``` + +**Evidence** — `Qwen3-0.6B/README.md:35-37` + +**Source quote** + +``` +All perplexities use **identical eval code on the identical 300k-token FineWeb-Edu +val slice** ([`eval_original_vs_repro.py`](builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py)), +so every row is directly comparable. +``` + +**Confidence** — measured from code + +**Caveat** — Root cause: under the ORIGINAL splitter (git show e791875:.../train_qwen3.py, stream_tokens line 33 'cache = RESULTS / f"tokcache_{n_train}_{n_val}.pt"' with val = the sequential stream continuation AFTER n_train tokens), a different n_train yields a different val tail. n_train was 133,072,000 for the sweep and 1,191,478,400 for the 2-TPP runs. CONSEQUENCE: the '2.14x gap vs the original' (28.65/13.40) and the '1.76x' (23.52/13.40) are CROSS-SLICE ratios and should NOT be stated on a model card as a like-for-like gap. The 3.5x (46.31/13.40) is same-slice and is fine. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +I reproduced this empirically rather than taking it on trust. torch.load(..., mmap=True) on both caches: tokcache_133072000_300000.pt val sha1 = 8ad9e246b0bf63bd, first10 = [10879, 5547, 481, 37969, 1935, 82, 481, 6467, 11, 18652]; tokcache_1191478400_300000.pt val sha1 = ad3513719d0f81e4, first10 = [38131, 6022, 369, 66863, 25471, 2757, 374, 3709, 5313, 6529]. Both len 300,000, dtype int64. Exact match to the claimed values. Qwen3-0.6B/README.md:35-37 quote is verbatim and is FALSE as written. Two nits: (a) the caveat cites 'stream_tokens line 33' for the old cache-key line — the actual line in `git show e791875:...train_qwen3.py` is 127, not 33 (substance correct, line number wrong); (b) the same false same-slice claim appears a SECOND time at Qwen3-0.6B/results_overview/plots/README.md:50 ('36T-token model evaluated on the same 300k-token val set'), so a card fix must touch both docs. The 2.14x and 1.76x ratios are cross-slice and must not be published as like-for-like gaps. +``` + + +### 3.16 How slice-sensitive is this metric in practice? + +**Value** + +``` +Very. The SAME faithful checkpoint (checkpoint_qwen3_baseline2tpp.pt) that scores 28.65 on its own 300k slice scores 24.55 on the dataset-forge held-out FineWeb-Edu split under text-lm-v2 windowing (SEQ=1024, STRIDE=512, 202 docs / 204,600 scored tokens) — a ~14% swing from slice + windowing alone. +``` + +**Evidence** — `research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md:102` + +**Source quote** + +``` +| **§C13 forgetting — FineWeb-Edu** (`fineweb_edu_sample10bt_heldout`, 202 docs / 204,600 scored tok) | 24.5514 | 24.7331 | **+0.18 (+0.74%)** | 8.0155 | **retained — not significant** | +``` + +**Confidence** — results JSON + +**Caveat** — The same ledger doc (line 113-114) mis-attributes 28.65 to eval_original_vs_repro.py; the actual source is the in-loop train_qwen3.evaluate() AFTER eval in qwen3_baseline2tpp_train.log:396 / qwen3_baseline2tpp_after.txt:2. eval_original_vs_repro.py only ever scored the HF original + lr17/lr24/lr30. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +research/ledger/runs/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning.md:102 matches verbatim. Backed by an actual results JSON: Qwen3-0.6B/experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/eval/brief_probes_results.json gives base_ppl 24.55139646501548, base_ckpt = .../checkpoint_qwen3_baseline2tpp.pt, n_tokens 204600, and an explicit field `"base_ppl_claim_readme": 28.65` with `"base_ppl_measured_vs_claim_delta": -4.09860353498452` (= -14.3%). Windowing SEQ=1024 STRIDE=512 MAX_WINDOWS=200 confirmed at the same ledger doc line 95. The caveat is confirmed too: lines 113-114 do mis-attribute 28.65 to eval_original_vs_repro.py; the real source is train_qwen3.evaluate's AFTER eval (log:396 / after.txt:2). +``` + + +### 3.17 Were the val slices behind these four numbers decontaminated / document-disjoint? + +**Value** + +``` +NO. All four numbers used caches built by the ORIGINAL splitter, which the repo's own current code calls leak-suspect. The doc-disjoint seeded-hash split + 13-gram decontamination was added LATER (caches carrying it are named with a _seed0_ suffix, e.g. tokcache_133072000_300000_seed0_Qwen3-0.6B-Base.pt dated 2026-06-18). The caches used here have no seed/tokenizer suffix and no decontam record. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:130-141` + +**Source quote** + +``` +"""Stream FineWeb-Edu sample-10BT, tokenize on the fly, and build a + DOCUMENT-DISJOINT, DECONTAMINATED train/val split (audit fix DATA-1/3): + + * each whole document is routed to train or val by a seeded hash + (`is_val_doc`), so train/val never share a document and no document spans + the boundary (the old code cut the stream by token count — val was the + sequential continuation of train, leak-suspect); + * val documents whose 13-gram word overlap with a bounded sample of train + documents exceeds `decontam_threshold` are DROPPED +``` + +**Confidence** — measured from code + +**Caveat** — results/decontam_report.json exists but is dated 2026-07-07 and belongs to a LATER cache (tokcache_422020224_300000_seed0_...). It does NOT cover the 133M or 1191M caches. Separately, for 13.40 specifically: the released Qwen3-0.6B-Base's 36T corpus may itself contain FineWeb-Edu / its CommonCrawl sources, so this slice is not verifiably held out FOR THAT MODEL. That is an inference from the tech-report data description, not something provable on disk. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_qwen3.py:130-141 matches the quote verbatim, including the self-indictment 'the old code cut the stream by token count — val was the sequential continuation of train, leak-suspect'. I proved the two caches predate the fix independently of filenames: neither contains a 'decontam' key (keys == ['train','val']), whereas the post-fix splitter saves {'train','val','decontam'} (train_qwen3.py:190). Cache-name scheme with seed/tokenizer suffix confirmed at train_qwen3.py:145; tokcache_133072000_300000_seed0_Qwen3-0.6B-Base.pt mtime 2026-06-18 13:45. decontam_report.json mtime 2026-07-07 15:13, matching tokcache_422020224_300000_seed0_*.pt (2026-07-07 15:13). One precision note: 'belongs to a LATER cache' is an mtime inference — decontam_report.json contains no cache-name field (its keys are split_seed, val_fraction, ngram_n, overlap_threshold, n_train_docs, n_val_docs_raw, docs_dropped, n_val_docs_kept, method, train_sample_docs_for_index) — but the operative claim (it does not cover the 133M/1191M caches) is proven by the missing 'decontam' key. The Qwen3-36T-corpus-overlap point is correctly labelled an inference, not disk evidence. +``` + + +### 3.18 Does this repo's own governance permit these four numbers as a model-card headline? + +**Value** + +``` +NO. The repo explicitly classifies the 28.65 / 23.52 / 29.54 val-PPL family as the 'founding-mistake metric', banned as a sole or headline signal by contract §C25.7.3, and reports it only as cross-check context. All four numbers are n=1, single-seed, no CI, in-distribution val PPL. +``` + +**Evidence** — `research/eval/base_eval_verdict.md:59` + +**Source quote** + +``` +- val-PPL headline (28.65 / 23.52 / 29.54) source `Qwen3-0.6B/PLOTS_INDEX.md:20,22,24`. It is **n=1 FineWeb val-PPL — the founding-mistake metric, banned as a sole/headline signal by §C25.7.3** — reported here as cross-check context only. Published Qwen3-0.6B reference is 13.40 (`PLOTS_INDEX.md:37`); the reproductions are ~1.76–2.21× above it, consistent with the ≤1.19B-token undertraining. +``` + +**Confidence** — PROSE ONLY + +**Caveat** — Corroborated at research/eval/per_stage_eval_batteries.md:9 ('the three-build pretraining headline ... was shipped on n=1 FineWeb val PPL — no downstream, no seed CI, no contamination performance-check'). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +research/eval/base_eval_verdict.md:59 matches the quote verbatim, including '**n=1 FineWeb val-PPL — the founding-mistake metric, banned as a sole/headline signal by §C25.7.3**'. Corroboration confirmed at research/eval/per_stage_eval_batteries.md:9 verbatim ('shipped on **n=1 FineWeb val PPL** — no downstream ... no seed CI, no contamination performance-check'). The 'prose-only' confidence label is the right one — these are governance documents, not results files. +``` + + +### 3.19 What ARE the §C10-comparable, suite-stamped numbers for the 28.65 checkpoint (safer for a model card)? + +**Value** + +``` +From /eval-harness suite text-lm-v2, run 2026-06-16 on checkpoint_qwen3_baseline2tpp.pt: wikitext2_raw_v1_val PPL 37.0101 / BPB 1.22562 (204,600 tokens, 869,710 bytes); codeparrot_clean_valid PPL 438.673 / BPB 2.12860 (204,600 tokens, 843,643 bytes). Windowing SEQ=1024, STRIDE=512, MAX_WINDOWS=200. Corpora revision-pinned. Tokenizer Qwen/Qwen3-0.6B-Base. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3-faithful_eval-first/eval/suite_results.json:1-25` + +**Source quote** + +``` +"suite_version": "text-lm-v2", + "tokenizer_repo": "Qwen/Qwen3-0.6B-Base", + "target_ckpt": ".../checkpoint_qwen3_baseline2tpp.pt", + "ppl": { + "wikitext2_val": { + "corpus_id": "wikitext2_raw_v1_val", + "target": 37.010055463333096, + "n_tokens": 204600, + "bpb": 1.2256204566076285, +``` + +**Confidence** — results JSON + +**Caveat** — Window config at Qwen3-0.6B/experiments/2026-06-16_qwen3-faithful_eval-first/eval_suite.py:65 ('SEQ, STRIDE, MAX_WINDOWS = 1024, 512, 200'); wikitext loaded as Salesforce/wikitext config wikitext-2-raw-v1 split validation with a pinned revision (eval_suite.py:165-166), codeparrot/codeparrot-clean-valid split train pinned to 4db92d2ec0c1b4c41eeb439cfae16854511d9dcd (eval_suite.py:61,174-175). These windows OVERLAP (stride 0.12591 @168M -> 0.07169 @420M, trend verdict 'CONVERGES', rationale 'gap shrinks toward 0 with scale and falls within the noise floor at the largest budget — an early-training speedup that converges away (no advantage at scale)'. n_seeds [3,3] at every rung. research/ledger/ledger.json records run_id 2026-07-05_qwen3-0.6b_scaling-persistence with status 'done', verdict 'null'. Note the code_py corpus plateaus rather than converges (0.50156 -> 0.17578 -> 0.17709), so 'converges' is corpus-dependent — do not overstate in either direction. Meanwhile Qwen3-0.6B/README.md:52 still advertises 'NorMuon > AdamW | wikitext -0.474 bpb | significant win', which is the 42M rung only. A reviewer would consider the omission of this later null material. +``` + + +### 3.G Gaps — not determinable from disk + +- Environment versions (torch / transformers / datasets) at the time of the 2026-06-09 13.40 measurement are not recorded in any file I could find. The only hint is a deprecation banner in results/original_eval_run2.log:1 ('`torch_dtype` is deprecated! Use `dtype` instead!'), which gives no version. +- The HuggingFace revision (commit sha) of Qwen/Qwen3-0.6B-Base actually downloaded for the 13.40 run is NOT recorded — eval_original_vs_repro.py:49 calls from_pretrained(REPO) with no revision argument, and no lockfile/manifest exists in the build folder. +- The HuggingFaceFW/fineweb-edu dataset revision behind tokcache_133072000_300000.pt and tokcache_1191478400_300000.pt is NOT recorded — train_qwen3.py:151 passes no revision. (A pinned sha 87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 appears in research/eval/private_heldout_v1/private_prose_v1.txt:455 but that is the LATER dataset-forge prep, not these caches.) +- No results .json exists for any of the four numbers. 13.40 and 46.31 live only in results/original_vs_repro.txt (+ .log); 28.65 in qwen3_baseline2tpp_after.txt / _train.log; 23.52 only in qwen3_imu1_2tpp_train.log. There is no suite_version stamp on any of them. +- Whether the released Qwen3-0.6B-Base saw these exact FineWeb-Edu val documents during its 36T-token pretraining is undeterminable from disk; the training_plan.md summary of the tech report describes a web-heavy corpus but no overlap test was or could be run here. +- The exact CLI arguments used to launch the IMU-1 2-TPP run are not on disk — qwen3_imu1_2tpp_train.log echoes only the bundle flags, param split and tok/step, not the full argparse namespace (unlike the faithful runs). The LR / z-weight / weight-decay values I report are train_imu1.py:96-107 DEFAULTS, unverified against the actual invocation. +- No re-measurement of the released Qwen3-0.6B-Base exists on the 1191478400 val slice, so the true same-slice gap between 28.65/23.52 and the released model is unknown; it can only be obtained by re-running eval_original_vs_repro.py against that cache. + +--- + +## 4. −0.474 bpb — NorMuon vs AdamW + +Audit dimension: the -0.474 bpb figure (NorMuon vs AdamW) + +### 4.1 Which wikitext is the -0.474 bpb measured on: wikitext-2-raw-v1 or wikitext-103-raw-v1? + +**Value** + +``` +wikitext-2-raw-v1 (HF dataset `Salesforce/wikitext`, config `wikitext-2-raw-v1`), pinned revision b08601e04326c79dfdd32d625aee71d232d685c3. NOT wikitext-103. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/score_cohort.py:54` + +**Source quote** + +``` +wt = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation", + revision=WIKITEXT_REV) [WIKITEXT_REV = "b08601e04326c79dfdd32d625aee71d232d685c3", score_cohort.py:25] +``` + +**Confidence** — measured from code + +**Caveat** — Same corpus definition is the versioned suite standard: .claude/skills/eval-harness/references/suite.md:133 `| wikitext2_val | Salesforce/wikitext | wikitext-2-raw-v1 / validation | b08601e04326c79dfdd32d625aee71d232d685c3 |`. Corpus text is assembled as "\n\n".join of non-empty ex["text"] (score_cohort.py:56), i.e. one concatenated stream, not per-document. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Opened Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/score_cohort.py. Line 54 reads exactly: `wt = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation",` and line 55 `revision=WIKITEXT_REV)`. WIKITEXT_REV is defined at score_cohort.py:25 as `b08601e04326c79dfdd32d625aee71d232d685c3`. The '-raw-' variant IS what the script names — no substitution. Line 56 confirms the text-assembly claim: `wt_text = "\n\n".join(e["text"] for e in wt if e["text"].strip())`. Cross-check .claude/skills/eval-harness/references/suite.md:133 matches the quote verbatim, including the same pinned sha. Independent third corroboration at RESULT.md:72 ('`Salesforce/wikitext` wikitext-2-raw-v1 rev `b08601e04326...`'). No occurrence of wikitext-103 anywhere in the scorer. +``` + + +### 4.2 Which split? + +**Value** + +``` +validation (`split="validation"`). The scored token stream is 204,600 tokens / 869,710 bytes. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/score_cohort.py:54` + +**Source quote** + +``` +wt = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation", +``` + +**Confidence** — measured from code + +**Caveat** — n_tokens/n_bytes per cell confirmed identical across all 6 cells in Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/cohort_bpb.json:8-9 ("n_tokens": 204600, "n_bytes": 869710). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +score_cohort.py:54 literally contains `split="validation"`. Verified n_tokens/n_bytes are identical across all six cells by reading the whole file, not just the cited lines: cohort_bpb.json lines 8-9 (adamw_seed0), 22-23, 36-37, 50-51, 64-65, 78-79 all read `"n_tokens": 204600, "n_bytes": 869710`. MATERIAL CLARIFICATION a reviewer will want: 204,600 is NOT the size of the wikitext-2 validation split. It is 200 windows x 1023 label tokens under the MAX_WINDOWS=200 cap (score_cohort.py:24, 37-38), and because STRIDE(512) < SEQ(1024) those 204,600 token-scorings come from only 102,911 DISTINCT label positions (windows start at b = 0,512,...,101888; union of label spans = [1, 102912)). So the eval scores the first ~103k tokens of the split, each counted ~twice — it does not score the whole split. +``` + + +### 4.3 Is it bits-per-byte (bpb) or bits-per-token? How is bpb computed (which byte count / normalization)? + +**Value** + +``` +True bits-per-byte. bpb = (sum of per-token NLL in nats over the eval windows / ln2) / (UTF-8 byte count of the decoded LABEL span of those same windows). Denominator for wikitext-2 = 869,710 bytes; for code = 843,643 bytes. Per-token PPL is reported separately, never as the headline. +``` + +**Evidence** — `research/eval_metrics.py:33` + +**Source quote** + +``` +def bits_per_byte(total_nll_nats, total_bytes): + """Bits-per-byte = (Σ NLL in nats / ln2) / (raw UTF-8 byte count). +... + return (float(total_nll_nats) / _LN2) / total_bytes +``` + +**Confidence** — measured from code + +**Caveat** — The byte count is accumulated per window as `nbytes += len(tok.decode(labels[0].tolist()).encode("utf-8"))` (score_cohort.py:46) and NLL as `F.cross_entropy(..., reduction="sum")` (score_cohort.py:43-44). Because STRIDE(512) < SEQ(1024) the windows overlap and roughly half the corpus is counted twice — but numerator and denominator are double-counted identically, which eval_metrics.py:39-41 explicitly requires ("under overlapping eval windows, the byte denominator must match the double-counted token span"). Logits are cast .float() before CE (score_cohort.py:41) though the model runs bf16 (score_cohort.py:30). + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Value body is correct. The CAVEAT contains a numeric error: 'roughly half the corpus is counted twice' is wrong. Correct statement: essentially ALL of the scored span is counted twice — 204,600 scored token-positions come from 102,911 distinct positions (ratio 1.988), i.e. ~98.8% of the span is counted exactly twice, only ~1,222 positions once. Additionally, MAX_WINDOWS=200 caps the scan, so the metric covers only the first 102,911 label positions of the tokenized corpus, not the whole validation split. +``` + +**Verifier note** + +``` +The formula IS confirmed: research/eval_metrics.py:33-45 defines `def bits_per_byte(total_nll_nats, total_bytes)` returning `(float(total_nll_nats) / _LN2) / total_bytes`, with `_LN2 = math.log(2.0)` at line 28. Docstring lines 39-41 do say the byte denominator 'must match the double-counted token span'. score_cohort.py:46 `nbytes += len(tok.decode(labels[0].tolist()).encode("utf-8"))` and :43-44 `F.cross_entropy(..., reduction="sum")` confirmed. `.float()` at :41 and `DTYPE = torch.bfloat16` at :30 confirmed. Byte denominators 869,710 (wikitext) / 843,643 (code) confirmed in cohort_bpb.json. eval_metrics.py:48-50 confirms PPL is per-token and 'NOT comparable across tokenizers — report alongside bits_per_byte, never as the sole cross-run headline'. Only the double-counting arithmetic in the caveat is wrong, and it is wrong in a way that understates how truncated/overlapped the eval window set is. +``` + + +### 4.4 Sequence length, stride, tokenizer used for the eval + +**Value** + +``` +Eval window SEQ = 1024, STRIDE = 512, MAX_WINDOWS = 200 (so 200 windows × 1023 label tokens = 204,600 tokens scored). Tokenizer = HF `Qwen/Qwen3-0.6B-Base` AutoTokenizer (the model's own tokenizer, vocab 151,936). Note this is the EVAL seq len; TRAINING used seq_len 4096. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/score_cohort.py:24` + +**Source quote** + +``` +SEQ, STRIDE, MAX_WINDOWS = 1024, 512, 200 # text-lm-v2 constants +... + tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base") [score_cohort.py:72] +... +SEQ_LEN, MICRO_BATCH, GRAD_ACCUM = 4096, 4, 4 [train_ablation.py:51] +``` + +**Confidence** — measured from code + +**Caveat** — suite.md:115-116 pins the same constants for text-lm-v2 (`window SEQ | 1024`, `STRIDE | 512`) and says "never lift SEQ without a version bump". The tokenizer is loaded from the HF hub at score time (network dependency), not from a local pinned copy. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +score_cohort.py:24 reads exactly `SEQ, STRIDE, MAX_WINDOWS = 1024, 512, 200 # text-lm-v2 constants`. score_cohort.py:72 reads `tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base")`. train_ablation.py:51 reads `SEQ_LEN, MICRO_BATCH, GRAD_ACCUM = 4096, 4, 4`. Vocab 151,936 verified independently at Qwen3-0.6B/model.py:37 `vocab_size: int = 151_936 # config.json: vocab_size`. suite.md:115-117 pins `window SEQ | 1024`, `STRIDE | 512`, `MAX_WINDOWS (main metric) | 200`; suite.md:123 contains 'never lift `SEQ` without a version bump'. 200 x 1023 = 204,600 matches cohort_bpb.json's n_tokens exactly. The network-dependency caveat (tokenizer pulled from the hub at score time, no local pin) is correct as read. +``` + + +### 4.5 Which two arms, at what model size, what token budget, how many seeds? + +**Value** + +``` +Arms: AdamW @ peak_lr 2.4e-3 vs NorMuon @ lr 0.011, applied ONLY to the 196 2D non-embedding weight matrices; the 114 embedding/1D params are AdamW@2.4e-3 wd=0 in BOTH arms; 2D weight_decay=0.1 in both. Model: full faithful Qwen3-0.6B, 596,049,920 total params / 440,467,456 non-embedding. Budget: 640 steps × 65,536 tok/step = 41,943,040 tokens per cell ("42M"), iso-FLOP. Seeds: 3 per arm (6 cells total), seed varies init + DataLoader shuffle only. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/train_ablation.py:82` + +**Source quote** + +``` +ap.add_argument("--optimizer", choices=["adamw", "normuon"], required=True) + ap.add_argument("--seed", type=int, required=True) + ap.add_argument("--steps", type=int, default=600) # ~39M tokens + ap.add_argument("--peak_lr", type=float, default=2.4e-3) # faithful-tuned AdamW LR + ap.add_argument("--normuon_lr", type=float, default=0.011) # IMU-1-tuned NorMuon 2D LR + ap.add_argument("--weight_decay", type=float, default=0.1) # on 2D, BOTH arms (held equal) +[train_ablation.py:53] TOK_PER_STEP = SEQ_LEN * MICRO_BATCH * GRAD_ACCUM # 65,536 +``` + +**Confidence** — measured from code + +**Caveat** — Budget arithmetic + arm split also stated in RESULT.md:68-70 and recorded in research/ledger/ledger.json:466-469 ("tokens_per_cell": 41943040, "cells": 6, "source": "6-cell cohort (2 arms x 3 seeds), 640 steps x 65536 tok"). n_params_nonembed 440467456 is from ledger.json metrics.systems (line ~489 block); total 596,049,920 from Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/architecture_plan.md:60. IMPORTANT confound disclosed by the repo itself (RESULT.md:47): all 6 cells share a FIXED data split (SPLIT_SEED=0), so the ±0.006–0.008 SEM under-estimates true end-to-end seed variance. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +All stated values confirmed, but a MATERIAL caveat is missing and one line pointer is wrong. MISSING CAVEAT (RESULT.md:43, Limitation #2): 'Both arms use 2D wd=0.1 (NorMuon's/IMU-1's tuned value). AdamW's faithful recipe was tuned at wd=0.01. So AdamW runs at a 10x wd it was never tuned for, on a budget it was never tuned for — part of the gap may be baseline handicap rather than genuine optimizer advantage.' Stating 'weight_decay=0.1 in both' without this reads as if the control were neutral. LINE FIX: n_params_nonembed 440467456 is at research/ledger/ledger.json:508, not '~489'. +``` + +**Verifier note** + +``` +Verified: 196/114 split is MEASURED, printed by the trainer itself — results/adamw_seed0.log:11 `param split: 196 2D->adamw | 114 rest->AdamW` and results/normuon_seed0.log:11 `param split: 196 2D->normuon | 114 rest->AdamW`. 640 steps is MEASURED not defaulted — adamw_seed0.log:8 `start: optimizer=adamw seed=0 steps=640 tok_budget=41,943,040 peak_lr=0.0024 normuon_lr=0.011 wd=0.1` (the script DEFAULT at train_ablation.py:84 is 600; run_arms.sh passes $STEPS). train_ablation.py:82-87 quoted correctly. train_ablation.py:69 confirms the split rule `(twod if (p.dim() == 2 and "embed_tokens" not in name) else rest)`; :70-77 confirm rest->AdamW wd=0.0 in both arms. SPLIT_SEED=0 at :52. 596,049,920 is a real runtime-printed count for this Qwen3Config (e.g. Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/results/sft_seed0.log:7 `params=596,049,920`) and matches architecture_plan.md:60. Budget arithmetic corroborated at RESULT.md:68 and ledger.json:466-469. Iso-FLOP is asserted-by-construction, not measured: verifier_report.json:31 states 'a numeric metrics.train_flops field is not stored in any results JSON; iso-FLOP holds by construction + the recorded confound_check flag.' +``` + + +### 4.6 What is the sign convention — is -0.474 NorMuon better? + +**Value** + +``` +Yes. The on-disk JSON stores a POSITIVE +0.47432550192416323 as `improvement_bpb` = adamw_mean(2.1098) − normuon_mean(1.6355), i.e. NorMuon's bpb is 0.474 LOWER (better; lower bpb is better). The "−0.474" form used in Qwen3-0.6B/README.md is the same number expressed as NorMuon's signed delta relative to AdamW. Both mean NorMuon better by 0.474 bpb. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/score_ladder.py:85` + +**Source quote** + +``` +# claim. Sign convention: gap_bpb = adamw_mean - normuon_mean (eval_stats.seed_delta_significant, +# direction="lower_is_better"), so gap > 0 == NorMuon better. +``` + +**Confidence** — measured from code + +**Caveat** — Cross-checked against research/eval_stats.py:138 `improvement = (b_mean - t_mean) if direction == "lower_is_better" else ...` and against Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/verdict.json:19 `"improvement_bpb": 0.47432550192416323`. The literal string "-0.474" appears on disk only at Qwen3-0.6B/README.md:52 and :243. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +score_ladder.py:85-86 reads exactly: '# claim. Sign convention: gap_bpb = adamw_mean - normuon_mean (eval_stats.seed_delta_significant,' / '# direction="lower_is_better"), so gap > 0 == NorMuon better.' Cross-checked research/eval_stats.py:138 `improvement = (b_mean - t_mean) if direction == "lower_is_better" else (t_mean - b_mean)`, and :115 'the improvement delta (positive == better in the given direction)'. results/verdict.json:19 `"improvement_bpb": 0.47432550192416323`; means at :10 (2.1098171365956357) and :17 (1.6354916346714725) — 2.1098171 - 1.6354916 = 0.4743255, arithmetic checks. The sub-claim about the literal minus form holds: a repo-wide grep for `[-−–]0.474` returns hits ONLY at Qwen3-0.6B/README.md:52 and :243. +``` + + +### 4.7 What exactly is the 42M-token headline number and its CI? + +**Value** + +``` +wikitext-2 val BPB: AdamW mean 2.1098171365956357 (seeds 2.104968, 2.102416, 2.122067), NorMuon mean 1.6354916346714725 (seeds 1.649904, 1.624842, 1.631729); improvement +0.47432550192416323 bpb, 95% CI [0.4434844613250229, 0.5051665425233036], Welch-t df 3.861, significant=true, n=[3,3], suite_version text-lm-v2, verdict "win". +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/verdict.json:19` + +**Source quote** + +``` +"improvement_bpb": 0.47432550192416323, + "ci95": [ + 0.4434844613250229, + 0.5051665425233036 + ], + "significant": true, + "df": 3.8610499345338067, + "n": [3, 3] +... + "headline_corpus": "wikitext2_val", + "verdict": "win" +``` + +**Confidence** — results JSON + +**Caveat** — This is a number this repo MEASURED itself (raw per-seed BPB in results/cohort_bpb.json, produced by score_cohort.py from the 6 on-disk checkpoints). Independently re-derived in results/verifier_report.json:16,36 ("reproduces the recorded wikitext-2 result bit-for-bit"). Not copied from any paper. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Every digit verified against Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/verdict.json: suite_version :2, adamw_bpb :5-9, adamw_mean :10, normuon_bpb :12-16, normuon_mean :17, improvement_bpb :19, ci95 :20-23, significant :24, df :26 (3.8610499345338067), n :27-30, headline_corpus :61, verdict :62. Per-seed values independently traced to results/cohort_bpb.json:7, 21, 35 (adamw) and :49, 63, 77 (normuon). Re-derivation confirmed at results/verifier_report.json:16, 21, 26 (ci_match true) and :36 ('reproduces the recorded wikitext-2 result bit-for-bit'). This is a repo-MEASURED number (produced by score_cohort.py from six on-disk checkpoints), not copied from a paper. THREE small notes for a card author: (1) the quoted JSON snippet silently elides `"warning": null` (verdict.json:25) between `significant` and `df`; (2) the df 3.861 is the Welch-Satterthwaite df, but the CI was computed with df FLOORED to 3 / t_crit 3.182 (verifier_report.json:19-20) — conservative, but 'df 3.861' alone misdescribes the interval; (3) the test is UNPAIRED Welch on a design that is paired-by-seed — the ladder's own verdict.json:257 flags this ('a paired-t on per-seed diffs is the stricter test'). +``` + + +### 4.8 CURRENT status of the claim — did a later run null it? + +**Value** + +``` +YES, NULLED AT SCALE. The scaling-persistence ladder (fixed N=596M, token budget swept 42M/168M/420M, n=3 seeds per arm at EVERY rung) records ledger_verdict "null" and trend CONVERGES on BOTH corpora. wikitext-2 gap: 0.47432550 (42M) -> 0.12590584 (168M) -> 0.07169398 (420M). The ledger run 2026-07-05_qwen3-0.6b_scaling-persistence carries "verdict": "null", status "done". +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json:194` + +**Source quote** + +``` +"rationale": "gap shrinks toward 0 with scale and falls within the noise floor at the largest budget — an early-training speedup that converges away (no advantage at scale)", +[line 199] "ledger_verdict": "null", +[line 6] "question": "Does NorMuon's +0.474 BPB win over AdamW (2D weights, fixed N=596M) persist or converge with token budget?" +[lines 27/44/61] "gap_bpb": 0.47432550192416323, ... "gap_bpb": 0.12590584068581911, ... "gap_bpb": 0.07169397744785555, +[research/ledger/ledger.json:1554,1566] "run_id": "2026-07-05_qwen3-0.6b_scaling-persistence", ... "verdict": "null", +``` + +**Confidence** — results JSON + +**Caveat** — THREE nuances a reviewer must not lose. (1) The null is a BUDGET-scaling null at FIXED model size N=596M — it is not evidence about larger N. (2) The 420M wikitext gap is still nominally SIGNIFICANT as measured (+0.0717, CI [0.0553, 0.0881], excludes 0, verdict.json:61-71); the "falls within the noise floor" phrase refers to the OLS-FITTED gap at the top rung (edge_at_top / gap_hi_fit = 0.029726712435672376 < gap_noise 0.03675972213287565, verdict.json:141-146), not to the measured rung. edge_resolved=false on wikitext, TRUE on code. (3) The `null` word is partly gate-driven: verdict.json:207-214 records the §C25 `scaling` HARD battery as INCOMPLETE (missing log_rmse_r2, holdout_extrapolation_pctdev, bootstrap_forecast_ci) so `win` was unreachable regardless — but verdict.json:203 shows significance_verdict was independently "null" from the CONVERGES trend mapping, with c17_cap_applied=false. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Read the full 258-line Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json. Confirmed: question :6; wikitext gaps :27 (0.47432550192416323), :44 (0.12590584068581911), :61 (0.07169397744785555); n_seeds [3,3] at all three rungs (:35-38, :52-55, :69-72); trend_verdict CONVERGES :198; ledger_verdict 'null' :199; rationale :194 (and the identical text at :149). ledger.json:1554 run_id, :1565 status 'done', :1566 verdict 'null'. All three of the fact's nuances verified: (1) budget-only sweep at fixed N=596M — BUDGETS at score_ladder.py:42 sweep tokens only; (2) 420M wikitext IS nominally significant as measured — verdict.json:61-66 gap 0.0717, ci95 [0.05525251860105098, 0.08813543629466011], `"significant": true`; the 'noise floor' phrase refers to the FITTED gap_hi_fit/edge_at_top 0.029726712435672376 (:140, :142) vs gap_noise 0.03675972213287565 (:146); edge_resolved false on wikitext (:143), true on code (:165); (3) §C25 incompleteness at :207-214 AND significance_verdict independently 'null' at :203 with c17_cap_applied false at :206. ADDITIONAL PROVENANCE CAVEAT I found: this verdict.json is NOT tracked by git (`git ls-files --error-unmatch` errors on it), nor is the ladder's per-seed ladder_bpb.json — the null's evidence is working-tree-only, while the 42M 'win' evidence (cohort_bpb.json, verdict.json) IS committed. +``` + + +### 4.9 What are the companion numbers on the second (code) corpus? + +**Value** + +``` +Corpus = codeparrot/codeparrot-clean-valid, split `train` (streaming), pinned rev 4db92d2ec0c1b4c41eeb439cfae16854511d9dcd, first 500,000 chars, 204,600 tokens / 843,643 bytes, same SEQ 1024 / STRIDE 512. 42M: AdamW 3.3846985755955523 vs NorMuon 2.8831399238053073, gap +0.5015586517902451, CI [0.4559911731303807, 0.5471261304501094], significant. 168M: gap +0.1757807425441804, CI [0.1369433296166722, 0.21461815547168864]. 420M: gap +0.17708989020863175, CI [0.13074040899389106, 0.22343937142337245]. Trend verdict CONVERGES (slope -0.34197431351062624 over log10 tokens, r2 0.8412726939487719) but edge_resolved=true (still above noise at the top rung). +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json:80` + +**Source quote** + +``` +"gap_bpb": 0.5015586517902451, [42M] + "gap_bpb": 0.1757807425441804, [168M, line 97] + "gap_bpb": 0.17708989020863175, [420M, line 114] +[line 171] "rationale": "gap shrinks toward 0 with scale; still above noise at the largest measured budget but trending out — the edge is eroding, extend the ladder before claiming it" +[score_cohort.py:57-58] cp = load_dataset("codeparrot/codeparrot-clean-valid", split="train", + streaming=True, revision=CODEPARROT_REV) +``` + +**Confidence** — results JSON + +**Caveat** — The code corpus does NOT converge monotonically: 168M 0.17578 -> 420M 0.17709 is a slight INCREASE, i.e. a plateau over the last two rungs, and only the fitted slope is negative. MEMORY's "plateau, not convergence — scrutinize that label" is correct and traceable: the CONVERGES word for code_py comes from an OLS slope across 3 points dominated by the 42M rung, not from the last two rungs. Also note the code corpus is a raw 500k-char prefix of a streamed split, so it is a fixed but arbitrary slice. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +score_cohort.py:57-58 reads exactly `cp = load_dataset("codeparrot/codeparrot-clean-valid", split="train",` / `streaming=True, revision=CODEPARROT_REV)`; CODEPARROT_REV = '4db92d2ec0c1b4c41eeb439cfae16854511d9dcd' at :26; CODE_CHARS = 500_000 at :27. Ladder verdict.json code block verified line-by-line: 42M gap :80, ci95 :81-84; 168M gap :97, ci95 :98-101; 420M gap :114, ci95 :115-118; all `"significant": true` (:85, :102, :119). trend_by_corpus.code_py: verdict CONVERGES :154, slope :156, r2 :158, edge_resolved true :165, gap_noise 0.046349481214740695 :168, rationale :171. 843,643 bytes / 204,600 tokens confirmed in cohort_bpb.json:14-15. The MEMORY-flagged 'plateau not convergence' point is CORRECT and I reproduced it: 0.1757807 (168M) -> 0.1770899 (420M) is an INCREASE of +0.0013, and the two CIs overlap almost entirely — only the 3-point OLS slope, dominated by the 42M rung, is negative. MINOR IMPRECISION: 'first 500,000 chars' overstates exactness — score_cohort.py:59-63 appends WHOLE documents until cumulative `len(content)+2` EXCEEDS 500,000, so the slice is >=500,000 chars; suite.md:134 phrases it correctly as 'until > 500,000 chars'. +``` + + +### 4.10 Does the model card / README currently present -0.474 as a standing win despite the null? + +**Value** + +``` +YES — the de-facto model card does. Qwen3-0.6B/README.md:52 and :243 both present it as a "significant win" with NO mention of the scaling ladder, CONVERGES, or the null verdict. grep of Qwen3-0.6B/README.md for scaling-persistence / converge / CONVERGES / 0.126 / 0.072 / persist returns ZERO hits. The ledger entry for the 42M run also still reads verdict "win" (research/ledger/ledger.json:482), and technique `normuon-optimizer`'s run_ids list does not include the ladder run (which carries technique_slug: null, ledger.json:1557). +``` + +**Evidence** — `Qwen3-0.6B/README.md:52` + +**Source quote** + +``` +| **NorMuon > AdamW** | wikitext −0.474 bpb [0.444, 0.505] · code −0.502 [0.456, 0.547] | **significant win** | +[Qwen3-0.6B/README.md:243] | **Optimizer ablation (clean, single-variable)** | NorMuon **beats** AdamW: wikitext **−0.474 bpb** (95% CI [0.444, 0.505]), code −0.502 bpb ([0.456, 0.547]) — **significant win** | ledger `2026-06-16_qwen3_normuon-vs-adamw` | +``` + +**Confidence** — measured from code + +**Caveat** — To be fair to the source run: RESULT.md:7 DOES scope it correctly ("This is an early-training optimization-speed signal at one architecture and one budget; we do NOT claim it holds at scale") and RESULT.md:45 predicts the fade. The failure is that Qwen3-0.6B/README.md dropped those qualifiers. Any model card must carry the ladder null. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +The YES answer and every cited line are correct, but the exposure is UNDERCOUNTED. There are at least FOUR un-caveated standing-win presentations, not two: Qwen3-0.6B/README.md:52, :175 ('NorMuon beats AdamW by **+0.474 bpb on wikitext-2 (95% CI [0.444, 0.505])** and +0.502 on code — significant.'), :243, and Qwen3-0.6B/PLOTS_INDEX.md:73 ('+0.474 bpb, significant'). Also add: the 42M run's own ledger caveats field, research/ledger/ledger.json:512, still asserts 'no scaling curve' — a statement that became FALSE when the ladder completed. +``` + +**Verifier note** + +``` +Verified by running the exact greps. `grep -in 'scaling-persistence|converge|persist|0\.126|0\.072' Qwen3-0.6B/README.md` returns ZERO hits for every term — confirmed. README.md:52 and :243 quoted verbatim and match. ledger.json:482 `"verdict": "win"` confirmed; I read the entire 42M entry (ledger.json:461-535) and found NO pointer to the ladder or the null. normuon-optimizer run_ids at ledger.json:153-154 are exactly ['2026-06-16_qwen3_normuon-vs-adamw', '2026-07-23_qwen3-0.6b_normuon-at-scale'] — the ladder run is absent; ledger.json:1557 `"technique_slug": null` confirmed. Staleness corroborated by mtime: Qwen3-0.6B/README.md was last modified 2026-07-06, i.e. BEFORE the ladder's first completion (2026-07-12) and long before the 2026-07-28 re-score. SEPARATE ROUNDING DEFECT worth flagging to a card author: README.md:52/:175/:243 all print the CI lower bound as 0.444, but the on-disk value is 0.4434844613250229, which rounds to 0.443 at 3dp (root README.md:112 gets it right). +``` + + +### 4.11 Is the root README's account of the ladder current? + +**Value** + +``` +NO — root README.md:105-121 is STALE relative to verdict.json (which was re-scored 2026-07-28 after the 3rd 420M seed landed). README says "420M ×2 seeds", "+0.073 [−0.038, +0.184] at 420M ... not significant at the top", "code_py: +0.502 → +0.176 → +0.192", code slope "−0.328 (r² 0.81)", and "Verdict: directional, not a headline — the 420M rung is n=2". Current verdict.json: n=3/arm at 420M, wikitext gap +0.0717 CI [0.0553, 0.0881] SIGNIFICANT, code 420M +0.1771, code slope −0.34197 r² 0.84127, headline_capped_by_c17_power false, ledger_verdict "null". +``` + +**Evidence** — `README.md:112` + +**Source quote** + +``` +(AdamW − NorMuon, BPB): **+0.474** [+0.443, +0.505] at 42M → **+0.126** + [+0.089, +0.163] at 168M → **+0.073** [−0.038, +0.184] at 420M — significant at + the two smaller budgets, **not significant** at the top. code_py: +0.502 → +0.176 + → +0.192. OLS over log10(tokens) gives slope **−0.416** (r² 0.92) on wikitext and + **−0.328** (r² 0.81) on code → **CONVERGES** on both. +[README.md:117] - **Verdict: directional, not a headline** — the 420M rung is n=2 (< 3 seeds, §C17). +``` + +**Confidence** — measured from code + +**Caveat** — The 3rd 420M seed pair was trained by run 2026-07-23_qwen3-0.6b_normuon-at-scale (checkpoints persist_420M_{adamw,normuon}_s2.pt dated 2026-07-25/26) and re-scored 2026-07-28. research/ledger/runs/2026-07-23_qwen3-0.6b_normuon-at-scale.md:355-360 documents the n=2 -> n=3 upgrade and the current numbers. The direction of the staleness matters: the top-rung gap became MORE statistically resolved (significant), while the overall verdict stayed `null` via the CONVERGES trend + §C25 gate. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Read /home/yashb98/Downloads/BuildFromScratch/README.md:105-125. Every stale value quoted is present verbatim: ':109 + **420M ×2 seeds**, ten cells'; ':112-116' the full '+0.474 [+0.443, +0.505] at 42M → +0.126 [+0.089, +0.163] at 168M → +0.073 [−0.038, +0.184] at 420M — significant at the two smaller budgets, **not significant** at the top. code_py: +0.502 → +0.176 → +0.192. OLS ... slope −0.416 (r² 0.92) on wikitext and −0.328 (r² 0.81) on code'; ':117 - **Verdict: directional, not a headline** — the 420M rung is n=2 (< 3 seeds, §C17).' Current values all confirmed in verdict.json (:13-16 top_budget_seeds [3,3]; :17 headline_capped_by_c17_power false; :19 cap_reason 'top rung has >=3 seeds'; :61-66; :114; :156; :158; :199). The upgrade provenance is confirmed at research/ledger/runs/2026-07-23_qwen3-0.6b_normuon-at-scale.md:356-360, which states the rung moved from n_seeds [2,2] with warning 'CI is wide/unreliable' to [3,3] warning null, and quotes the current wikitext gap 0.0717 CI95 [0.0553, 0.0881] and code 0.1771 [0.1307, 0.2234]. Independently corroborated by mtimes: root README.md 2026-07-23 18:05, verdict.json 2026-07-28 19:17. The fact's own framing of the staleness direction (top rung became MORE resolved, verdict stayed null) is accurate. +``` + + +### 4.12 Was the ladder scored with the SAME eval pipeline as the 42M headline (i.e. are the numbers comparable)? + +**Value** + +``` +Yes. score_ladder.py imports the IMU-1 scorer directly and reuses its score()/load_corpora() — identical SEQ/STRIDE/corpora/bpb — and the 42M rung is not re-run but copied verbatim from cohort_bpb.json. Same suite_version text-lm-v2 stamped on both. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/score_ladder.py:40` + +**Source quote** + +``` +import score_cohort as sc # noqa: E402 (reuse score() + load_corpora(): text-lm-v2 SEQ/STRIDE/bpb) +[line 454] corpora = sc.load_corpora(tok) +[line 238] return {c: sc.score(model, ids, tok) for c, ids in corpora.items()} +[line 505] r = seed_delta_significant(a, n, direction="lower_is_better") +``` + +**Confidence** — measured from code + +**Caveat** — Per-seed ladder BPBs are durably recorded in Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/ladder_bpb.json (18 cells, run_id stamped as the LADDER's, scored 2026-07-28) — note the file lives in the 42M experiment's results/ dir by a documented location decision (score_ladder.py:53-60), which is easy to misread as belonging to the 42M run. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Pipeline-identity claim is fully CONFIRMED. The caveat's word 'durably recorded' needs qualifying: Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/ladder_bpb.json is NOT tracked by git (`git ls-files --error-unmatch` errors on it), whereas the 42M rung's cohort_bpb.json and verdict.json ARE tracked. Given this repo's own recorded incident of a branch switch destroying untracked evidence, the null's per-seed evidence exists on the working tree only. Minor: the location-decision comment spans score_ladder.py:53-62, not 53-60. +``` + +**Verifier note** + +``` +Verified every cited line. score_ladder.py:40 reads exactly `import score_cohort as sc # noqa: E402 (reuse score() + load_corpora(): text-lm-v2 SEQ/STRIDE/bpb)`; :454 `corpora = sc.load_corpora(tok)`; :238 `return {c: sc.score(model, ids, tok) for c, ids in corpora.items()}`; :505 `r = seed_delta_significant(a, n, direction="lower_is_better")`. :455 confirms the same model class/dtype (`sc.DEVICE, sc.DTYPE`). :456 `reuse = json.loads((RES / "cohort_bpb.json").read_text())["cells"] # 42M rung` confirms the 42M rung is copied, not re-run. I loaded ladder_bpb.json directly: 18 cells, `run_id` = '2026-07-05_qwen3-0.6b_scaling-persistence', `scored` = '2026-07-28', and every 42M cell carries `"source": "reused:cohort_bpb.json"` while the 420M s2 cells carry `source: scored`. suite_version 'text-lm-v2' stamped in both ladder_bpb.json and cohort_bpb.json:2. verdict.json:21 `per_seed_bpb_file` points back at the file, so the cross-reference the caveat describes is real. Also note score_ladder.py itself is uncommitted-modified (git status ` M`), though its mtime 19:09 precedes verdict.json's 19:17, so verdict.json was produced by the current scorer. +``` + + +### 4.G Gaps — not determinable from disk + +- No 840M rung exists. c5_evidence_scale_ext.json declared a 840M (n=1) point but it was descoped before launch; SEEDS in score_ladder.py:49 still declares 840_000_000, and no checkpoint_persist_840M_*.pt is on disk. So the trend fit rests on exactly 3 budgets (42M/168M/420M), the minimum for a non-descriptive fit. +- No per-horizon LR re-tuning exists anywhere on disk. Both AdamW 2.4e-3 and NorMuon 0.011 were tuned at the 42M horizon and held fixed at 168M/420M (verdict.json:257 'Inherited confound: AdamW/NorMuon LRs tuned at 42M, not re-tuned per horizon'). Part of the observed fade could therefore be an LR artifact; nothing on disk separates the two. +- The three §C25 HARD scaling-battery items (log_rmse_r2, holdout_extrapolation_pctdev, bootstrap_forecast_ci) were never computed — no file on disk contains them (verdict.json:210-214, 236-241). A §C26 figure for the ladder is also missing ('c25_report_missing': ['figure']). +- No ledger detail doc exists for the ladder itself: research/ledger/runs/ contains 2026-06-16_qwen3_normuon-vs-adamw.md and 2026-07-23_qwen3-0.6b_normuon-at-scale.md, but NO 2026-07-05_qwen3-0.6b_scaling-persistence.md. The ladder's narrative lives only in verdict.json and inside the 2026-07-23 child doc. +- The 42M run's ledger entry has never been amended to reference the ladder: research/ledger/ledger.json:482 still reads "verdict": "win" with no pointer to the null, and the technique entry 'normuon-optimizer' run_ids = [2026-06-16_qwen3_normuon-vs-adamw, 2026-07-23_qwen3-0.6b_normuon-at-scale] omits the ladder run (whose technique_slug is null, ledger.json:1557). A ledger query by technique will not surface the null directly. +- Nothing on disk measures whether the convergence holds at model sizes other than N=596M — the ladder sweeps token budget only, at one fixed N. Any 'NorMuon does not help' generalization beyond 596M / 420M tokens is unsupported by this repo. +- research/ledger/ledger.json is currently uncommitted-modified (git status), so the ledger values quoted here are the working-tree state, not a committed state. + +--- + +## 5. Reproduce — commands, versions, parity, determinism + +Audit dimension: A real "Reproduce" section for a HuggingFace model card — exact commands, parity-check semantics (device/dtype/determinism/tolerance/inputs), recorded software versions, and commit/provenance stamping, for Qwen3-0.6B and SmolLM2-134(base). + +### 5.1 Qwen3-0.6B: exact command to run the architecture-parity / bit-exactness check (script form) + +**Value** + +``` +cd /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B && python3 verify.py — prints max |Δlogits| and asserts < 1e-3 + argmax agreement. Writes NO file. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:11` + +**Source quote** + +``` +python verify.py +``` + +**Confidence** — measured from code + +**Caveat** — Corroborated by Qwen3-0.6B/README.md:498 `python verify.py # parity gate — runs on CPU, no GPU needed` and root README.md:182 `cd Qwen3-0.6B && python verify.py`. This form is stdout-only — it produces no artifact on disk. There is no committed stdout log of Qwen3-0.6B/verify.py anywhere in the repo (searched for *.log under Qwen3-0.6B/); the only on-disk Qwen3 parity artifact comes from verify_run.py (next fact). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/verify.py:11 is exactly ` python verify.py`. I read the whole file (89 lines): it contains no write/open/json.dump call, so 'writes NO file' holds. Asserts confirmed at verify.py:74 (`assert max_abs < 1e-3`) and :81 (`assert hf_next == our_next`). The `cd` is load-bearing: verify.py:16 does `from model import Qwen3ForCausalLM, Qwen3Config` with no sys.path manipulation, so it only runs from Qwen3-0.6B/. Corroborations verified by line: Qwen3-0.6B/README.md:498 `python verify.py # parity gate — runs on CPU, no GPU needed`; README.md:182 `cd Qwen3-0.6B && python verify.py`. The 'no committed stdout log' claim is confirmed independently: `grep -rl "Architecture parity verified" Qwen3-0.6B/` returns only Qwen3-0.6B/verify.py, and `grep -rl "max |Δlogits|" Qwen3-0.6B/` returns only verify.py, README.md, architecture_plan.md, verify_run.py — no log artifact. +``` + + +### 5.2 Qwen3-0.6B: exact command that produces the machine-readable parity artifact (verify.json) + +**Value** + +``` +cd /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b && python3 verify_run.py — writes results/verify.json, exit 0 on pass / 1 on fail +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/verify_run.py:9,31,93` + +**Source quote** + +``` +line 9: python verify_run.py +line 31: OUT_JSON = RESULTS_DIR / "verify.json" +line 93: OUT_JSON.write_text(json.dumps(result, indent=2)) +``` + +**Confidence** — measured from code + +**Caveat** — This is the command the paper's reproducibility appendix lists first (research/papers/qwen3-imu1-matched-compute/sections/reproducibility.tex:15 `python verify_run.py`). It imports REPO + load_official_weights_into_ours from the parent Qwen3-0.6B/verify.py (verify_run.py:26), so it exercises the same code path as verify.py but additionally serializes the result. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +All three cited lines match verbatim: verify_run.py:9 ` python verify_run.py`, :31 `OUT_JSON = RESULTS_DIR / "verify.json"`, :93 `OUT_JSON.write_text(json.dumps(result, indent=2))`. Exit semantics verified: verify_run.py:99 `return 1` on fail, :102 `return 0` on pass, :106 `raise SystemExit(main())`. Import path confirmed at verify_run.py:26 `from verify import REPO, load_official_weights_into_ours`. Paper cite confirmed: research/papers/qwen3-imu1-matched-compute/sections/reproducibility.tex:15 `python verify_run.py` and it is the FIRST command in the \begin{verbatim} block (block starts :13). +``` + + +### 5.3 Qwen3-0.6B: exact command to recompute the perplexity numbers + +**Value** + +``` +cd /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b && python3 eval_original_vs_repro.py — writes results/original_vs_repro.txt +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py:22-23,75` + +**Source quote** + +``` +line 22: CACHE = HERE / "results" / "tokcache_133072000_300000.pt" +line 23: OUT = HERE / "results" / "original_vs_repro.txt" +line 75: OUT.write_text(report + "\n") +``` + +**Confidence** — measured from code + +**Caveat** — REQUIRES GPU (line 41 `device = torch.device("cuda")` — hardcoded, no CPU fallback) and requires two large gitignored artifacts that DO exist on this box but are not in git: results/tokcache_133072000_300000.pt (1,066,978,101 B, dated Jun 8) and checkpoint_qwen3_lr17/lr24/lr30.pt (3,576,719,229 B each). A fresh clone CANNOT run this command. Qwen3-0.6B/README.md:489-490 states checkpoints and token caches are gitignored and must be regenerated with the training scripts. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Cited lines match verbatim: eval_original_vs_repro.py:22 CACHE, :23 OUT, :75 `OUT.write_text(report + "\n")`. GPU-hardcoded confirmed at :41 `device = torch.device("cuda")` (no is_available() fallback anywhere in the file). File sizes independently re-measured with `ls -la`: results/tokcache_133072000_300000.pt = 1066978101 bytes, mtime 2026-06-08 21:13; checkpoint_qwen3_lr17.pt / _lr24.pt / _lr30.pt = 3576719229 bytes each. Gitignore status independently confirmed: `git check-ignore -v` reports .gitignore:20 `*.pt` for both. README cite verified: Qwen3-0.6B/README.md:489-490 'Checkpoints (`*.pt`, ~3.5 GB each) and token caches are **gitignored** — regenerate / them with the training scripts.' +``` + + +### 5.4 SmolLM2-134(base): exact commands to run the architecture-parity / bit-exactness check + +**Value** + +``` +Script form: cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" && python3 verify.py | Pytest form: cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" && pytest tests/ -v +``` + +**Evidence** — `README.md:159-162` + +**Source quote** + +``` +# Architecture parity gate (the non-negotiable test before training). +pytest tests/ -v +# or, the script form: +python verify.py +``` + +**Confidence** — measured from code + +**Caveat** — Both forms verified present: SmolLM2-134(base)/verify.py:11 docstring ` python verify.py`; SmolLM2-134(base)/tests/test_parity.py:8 ` pytest tests/ -v`. The pytest form is STRICTLY STRONGER than verify.py — it adds param-count (134,515,008), tied-embedding storage-pointer, 512-token long-context, and per-layer (all 30 blocks) parity assertions (test_parity.py:49-132). The committed parity.log is the output of the SCRIPT form only. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Both commands are correct and both exist. But the pytest form is not unconditionally stronger: SmolLM2-134(base)/tests/test_parity.py:34-41 wraps the model load in try/except and calls `pytest.skip(...)` on ImportError or on any load failure ('can't load {REPO} (no internet or HF cache miss?)'), so with no network/HF cache the suite reports SKIPPED, not FAILED — a green pytest run does not by itself prove parity. It also never prints the max|Δlogits| value, only asserts on it. +``` + +**Verifier note** + +``` +Commands verified: README.md:159-162 matches the quote line-for-line (159 '# Architecture parity gate (the non-negotiable test before training).', 160 'pytest tests/ -v', 161 '# or, the script form:', 162 'python verify.py'). SmolLM2-134(base)/verify.py:11 and tests/test_parity.py:8 confirmed. Extra-coverage claims all confirmed by reading test_parity.py: :53 `assert n == 134_515_008`, :61 data_ptr tie check, :89-101 512-token long-context (`max_length=512` at :93), :104-132 per-layer over exactly 30 blocks (:127 `assert len(hf_states) == len(our_states) == 30`). One further mismatch worth knowing: the pytest fixture loads with the MODERN kwarg (test_parity.py:39 `dtype=torch.float32`) while verify.py:53 uses the deprecated `torch_dtype=`, so the two forms are not the identical call. The claim that parity.log is the SCRIPT form's output is confirmed — parity.log:6-9 reproduces verify.py's exact print strings (:70, :71, :81, :82). +``` + + +### 5.5 SmolLM2-134(base): exact command to recompute the perplexity numbers + +**Value** + +``` +There is NO standalone perplexity script. The only path is to regenerate and execute the notebook: cd "/home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)" && python3 _build_notebook.py && jupyter nbconvert --to notebook --execute results.ipynb --output results.ipynb --ExecutePreprocessor.timeout=2400 +``` + +**Evidence** — `SmolLM2-134(base)/results/README.md:66-73` + +**Source quote** + +``` +# from /home/yashb98/Downloads/BuildFromScratch/SmolLM2-134(base)/ +python3 _build_notebook.py # writes results.ipynb (28 cells, no outputs) +jupyter nbconvert --to notebook \ + --execute results.ipynb \ + --output results.ipynb \ + --ExecutePreprocessor.timeout=2400 +# total runtime ~3 minutes (perplexity & training are the slow cells) +``` + +**Confidence** — measured from code + +**Caveat** — MAJOR reproducibility weakness for a model card: the PPL code is a string literal inside a notebook GENERATOR (_build_notebook.py:230-273 `code("""# %% Perplexity on wikitext-2 validation ...""")`), not an importable/runnable .py. Running this command also re-executes a 150-step training cell that OVERWRITES ../checkpoint.pt (_build_notebook.py cell at :431-447 + results.ipynb cell 23 output 'Saved checkpoint.pt'). There is no way to recompute only the PPL without extracting the cell by hand. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +The command is correct, but 'There is NO standalone perplexity script' is false. SmolLM2-134(base)/eval_after_vs_base.py IS a standalone .py that computes wikitext-2-raw-v1 validation PPL — :74 `wk = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation")`, with helper :50 `def ppl(model, text, seq=1024, stride=512, max_windows=200)` — for both the base (official weights in our class) and the TinyStories-trained checkpoint. The accurate statement is: no standalone script recomputes the HEADLINE 15.371 number. eval_after_vs_base.py would produce a DIFFERENT number (max_windows=200 vs the notebook's 61 windows; :29 `dtype = torch.bfloat16 if torch.cuda.is_available()` vs the notebook's fp32), it requires checkpoint_tinystories.pt (:43), and it has never been run to disk — its declared outputs results/tinystories_vs_base.md/.json are ABSENT from SmolLM2-134(base)/results/. +``` + +**Verifier note** + +```` +The nbconvert command matches results/README.md:67-73 verbatim (cited range 66-73 also picks up the ```bash fence at :66). The PPL code being a string literal in the generator is confirmed: _build_notebook.py:230 `code("""# %% Perplexity on wikitext-2 validation` through :273 `'seq_len': SEQ, 'stride': STRIDE}, f, indent=2)""")`. results.ipynb has exactly 28 cells (json len). Checkpoint-overwrite claim is TRUE but the line cite is wrong: the save is at _build_notebook.py:480-483 (`torch.save({'model': demo_model.state_dict(), ...` / `'checkpoint.pt')` / `print('Saved checkpoint.pt')`), NOT :431-447 (which is the cell's import block through `torch.manual_seed(0)`). Corroborated by results.ipynb cell 23 output 'Saved checkpoint.pt' and SmolLM2-134(base)/checkpoint.pt (538173921 B, mtime 2026-05-13 22:20, same minute as results.ipynb). +```` + + +### 5.6 Was Qwen3 parity verified in fp32 on CPU only, or also on GPU? + +**Value** + +``` +CPU ONLY, fp32. Neither Qwen3-0.6B/verify.py nor verify_run.py contains any .to(device)/.cuda()/device= call — tensors stay on the PyTorch CPU default. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:51,55,61-64` + +**Source quote** + +``` +line 51: hf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) +line 55: ours = Qwen3ForCausalLM(Qwen3Config()) +line 61: input_ids = tokenizer(text, return_tensors="pt").input_ids +line 63: hf_out = hf_model(input_ids).logits # (1, T, V) +line 64: our_out = ours(input_ids)["logits"] +``` + +**Confidence** — measured from code + +**Caveat** — Independently asserted in prose at Qwen3-0.6B/README.md:498 `python verify.py # parity gate — runs on CPU, no GPU needed`. Note verify.py also does NOT `import safe_cuda` — it is the one PyTorch entry point exempt from the CLAUDE.md §C1 rule, which is consistent with it never touching CUDA. verify_run.py:42 likewise uses `dtype=torch.float32` with no device move. No GPU parity check exists for Qwen3 anywhere in the repo. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Independently re-grepped: `grep -n "manual_seed|deterministic|benchmark|tf32|matmul_precision|cuda|device"` across Qwen3-0.6B/verify.py, SmolLM2-134(base)/verify.py and verify_run.py returns ONLY two comment lines (Qwen3 verify.py:59 and SmolLM2 verify.py:61, both '# Same prompt, same dtype, same device.'). No .cuda(), no .to(), no device= anywhere. Cited lines match: verify.py:51 `dtype=torch.float32`, :55, :61, :63, :64. verify_run.py:42 `dtype=torch.float32` confirmed. safe_cuda absence confirmed (neither verify.py nor verify_run.py imports it). README.md:498 corroboration confirmed. No GPU parity artifact exists for Qwen3 — unlike SmolLM2, which does have one (see next fact). +``` + + +### 5.7 Was SmolLM2 parity verified in fp32 on CPU only, or also on GPU? + +**Value** + +``` +verify.py: CPU ONLY, fp32 (no device move). A SEPARATE script, compare_with_hf.py, runs an expanded parity battery on GPU-if-available — but its JSON output was never written to disk. +``` + +**Evidence** — `SmolLM2-134(base)/verify.py:53,57,63-66` + +**Source quote** + +``` +line 53: hf_model = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.float32) +line 57: ours = SmolLM2ForCausalLM(SmolLM2Config()) +line 63: input_ids = tokenizer(text, return_tensors="pt").input_ids +line 65: hf_out = hf_model(input_ids).logits # (1, T, V) +line 66: our_out = ours(input_ids)["logits"] +``` + +**Confidence** — measured from code + +**Caveat** — GPU variant: SmolLM2-134(base)/compare_with_hf.py:40 `device = torch.device("cuda" if torch.cuda.is_available() else "cpu")` and :45/:48 `.to(device)`. It is designed to write results/comparison_with_hf.json (compare_with_hf.py:259-260), but that file does NOT exist on disk — `ls SmolLM2-134(base)/results/` shows only comparison_with_hf.md. So no GPU parity NUMBER is stamped anywhere. Prose confirmation that the committed number is CPU: SmolLM2-134(base)/results/POST_DATA.md:13 `| **0.000e+00** | `max \|Δlogits\|` between ours and HuggingFace (CPU fp32) | `results/parity.log` |`. + +**Verdict — ❌ WRONG** + +**Corrected value** + +``` +SmolLM2 parity was run on BOTH CPU and GPU, and the GPU numbers ARE recorded on disk (in prose, not JSON). SmolLM2-134(base)/results/comparison_with_hf.md:10 records 'Final-logits parity ... | max|Δ| = **4.72e-05** [GPU] | max|Δ| = **0.00e+00** [CPU]' and :11 'Per-layer hidden-state parity (30 layers) | max|Δ| = **1.95e-03** at layer 14 [GPU] | max|Δ| = **0.00e+00** at every layer [CPU]', plus :14 'Long-context (401-token RoPE) | max|Δ| = **4.01e-05**'. The same table is duplicated at SmolLM2-134(base)/README.md:71-74. Critically for a model card: the GPU per-layer delta 1.95e-03 EXCEEDS the repo's own 1e-3 gate — comparison_with_hf.md:49 is a whole section titled 'What the earlier ✗ at "1.953e-3" meant — and didn't mean'. The bit-exact 0.0 claim is CPU-ONLY; on GPU the reproduction is close but not bit-exact, attributed at comparison_with_hf.md:22-42 to SDPA backend dispatch (HF explicit mask vs our `is_causal=True`). +``` + +**Verifier note** + +``` +Only the machine-readable JSON is genuinely absent — `ls SmolLM2-134(base)/results/` confirms comparison_with_hf.json is not present while comparison_with_hf.md (4863 B, 2026-05-13 22:07) is. The verifier listed that .md but did not open it. Everything else in the fact checks out: verify.py:53/57/63/65/66 as quoted; compare_with_hf.py:39 manual_seed(0), :40 `device = torch.device("cuda" if torch.cuda.is_available() else "cpu")`, :45/:48 `.to(device)`, :259-260 the unwritten JSON dump; POST_DATA.md:13 '| **0.000e+00** | `max |Δlogits|` between ours and HuggingFace (CPU fp32) | `results/parity.log` |'. Mark the GPU numbers confidence 'prose-only' — no JSON or stdout log backs them; grep for '4.72e-05|1.95e-03' across the repo hits only comparison_with_hf.md and README.md. +``` + + +### 5.8 Which dtype kwarg does each verify.py use (version-sensitivity signal)? + +**Value** + +``` +Qwen3 uses the MODERN `dtype=`; SmolLM2 uses the DEPRECATED `torch_dtype=`, which emits a deprecation warning captured in the committed parity.log. +``` + +**Evidence** — `SmolLM2-134(base)/results/parity.log:1` + +**Source quote** + +``` +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +``` + +**Confidence** — results JSON + +**Caveat** — SmolLM2-134(base)/verify.py:53 `torch_dtype=torch.float32` vs Qwen3-0.6B/verify.py:51 `dtype=torch.float32`. The warning proves parity.log was produced under a transformers version that had already deprecated torch_dtype (consistent with the pinned transformers==5.8.0), but the log does NOT record the version number. The same deprecation line appears in Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/original_eval_run2.log:1, because eval_original_vs_repro.py:49 also uses torch_dtype=. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +SmolLM2-134(base)/results/parity.log:1 is exactly '[transformers] `torch_dtype` is deprecated! Use `dtype` instead!'. Qwen3-0.6B/verify.py:51 `dtype=torch.float32`; SmolLM2-134(base)/verify.py:53 `torch_dtype=torch.float32`. Qwen3 .../results/original_eval_run2.log:1 carries the same warning, consistent with eval_original_vs_repro.py:49 `torch_dtype=torch.bfloat16`. Two nuances if this is stated on a card: (a) the split is per-FILE, not per-repo — SmolLM2's own tests/test_parity.py:39 uses the modern `dtype=`, so 'SmolLM2 uses torch_dtype' is true only of verify.py; (b) the warning bounds transformers from below but names no version — parity.log carries no version stamp at all, so 'consistent with transformers==5.8.0' is inference, not evidence. +``` + + +### 5.9 What determinism flags are set (torch.manual_seed, use_deterministic_algorithms, cudnn.deterministic/benchmark, TF32, CUBLAS_WORKSPACE_CONFIG) in the two verify.py files? + +**Value** + +``` +NONE. Neither verify.py sets any seed or any determinism/TF32 flag. Both files' complete import+setup blocks are two imports and a module constant. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:13-19` + +**Source quote** + +``` +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +from model import Qwen3ForCausalLM, Qwen3Config + + +REPO = "Qwen/Qwen3-0.6B-Base" +``` + +**Confidence** — measured from code + +**Caveat** — SmolLM2-134(base)/verify.py:13-19 is structurally identical (`import torch` / `from transformers import ...` / `from model_full import ...` / `REPO = "HuggingFaceTB/SmolLM2-135M"`). Absence of a seed is defensible here — both scripts are pure forward passes with no sampling — but it means the scripts carry no determinism contract of their own. + +**Verdict** — _no 1:1 verifier entry; see Additional verifier findings below._ + + +### 5.10 Are TF32 / deterministic-algorithm / cuDNN flags set ANYWHERE in the repo? + +**Value** + +``` +NO — zero occurrences repo-wide of allow_tf32, use_deterministic_algorithms, cudnn.deterministic, cudnn.benchmark, CUBLAS_WORKSPACE_CONFIG, or set_float32_matmul_precision. +``` + +**Evidence** — `(repo-wide grep, /home/yashb98/Downloads/BuildFromScratch)` + +**Source quote** + +``` +$ grep -rn "allow_tf32\|use_deterministic_algorithms\|cudnn.deterministic\|cudnn.benchmark\|CUBLAS_WORKSPACE_CONFIG\|set_float32_matmul_precision" --include="*.py" --include="*.sh" --include="*.md" . | grep -v "\.git/" +(no output) +``` + +**Confidence** — measured from code + +**Caveat** — This is a definitive NEGATIVE finding, not an absence of searching. It matters for the GPU-side numbers (the SmolLM2 PPL and all Qwen3 PPL/training), where TF32 on Blackwell is a real numerical variable that is left at the PyTorch default and never recorded. It does NOT affect the CPU fp32 parity checks. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +I re-ran the grep with the stated includes (exit 1, no output) AND without any --include filter across every file type in the repo. The only hits for the broader term 'tf32/TF32' are prose/constants unrelated to a PyTorch flag: mfu_meter.py:115 (an error-message string) and research/systems/roofline_hybridssm.py:45-49/:471-472/:754 (a `peak_fp32_tf32_tflops_assumed` device-peak constant). Not one of the six actual flags appears anywhere. Definitive negative finding. +``` + + +### 5.11 Which seeds ARE set, and where (for the numbers that are not from verify.py)? + +**Value** + +``` +torch.manual_seed(0) in the notebook that produced the SmolLM2 PPL; torch.manual_seed(0) in compare_with_hf.py; torch.manual_seed(seed)+torch.cuda.manual_seed_all(seed) in the Qwen3 trainer. No seed in either verify.py. +``` + +**Evidence** — `SmolLM2-134(base)/_build_notebook.py:45-46` + +**Source quote** + +``` +torch.manual_seed(0) +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +``` + +**Confidence** — measured from code + +**Caveat** — Also: SmolLM2-134(base)/compare_with_hf.py:39 `torch.manual_seed(0)`; SmolLM2-134(base)/_build_notebook.py:201 `torch.manual_seed(42)` (generation cell) and :211/:447; Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:227-228 `torch.manual_seed(seed)` / `torch.cuda.manual_seed_all(seed)`. The paper appendix records seed 0 for training (research/papers/qwen3-imu1-matched-compute/sections/reproducibility.tex:6 'in bfloat16 with seed $0$'). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Every cited line verified: _build_notebook.py:45 `torch.manual_seed(0)` / :46 device line (quote matches exactly); :201 `torch.manual_seed(42)`; :447 `torch.manual_seed(0)`; compare_with_hf.py:39 and :211; train_qwen3.py:227 `torch.manual_seed(seed)` / :228 `torch.cuda.manual_seed_all(seed)`. Two completeness notes for a card: the Qwen3 trainer's set_seed also covers Python and NumPy (train_qwen3.py:225 `random.seed(seed)`, :226 `np.random.seed(seed)`), and the enumeration is not exhaustive — a repo-wide grep also finds SmolLM2-134(base)/train.py:95-96, train_tinystories.py:99-100, benchmark_training.py:23, eval_after_vs_base.py:112/:115. The paper cite is a cross-line merge: reproducibility.tex:5 ends '...memory) in' and :6 begins 'bfloat16 with seed $0$.' — the phrase is real, the single-line attribution to :6 is approximate. +``` + + +### 5.12 What exactly is the parity tolerance? + +**Value** + +``` +max |Δlogits| < 1e-3 (absolute), plus a hard next-token argmax-equality assertion. Identical threshold in all four parity implementations. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:74,81` + +**Source quote** + +``` +line 74: assert max_abs < 1e-3, f"Outputs diverge: {max_abs}. Architecture mismatch." +line 81: assert hf_next == our_next, "Next-token disagreement" +``` + +**Confidence** — measured from code + +**Caveat** — Same 1e-3 at: SmolLM2-134(base)/verify.py:76; Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/verify_run.py:33 `TOLERANCE = 1e-3`; SmolLM2-134(base)/tests/test_parity.py:73, :98, :132. NOTE a docstring inconsistency: SmolLM2-134(base)/verify.py:6 says the logits match 'to bf16 numerical tolerance' while the code comment at :73-75 and the run are fp32 — the module docstring is stale/wrong. Qwen3-0.6B/verify.py:6 correctly says 'fp32 numerical tolerance'. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Verified at Qwen3-0.6B/verify.py:74 and :81 (quote exact), SmolLM2-134(base)/verify.py:76, verify_run.py:33 `TOLERANCE = 1e-3`, tests/test_parity.py:73/:98/:132. The docstring inconsistency is real and correctly identified: SmolLM2-134(base)/verify.py:6 says 'the logits match to bf16 numerical tolerance' while :73-75 comments describe fp32 ~1e-5 noise and the run is fp32; Qwen3-0.6B/verify.py:6 correctly says 'fp32 numerical tolerance'. Count nuance: there are FIVE implementations, not four — compare_with_hf.py also gates on 1e-3 at :235, :240, :249. That is corroborating, but the enumeration is incomplete. Material context found while checking: SmolLM2-134(base)/results/comparison_with_hf.md:51 records 'The threshold in `compare_with_hf.py` was `1e-3`, picked for bf16 tolerance' — i.e. the repo itself notes the 1e-3 gate is loose for an fp32 claim, and the GPU per-layer run tripped it at 1.95e-03. +``` + + +### 5.13 Is "max error 0.0" a real reported value, and from which run/file? + +**Value** + +``` +YES — it is real and appears in FOUR on-disk artifacts, two of them primary machine outputs. SmolLM2: results/parity.log records max |Δlogits| = 0.000e+00. Qwen3: results/verify.json records the raw float "max_abs_error": 0.0. +``` + +**Evidence** — `SmolLM2-134(base)/results/parity.log:6-9` + +**Source quote** + +``` +max |Δlogits| = 0.000e+00 +relative = 0.000e+00 +HF next token : ' the' +Ours next : ' the' +``` + +**Confidence** — results JSON + +**Caveat** — Second primary artifact: Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/verify.json — `"max_abs_error": 0.0, "relative_error": 0.0, "hf_next_token_text": " Paris", "argmax_match": true, "passed": true`. Because verify.json stores the raw float, Qwen3's exact-zero is unambiguous; parity.log is a `.3e` format string (verify.py:70) so 0.000e+00 also implies exact zero (any nonzero would print an exponent, e.g. 1.234e-07). Derived/secondary copies: SmolLM2-134(base)/results/summary.json:7 `"max |Δlogits| vs HF": "0.000e+00"` and the executed notebook results.ipynb cell 6 output `max |Δlogits| = 0.000e+00`. The repo itself flags exact-zero as needing justification and explains it (SmolLM2-134(base)/README.md:708-718: 'The Δ = 0.0 result deserves a sanity check... Because we use the same PyTorch primitives ... in the same order, with the same dtypes, on the same input.'). + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +The value is real and correctly sourced, but it MUST be stated as a CPU-fp32-only result. The same repo records non-bit-exact GPU parity for SmolLM2 (comparison_with_hf.md:10-11: 4.72e-05 final-logits, 1.95e-03 per-layer at L14). A model card that says 'max error 0.0' without the device qualifier is misleading. Also the artifact count is 5+, not 4: comparison_with_hf.md:10 carries a fifth on-disk copy ('max|Δ| = **0.00e+00**' in the CPU column), plus README.md:56 and Qwen3-0.6B/README.md:39. +``` + +**Verifier note** + +``` +Primary artifacts verified verbatim. SmolLM2-134(base)/results/parity.log:6-9 matches the quote exactly, and verify.py:70 is `print(f'max |Δlogits| = {max_abs:.3e}')` so the .3e format does imply exact zero. Qwen3 results/verify.json read in full: "max_abs_error": 0.0, "relative_error": 0.0, "hf_next_token_text": " Paris", "argmax_match": true, "passed": true — a raw float, unambiguous. Derived copies confirmed: summary.json:7 '"max |Δlogits| vs HF": "0.000e+00"'; results.ipynb cell 6 stream output 'max |Δlogits| = 0.000e+00'. The self-scrutiny passage is at SmolLM2-134(base)/README.md:708-718 as cited and reads as quoted. +``` + + +### 5.14 What input(s) is parity checked on (prompt, token count, batch shape)? + +**Value** + +``` +A single 5-token prompt, batch size 1: "The capital of France is" → shape (1, 5). Same prompt for both models. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/verify.json` + +**Source quote** + +``` +"prompt": "The capital of France is", + "dtype": "float32", + "tolerance": 0.001, + "max_abs_error": 0.0, + "input_shape": [ + 1, + 5 + ] +``` + +**Confidence** — results JSON + +**Caveat** — Prompt set in code at Qwen3-0.6B/verify.py:60 `text = "The capital of France is"`, verify_run.py:34 `PROMPT = "The capital of France is"`, SmolLM2-134(base)/verify.py:62 (identical string). SmolLM2's 5 tokens are recorded explicitly as [504, 3575, 282, 4649, 314] (results.ipynb cell 6 output 'Tokens : [504, 3575, 282, 4649, 314]'; SmolLM2-134(base)/README.md:60). This is a THIN gate for a model card — n=1 prompt, 5 tokens, no batching, no long context. SmolLM2 alone has broader coverage via tests/test_parity.py (512-token long-context at :89-101, per-layer over all 30 blocks at :104-132); Qwen3 has NO long-context or per-layer parity check anywhere. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +verify.json read in full — "prompt": "The capital of France is", "dtype": "float32", "tolerance": 0.001, "max_abs_error": 0.0, "input_shape": [1, 5] — quote is exact. Prompt strings confirmed at Qwen3-0.6B/verify.py:60, verify_run.py:34, SmolLM2-134(base)/verify.py:62, test_parity.py:68/:79/:107. SmolLM2 token ids confirmed from results.ipynb cell 6 output 'Tokens : [504, 3575, 282, 4649, 314]'. The 'thin gate' judgement is sound. Two side findings: (1) SmolLM2-134(base)/README.md:60 attributes those token ids to `results/summary.json`, but I read summary.json in full and it has NO tokenization key (keys: Architecture, Param count (unique), lm_head tied, RoPE θ, Tokenizer, max |Δlogits| vs HF, Argmax for "France is", PPL ours, PPL HF, Demo-run final loss, Demo-run steps) — same misattribution class as fact 26; cite results.ipynb cell 6 instead. (2) The two models' argmaxes differ — Qwen3 predicts ' Paris' (verify.json), SmolLM2 predicts ' the' (parity.log:8) — so a card must not present one argmax as shared. +``` + + +### 5.15 What is the SmolLM2 perplexity number and its exact eval recipe (dataset id, config, split, seq_len, stride, tokenizer, device, dtype)? + +**Value** + +``` +ours_ppl 15.370989092449635 / hf_ppl 15.370989964425396 on 62,403 target tokens. Recipe: dataset 'Salesforce/wikitext', config 'wikitext-2-raw-v1', split 'validation'; non-blank rows joined by '\n\n'; SmolLM2's OWN tokenizer (HuggingFaceTB/SmolLM2-135M); SEQ=1024, STRIDE=512, capped at the first 32,000 tokens; fp32 weights with logits upcast .float(); device = cuda (NVIDIA GB10). +``` + +**Evidence** — `SmolLM2-134(base)/_build_notebook.py:233-242` + +**Source quote** + +``` +ds = load_dataset('Salesforce/wikitext', 'wikitext-2-raw-v1', split='validation') +text = '\\n\\n'.join(ex['text'] for ex in ds if ex['text'].strip()) +encodings = tokenizer(text, return_tensors='pt') +input_ids = encodings.input_ids[0] +print(f'Validation tokens: {len(input_ids):,}') + +# Slide a 1024-token window with stride 512 over the first 32K tokens. +SEQ = 1024 +STRIDE = 512 +N_TOKENS = min(len(input_ids), 32_000) +``` + +**Confidence** — results JSON + +**Caveat** — Values from SmolLM2-134(base)/results/perplexity.json. THREE caveats a model card must state: (1) NON-STANDARD sliding window — loss is taken over ALL 1023 shifted positions of every window (_build_notebook.py:251-254) while advancing by STRIDE=512, so every overlapped token is COUNTED TWICE. This is NOT the standard HF masked/target_len sliding-window PPL, so the 15.371 is not comparable to published wikitext-2 PPLs. I verified the arithmetic: range(0, 32000-1024, 512) = 61 windows x 1023 = 62,403, exactly matching perplexity.json's `"tokens": 62403`. (2) Only the first 32,000 of 268,140 validation tokens are used (results.ipynb cell 14 output 'Validation tokens: 268,140') — 12% of the split. (3) The PPL ran on GPU while parity ran on CPU (_build_notebook.py:244 `net = net.to(device).eval()` with device from :46); results.ipynb cell 16 output confirms 'Model on: cuda:0'. No dataset revision is pinned. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +This is the strongest-verified fact in the set. perplexity.json read in full: ours_ppl 15.370989092449635, hf_ppl 15.370989964425396, tokens 62403, dataset 'wikitext-2-raw-v1 validation', seq_len 1024, stride 512. The config name IS the -raw- variant in the code, not just the prose: _build_notebook.py:233 `ds = load_dataset('Salesforce/wikitext', 'wikitext-2-raw-v1', split='validation')`. Quoted block :233-242 matches line-for-line. I independently re-derived the arithmetic: range(0, 32000-1024, 512) = range(0, 30976, 512) = 61 starts (0…30720); 61 × 1023 = 62,403 = perplexity.json's `tokens`. The double-counting caveat is correct — window 1024 with stride 512 counts every overlapped target twice, so this is NOT standard HF masked sliding-window PPL and is not comparable to published wikitext-2 numbers. 268,140 total validation tokens confirmed at results.ipynb cell 14 output; 32000/268140 = 11.9%. Tokenizer confirmed: _build_notebook.py:112 `tokenizer = AutoTokenizer.from_pretrained(REPO)` with REPO imported at :43 from verify.py (= HuggingFaceTB/SmolLM2-135M). fp32 confirmed at :110 `hf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32)` and :251 `.float()` upcast. GPU confirmed via :244 `net = net.to(device).eval()` with device from :46; note the 'Model on: cuda:0' string the fact cites is from cell 16 (the attention cell), not the PPL cell — the PPL-on-GPU conclusion rests on the code path, which is sound. Windows actually span the first 31,744 tokens (last window 30720:31744), slightly less than the stated 32,000 cap. +``` + + +### 5.16 What is the Qwen3 perplexity number and its exact eval recipe? + +**Value** + +``` +Published Qwen3-0.6B-Base val PPL = 13.400; repro checkpoints lr17 46.892 / lr24 46.310 / lr30 49.276. Recipe: FineWeb-Edu (HuggingFaceFW/fineweb-edu, 'sample-10BT') 300,000-token val slice; 50 NON-overlapping windows of SEQ_LEN 4096 = 204,800 target tokens; Qwen3-0.6B-Base tokenizer; bfloat16; device cuda. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/original_vs_repro.txt:1-2` + +**Source quote** + +``` +[2026-06-09 16:51:36] Original vs reproduction — val=300,000 tokens, 50 windows x 4096 +ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = 13.400 (204,800 tok, 21s) +``` + +**Confidence** — results JSON + +**Caveat** — CRITICAL: this is NOT wikitext — the two models' PPL numbers are on DIFFERENT corpora and are not comparable to each other. Recipe from eval_original_vs_repro.py:21 `SEQ_LEN, MAX_WINDOWS = 4096, 50`, :30 `for begin in range(0, min(len(val) - SEQ_LEN, MAX_WINDOWS * SEQ_LEN), SEQ_LEN)` (stride == SEQ_LEN, so NO overlap and no double-counting, unlike SmolLM2), :41 `device = torch.device("cuda")`, :49 `AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.bfloat16)`. Corpus identity from train_qwen3.py:151 `ds = load_dataset("HuggingFaceFW/fineweb-edu", "sample-10BT", split="train", streaming=True)`; tokenizer from train_qwen3.py:281 `tokenizer = AutoTokenizer.from_pretrained(REPO)` with REPO='Qwen/Qwen3-0.6B-Base' (:59). bf16 (not fp32) means this number carries real numerical noise that is never quantified. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +original_vs_repro.txt read in full — all four PPL values and the header line match verbatim, including '(204,800 tok, 21s)'. Recipe lines all verified: eval_original_vs_repro.py:21 `SEQ_LEN, MAX_WINDOWS = 4096, 50`, :30 the range with stride == SEQ_LEN, :41 cuda, :49 `torch_dtype=torch.bfloat16`. I re-derived the token count: min(300000-4096, 50*4096) = 204800, range(0,204800,4096) = 50 windows, each contributing ids.size(1)-1 = 4096 → 204,800. Targets do not overlap, so the no-double-counting claim holds (inputs share exactly 1 boundary token per window). Corpus at train_qwen3.py:151 `load_dataset("HuggingFaceFW/fineweb-edu", "sample-10BT", split="train", streaming=True)`; tokenizer at :281 with REPO='Qwen/Qwen3-0.6B-Base' at :59. The 'different corpora, not comparable to SmolLM2' warning is correct and material. The bf16-noise-unquantified caveat is fair: no seed-repeat or CI exists for this number. +``` + + +### 5.17 Is the Qwen3 13.400 a number this repo MEASURED, or one COPIED from a paper / model card? + +**Value** + +``` +MEASURED by this repo. It is the output of eval_original_vs_repro.py loading the published Qwen/Qwen3-0.6B-Base weights and scoring them with the repo's own eval loop on the repo's own val slice. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py:46-51` + +**Source quote** + +``` +# --- the original published model --- + from transformers import AutoModelForCausalLM + t0 = time.time() + hf = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.bfloat16).to(device) + ppl_orig, n = eval_ppl(hf, val, device) + lines.append(f"ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = {ppl_orig:8.3f} ({n:,} tok, {time.time()-t0:.0f}s)") +``` + +**Confidence** — measured from code + +**Caveat** — The run is corroborated by a live stdout log with a real safe_cuda banner: results/original_eval_run2.log:2 '[safe_cuda] capped CUDA at 85% of 129 GB unified pool (~109 GB)' and :3 'Loading weights: 100%|██████████| 310/310'. The '36T tokens' attribution in the label IS copied from Qwen3 published material, not measured. Same for SmolLM2: perplexity.json's hf_ppl 15.370990 is measured by the same loop on the same slice (_build_notebook.py:263 `hf_ppl, _ = ppl(hf_model)`), not copied. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +eval_original_vs_repro.py:46-51 matches the quote line-for-line, including `hf = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.bfloat16).to(device)` at :49 and the ppl_orig eval at :50. Live-run corroboration confirmed: results/original_eval_run2.log:2 '[safe_cuda] capped CUDA at 85% of 129 GB unified pool (~109 GB); over-allocation now errors cleanly.' and :3 'Loading weights: 100%|██████████| 310/310'; :6 of that log carries the same 13.400 line, and :13 confirms it wrote original_vs_repro.txt. The separation the fact draws is correct and important: 13.400 is MEASURED, the '(36T tok)' label inside the same string is COPIED from Qwen3 published material and is not verifiable from any file in this repo. SmolLM2's hf_ppl is likewise measured by the same loop (_build_notebook.py:263 `hf_ppl, _ = ppl(hf_model)`). +``` + + +### 5.18 Does the Qwen3 PPL command use the decontaminated val split? + +**Value** + +``` +NO. eval_original_vs_repro.py hardcodes the PRE-audit token cache tokcache_133072000_300000.pt, whose filename lacks the seed/tokenizer tag that the decontamination fix introduced. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:130-136` + +**Source quote** + +``` +"""Stream FineWeb-Edu sample-10BT, tokenize on the fly, and build a + DOCUMENT-DISJOINT, DECONTAMINATED train/val split (audit fix DATA-1/3): + + * each whole document is routed to train or val by a seeded hash + (`is_val_doc`), so train/val never share a document and no document spans + the boundary (the old code cut the stream by token count — val was the + sequential continuation of train, leak-suspect); +``` + +**Confidence** — measured from code + +**Caveat** — The post-fix cache naming is train_qwen3.py:145 `cache = RESULTS / f"tokcache_{n_train}_{n_val}_seed{seed}_{tok_tag}.pt"`. eval_original_vs_repro.py:22 points at `tokcache_133072000_300000.pt` — no seed/tok tag — i.e. the OLD leak-suspect sequential split. Both files exist on disk: the old one dated Jun 8 (used for the 13.400 run on Jun 9) and tokcache_133072000_300000_seed0_Qwen3-0.6B-Base.pt dated Jun 18 (post-fix). results/decontam_report.json (docs_dropped: 0) is dated Jul 7 and corresponds to a later, larger cache, NOT to the 13.400 run. So the headline PPL row predates the decontamination fix and re-running the command today reproduces it on the same pre-fix split. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Strongly supported. train_qwen3.py:130-136 matches the quoted docstring verbatim, including 'the old code cut the stream by token count — val was the sequential continuation of train, leak-suspect'. Post-fix naming confirmed at train_qwen3.py:145 `cache = RESULTS / f"tokcache_{n_train}_{n_val}_seed{seed}_{tok_tag}.pt"`; eval_original_vs_repro.py:22 points at the untagged name. Both caches exist: tokcache_133072000_300000.pt mtime 2026-06-08 21:13 and tokcache_133072000_300000_seed0_Qwen3-0.6B-Base.pt mtime 2026-06-18 13:45 — and the 13.400 run is timestamped 2026-06-09 16:51:36 in original_vs_repro.txt:1, i.e. 9 days BEFORE the post-fix cache existed. decontam_report.json mtime 2026-07-07 15:13, content docs_dropped 0 / n_val_docs_kept 451 — confirmed. One softening: 'corresponds to a later, larger cache' is not verifiable — decontam_report.json contains no cache filename or token count (keys: split_seed, val_fraction, ngram_n, overlap_threshold, n_train_docs, n_val_docs_raw, docs_dropped, method, train_sample_docs_for_index), and train_qwen3.py:184 writes it to a fixed path `RESULTS / "decontam_report.json"` that any re-stream overwrites. What IS provable is the date ordering, which is sufficient for the claim. +``` + + +### 5.19 Every place a python/torch/CUDA version is PINNED in the repo + +**Value** + +``` +Exactly TWO files, both under SmolLM2-134(base), and they CONTRADICT each other. pyproject.toml has hard pins (torch==2.11.0, transformers==5.8.0, datasets==4.8.5, safetensors==0.7.0, accelerate==1.13.0, numpy==2.4.4, requires-python>=3.10, pytest==9.0.3). requirements.txt has loose floors (torch>=2.4, transformers>=4.40). +``` + +**Evidence** — `SmolLM2-134(base)/pyproject.toml:1-2,17,19-27` + +**Source quote** + +``` +line 1: # Concrete pins, matching the working environment that produced +line 2: # max|Δlogits|=0.0 against HuggingFaceTB/SmolLM2-135M. +line 17: requires-python = ">=3.10" +line 19: dependencies = [ +line 20: # Pinned to the versions that verified parity in this repo. +line 21: "torch==2.11.0", +line 22: "transformers==5.8.0", +line 23: "datasets==4.8.5", +line 24: "safetensors==0.7.0", +line 25: "accelerate==1.13.0", +line 26: "numpy==2.4.4", +line 27: ] +``` + +**Confidence** — measured from code + +**Caveat** — The contradiction: SmolLM2-134(base)/requirements.txt:1-2 reads `torch>=2.4` / `transformers>=4.40`, yet root README.md:157 offers them as equivalent (`pip install -e . # or: pip install -r requirements.txt`) directly under the comment at :156 'Install pinned dependencies that produced the 0.0 logit-diff result.' Following the requirements.txt branch can install torch 2.4 / transformers 4.x, under which SmolLM2's verify.py would still run (torch_dtype= is valid there) but bit-exactness is NOT the pinned-environment claim. No CUDA/cuDNN/driver version is pinned in either file. pyproject.toml:6-8 itself admits no lockfile exists: 'To freeze a true lockfile (recommended for reproducibility): uv pip compile pyproject.toml -o requirements.lock'. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +pyproject.toml read in full — :1-2 comment, :17 `requires-python = ">=3.10"`, :21-26 torch==2.11.0 / transformers==5.8.0 / datasets==4.8.5 / safetensors==0.7.0 / accelerate==1.13.0 / numpy==2.4.4, :31 pytest==9.0.3 — all as claimed. requirements.txt read in full: `torch>=2.4` / `transformers>=4.40` / safetensors / accelerate / datasets (the last three fully unpinned, which the fact does not mention but which strengthens it). The contradiction via root README.md:157 `pip install -e . # or: pip install -r requirements.txt` under :156's 'Install pinned dependencies that produced the 0.0 logit-diff result.' is confirmed. No CUDA/cuDNN/driver pin in either. pyproject.toml:4-7 does carry the no-lockfile admission ('To freeze a true lockfile (recommended for reproducibility): uv pip compile pyproject.toml -o requirements.lock') — cited as :6-8, the text actually spans :4-7. +``` + + +### 5.20 Every place a python/torch/CUDA version is STAMPED from an actual execution + +**Value** + +``` +Exactly ONE: the executed notebook SmolLM2-134(base)/results.ipynb. Cell 1 output stamps 'Torch: 2.11.0+cu130' and 'Device: cuda | NVIDIA GB10'; notebook metadata.language_info.version stamps python '3.12.11'. +``` + +**Evidence** — `SmolLM2-134(base)/results.ipynb (cell 1 output; metadata.language_info.version)` + +**Source quote** + +``` +Torch: 2.11.0+cu130 +Device: cuda | NVIDIA GB10 + +(metadata.language_info.version = "3.12.11") +``` + +**Confidence** — results JSON + +**Caveat** — Produced by SmolLM2-134(base)/_build_notebook.py:50-51 `print('Torch:', torch.__version__)` / `print('Device:', device, '|', torch.cuda.get_device_name(0) ...)`. A repo-wide grep for `torch.__version__|torch.version.cuda|sys.version|platform.python_version` in *.py returns only this line and one unrelated HybridSSM file (HybridSSM-0.2B/experiments/2026-07-29_hybrid-ssm-0.2b_throughput/step_runner.py:245 `"host": platform.node(), "python": platform.python_version(),`). So NO Qwen3 script and NEITHER verify.py stamps any version. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Read results.ipynb programmatically: metadata.language_info.version == '3.12.11', and cell 1's stream output is exactly 'Torch: 2.11.0+cu130\nDevice: cuda | NVIDIA GB10'. Generator lines confirmed at _build_notebook.py:50-51. I re-ran the repo-wide grep for `torch.__version__|torch.version.cuda|platform.python_version|sys.version` over all *.py: exactly two hits, _build_notebook.py:50 and HybridSSM-0.2B/experiments/2026-07-29_hybrid-ssm-0.2b_throughput/step_runner.py:245 — as claimed. I additionally checked whether step_runner's stamp reached disk: grep for '"python":' under that experiment dir returns only the .py itself, no output artifact, so 'exactly ONE' holds repo-wide. I also grepped all *.log/*.txt/*.json for 'Torch:|torch==|2.11.0': zero hits, confirming no result artifact carries a version. +``` + + +### 5.21 Do the repo's recorded versions AGREE with the live box (python 3.12.11, torch 2.11.0+cu130, CUDA 13.0, cuDNN 91900, driver 580.142, NVIDIA GB10)? + +**Value** + +``` +For SmolLM2: YES, exactly, on every field the repo records. For Qwen3: UNDETERMINABLE — the Qwen3 subproject records no versions at all. +``` + +**Evidence** — `SmolLM2-134(base)/pyproject.toml:21-23 vs live interpreter` + +**Source quote** + +``` +repo pins: "torch==2.11.0", "transformers==5.8.0", "datasets==4.8.5" +live box: python 3.12.11 / torch 2.11.0+cu130 / torch.version.cuda 13.0 / transformers 5.8.0 / datasets 4.8.5 +notebook: Torch: 2.11.0+cu130 | Device: cuda | NVIDIA GB10 | language_info.version 3.12.11 +``` + +**Confidence** — measured from code + +**Caveat** — I verified the live stack myself by running `python3 -c "import sys,torch,transformers,datasets; ..."` (no CUDA init): python 3.12.11, torch 2.11.0+cu130, torch.version.cuda 13.0, transformers 5.8.0, datasets 4.8.5. These match pyproject and the notebook stamp field-for-field, so the SmolLM2 results were NOT produced under an older stack — bit-exactness there is currently re-checkable. Two gaps remain: (a) cuDNN 91900 and driver 580.142 are recorded NOWHERE in the repo, so version-drift in those two cannot be detected; (b) pyproject pins the base version 'torch==2.11.0' without the +cu130 local tag, so a CUDA-12 or CPU build of the same version satisfies the pin. For Qwen3 there is NO requirements.txt and NO pyproject.toml — its only install instruction is fully unpinned. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +I re-ran the live check myself rather than trusting the report: python 3.12.11, torch 2.11.0+cu130, torch.version.cuda 13.0, torch.backends.cudnn.version() 91900, transformers 5.8.0, datasets 4.8.5; `nvidia-smi --query-gpu=driver_version,name` returns '580.142, NVIDIA GB10'. These match pyproject.toml:21-23 and the results.ipynb stamp field-for-field. Both stated gaps are real: (a) 91900 and 580.142 appear nowhere on disk — my grep of all *.md/*.json/*.toml/*.txt/*.log for 'cuDNN|cudnn_version|580.14|CUDA 13|CUDA 12' returned exactly one unrelated prose hit, jax_vs_pytorch_tradeoffs.md:44; (b) `torch==2.11.0` has no +cu130 local tag, so a CPU or CUDA-12 build satisfies the pin. Qwen3-0.6B has no requirements.txt and no pyproject.toml (confirmed by find, below). +``` + + +### 5.22 What does Qwen3-0.6B document as its dependency install? + +**Value** + +``` +An unpinned one-liner in the README. Qwen3-0.6B has no requirements.txt, no pyproject.toml, and no lockfile. +``` + +**Evidence** — `Qwen3-0.6B/README.md:496-499` + +**Source quote** + +```` +```bash +pip install torch transformers datasets safetensors accelerate +python verify.py # parity gate — runs on CPU, no GPU needed +``` +```` + +**Confidence** — measured from code + +**Caveat** — Confirmed by a repo-wide find for requirements*.txt / environment*.yml / pyproject.toml / *.lock / setup.py / Pipfile: the only hits are SmolLM2-134(base)/pyproject.toml, SmolLM2-134(base)/requirements.txt, and skills_showcase/server/requirements.txt (a FastAPI web server, unrelated to either model: fastapi/uvicorn/pydantic/anthropic). For a model card, Qwen3's bit-exact claim therefore has no reproducible environment attached. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +```` +Qwen3-0.6B/README.md:496-499 matches the quote exactly (496 ```bash, 497 `pip install torch transformers datasets safetensors accelerate`, 498 `python verify.py # parity gate — runs on CPU, no GPU needed`, 499 ```). I re-ran the repo-wide find for requirements*.txt / pyproject.toml / environment*.yml / setup.py / Pipfile: exactly three hits — SmolLM2-134(base)/pyproject.toml, SmolLM2-134(base)/requirements.txt, skills_showcase/server/requirements.txt. Nothing under Qwen3-0.6B/. The conclusion — Qwen3's bit-exact claim ships with no reproducible environment — is sound. +```` + + +### 5.23 Is there a git tag corresponding to these results? + +**Value** + +``` +NO. The repository has zero tags. +``` + +**Evidence** — `/home/yashb98/Downloads/BuildFromScratch (git)` + +**Source quote** + +``` +$ git tag -l +(no output) +``` + +**Confidence** — measured from code + +**Caveat** — Current HEAD is 3da9063 on branch harden-research-loop, which is ~2 months of commits after both parity artifacts were produced (parity.log May 13, verify.json Jun 8). A model card cannot point at a tag; the best available anchors are the commits that ADDED the artifacts (next fact). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +`git tag -l | wc -l` = 0. HEAD = 3da9063 on branch harden-research-loop, confirmed by `git rev-parse --short HEAD` / `git branch --show-current`. One precision fix: HEAD's commit date is 2026-07-24 (`git log -1 --date=short`), so the gap is ~1.5 months after verify.json (mtime 2026-06-08 14:36) and ~2.3 months after parity.log (mtime 2026-05-13 22:20) — 'about 2 months' is a fair rounding but not exact for either. +``` + + +### 5.24 Is there a recorded commit hash that these specific results correspond to? + +**Value** + +``` +NOT_FOUND as a provenance stamp. No results file for either model carries a commit/git_sha field. The only commit anchors are the git history entries that first added the artifacts: verify.json → e791875; parity.log → 84a96c0. +``` + +**Evidence** — `/home/yashb98/Downloads/BuildFromScratch (git log)` + +**Source quote** + +``` +$ git log --oneline -3 -- "Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/verify.json" +e791875 Add Qwen3-0.6B from-scratch reproduction + three-build experiment +$ git log --oneline -3 -- "SmolLM2-134(base)/results/parity.log" +84a96c0 Initial commit: SmolLM2-135M from-scratch reproduction + harness +``` + +**Confidence** — measured from code + +**Caveat** — These are inferences from file history, NOT provenance the scripts recorded. Direct inspection of the artifacts confirms the absence: verify.json's full key set is repo/prompt/dtype/tolerance/max_abs_error/relative_error/hf_next_token_id/our_next_token_id/hf_next_token_text/our_next_token_text/argmax_match/passed/input_shape/total_seconds — no commit, no versions, no device, no timestamp. perplexity.json's full key set is ours_ppl/hf_ppl/tokens/dataset/seq_len/stride — likewise none. parity.log is raw stdout. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Re-ran both git-log queries: verify.json → e791875 'Add Qwen3-0.6B from-scratch reproduction + three-build experiment' (2026-06-10); parity.log → 84a96c0 'Initial commit: SmolLM2-135M from-scratch reproduction + harness' (2026-05-20). Note both adding commits POSTDATE the artifact mtimes (Jun 10 vs Jun 8; May 20 vs May 13), so these commits bound the results from above, not identify the tree they ran against — which reinforces the fact's own 'inference, not provenance' framing. The absence claim is verified by direct inspection: verify.json's full key set is exactly repo/prompt/dtype/tolerance/max_abs_error/relative_error/hf_next_token_id/our_next_token_id/hf_next_token_text/our_next_token_text/argmax_match/passed/input_shape/total_seconds; perplexity.json's is exactly ours_ppl/hf_ppl/tokens/dataset/seq_len/stride. Neither has commit, versions, device, or timestamp. +``` + + +### 5.25 Does the repo's provenance/ledger machinery cover these two reproductions? + +**Value** + +``` +NO. research/ledger/ledger.py DOES auto-capture the repo HEAD into every run entry, and 29 runs carry a git_commit — but the earliest is 2026-06-16 and NEITHER reproduction (SmolLM2 parity/PPL, nor the Qwen3 faithful verify) has a ledger run entry. +``` + +**Evidence** — `research/ledger/ledger.py:639` + +**Source quote** + +``` +r["lineage"]["git_commit"] = git_head_commit() # auto-capture repo HEAD +``` + +**Confidence** — measured from code + +**Caveat** — Verified by enumerating ledger.json: run_ids run 2026-06-16_qwen3-faithful_eval-first (git_commit 86e79f3) through 2026-07-29_hybrid-ssm-0.2b_private-heldout-v1 (3da9063); no entry names the SmolLM2 repro or a Qwen3 verify/parity run. Additionally lineage.env is null for 28 of 29 runs (the sole exception is run[21], free text 'jax/flax on GB10; jax_safe_env guard active'), so even the covered runs stamp no software versions. research/provenance.py (§C22 span writer) exists but research/provenance/ contains a single file, 2026-07-14.jsonl, which post-dates both reproductions. No c5_evidence.json exists for either reproduction build — the 12 c5_evidence.json files all belong to later ablation/experiment runs. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +research/ledger/ledger.py:639 is exactly `r["lineage"]["git_commit"] = git_head_commit() # auto-capture repo HEAD`. I enumerated ledger.json programmatically: 29 runs, all 29 carry git_commit; first = 2026-06-16_qwen3-faithful_eval-first (86e79f3), last = 2026-07-29_hybrid-ssm-0.2b_private-heldout-v1 (3da9063). No run_id names SmolLM2 or a Qwen3 verify/parity run. lineage.env is null for 28 of 29; the sole exception is index 21, 2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0, env = 'jax/flax on GB10; jax_safe_env guard active' — exactly as claimed. research/provenance/ contains only 2026-07-14.jsonl. Minor count nuance: `find -name "c5_evidence*.json"` returns 12 paths, of which 11 are named exactly c5_evidence.json and one is c5_evidence_scale_ext.json; none is under a reproduction build — the substantive claim stands. +``` + + +### 5.26 Does the paper's reproducibility appendix record software versions? + +**Value** + +``` +NO. It records hardware (single NVIDIA GB10, ~119 GB unified), dtype (bfloat16), seed (0), batch/step config, and the exact commands — but no python, torch, CUDA, or transformers version. +``` + +**Evidence** — `research/papers/qwen3-imu1-matched-compute/sections/reproducibility.tex:4-10` + +**Source quote** + +``` +are released at \url{https://github.com/yashb98/BuildFromScratch}. All training used +a single NVIDIA GB10 (Grace Blackwell, unified $\approx\!119$\,GB CPU+GPU memory) in +bfloat16 with seed $0$. Because the unified pool can be exhausted by a single large +allocation, every entry point caps the process at $85\%$ of the pool before initializing +the accelerator and computes the $151{,}936$-way cross-entropy in chunks; \texttt{torch.compile} +is enabled. The effective batch is $4 \times 4$ accumulation $= 65{,}536$ tokens, and both +arms train for $18{,}150$ steps. +``` + +**Confidence** — measured from code + +**Caveat** — The appendix is otherwise unusually honest — reproducibility.tex:35-37 explicitly flags an unsaved artifact: 'its bundle-off-equals-baseline verification was run and recorded in the build documentation (printed to standard output rather than saved as a file---we note this explicitly so the provenance is not overstated).' Its command list (lines 15-27) matches the scripts on disk. Absence of a version block is the one clear gap for a model-card Reproduce section. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +I read reproducibility.tex in full (40 lines). Lines 4-10 match the quote verbatim. No version string appears anywhere in the file. The honesty passage is real, at :35-37 as cited: 'its bundle-off-equals-baseline verification was run and recorded in the build documentation (printed to standard output rather than saved as a file---we note this explicitly so the provenance is not overstated).' The command block at :13-28 lists verify_run.py, train_qwen3.py, verify_imu1.py, train_imu1.py, eval_original_vs_repro.py, figures/make_figures.py — all of which exist on disk. Worth pairing with fact 18 on a card: the appendix presents `python eval_original_vs_repro.py` as authoritative while that script reads the pre-decontamination cache. +``` + + +### 5.27 Is the Qwen3 README's sourcing of the parity/param claim accurate? + +**Value** + +``` +PARTLY WRONG. The README attributes params 596,049,920 to verify.json, but verify.json contains no params field. The max_abs_error = 0.0 and argmax " Paris" attributions ARE correct. +``` + +**Evidence** — `Qwen3-0.6B/README.md:39-40` + +**Source quote** + +``` +**Bit-exact reproduction** — `verify.json`: `max_abs_error = 0.0`, argmax `" Paris"`, +params **596,049,920**. Our `model.py` *is* Qwen3-0.6B. +``` + +**Confidence** — measured from code + +**Caveat** — verify.json's complete key list (quoted in an earlier fact) has no params/n_params/param_count key. 596,049,920 IS a genuine measured value, but it lives elsewhere: Qwen3-0.6B/model.py:306 prints it as an EXPECTED constant (`print(f"Expected: ~596,049,920 (596M-branded, '0.6B')")`), and it is emitted as a real measurement in training logs, e.g. Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/run_sft_seed0.log:7 'base loaded strict=True (base PPL on FineWeb-Edu was 28.650247632362024); params=596,049,920'. A model card should cite the log, not verify.json. SmolLM2's equivalent claim is better sourced: 134,515,008 is asserted by a real test (tests/test_parity.py:53) and recorded in results/summary.json:3. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/README.md:39-40 matches the quote exactly, and I read verify.json in full: no params/n_params/param_count key. The alternative sources check out: Qwen3-0.6B/model.py:306 `print(f"Expected: ~596,049,920 (596M-branded, '0.6B')")` — note this is a printed EXPECTED constant in a __main__ block, and the adjacent :305 `print(f"Unique params: {num_params(m):,}")` is the actual measurement, so model.py:306 alone is not a measurement either. The genuine measured stamp is Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/run_sft_seed0.log:7 'base loaded strict=True (base PPL on FineWeb-Edu was 28.650247632362024); params=596,049,920' — verified verbatim. SmolLM2's better sourcing also verified: tests/test_parity.py:53 `assert n == 134_515_008`, results/summary.json:3, and results/param_count.log:1 'params: 134,515,008 (target 134,515,008)'. +``` + + +### 5.28 Is the documented standardized-benchmark path (lm-evaluation-harness) actually exercised? + +**Value** + +``` +NO. SmolLM2-134(base)/scripts/run_lm_eval.sh exists and is documented in the root README quickstart, but its output directory results/lm_eval/ does not exist — it has never been run. +``` + +**Evidence** — `SmolLM2-134(base)/scripts/run_lm_eval.sh:22-27` + +**Source quote** + +``` +BASE_REPO="HuggingFaceTB/SmolLM2-135M" +TASKS="hellaswag,arc_easy,arc_challenge,piqa,winogrande,commonsense_qa,openbookqa,mmlu" +DEVICE="${DEVICE:-cuda:0}" +BATCH_SIZE="${BATCH_SIZE:-auto}" +NUM_FEWSHOT="${NUM_FEWSHOT:-0}" +OUT_DIR="results/lm_eval" +``` + +**Confidence** — measured from code + +**Caveat** — `ls "SmolLM2-134(base)/results/lm_eval"` → 'No such file or directory'. The script is advertised in root README.md:176-179 ('# Standardized benchmarks (lm-evaluation-harness wrapper): pip install lm-eval / bash scripts/run_lm_eval.sh'). A model card must not imply any downstream benchmark number for SmolLM2 from this repo — none exists. Note also the script would evaluate in bfloat16 (line 41 `dtype=bfloat16`), not the fp32 used for parity. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +scripts/run_lm_eval.sh:22-27 matches the quote line-for-line (22 BASE_REPO, 23 TASKS, 24 DEVICE, 25 BATCH_SIZE, 26 NUM_FEWSHOT, 27 OUT_DIR="results/lm_eval"). `ls "SmolLM2-134(base)/results/lm_eval"` → 'No such file or directory', and my full listing of results/ shows no lm_eval entry. Root README.md:176-179 advertises it as claimed. The bf16-vs-fp32 mismatch is confirmed at run_lm_eval.sh:41 `--model_args "pretrained=$BASE_REPO,dtype=bfloat16"`. Supporting evidence the verifier did not cite: SmolLM2-134(base)/results/comparison_with_hf.md:84-88 explicitly states downstream benchmarks were NOT computed ('Computing them from scratch would take a few hours per task') and asserts 'any benchmark score will match by construction' — an unverified claim a card must not repeat. Same non-execution pattern applies to eval_after_vs_base.py: its declared outputs results/tinystories_vs_base.md/.json are absent from disk. +``` + + +### 5.V Additional verifier findings (no 1:1 extracted fact) + +**5.V1 — ✅ CONFIRMED** · What determinism flags are set in the two verify.py files? + +**Checked against** + +``` +NONE. Neither verify.py sets any seed or any determinism/TF32 flag. +``` + +**Verifier note** + +``` +Verified by direct grep across both files for manual_seed / deterministic / benchmark / tf32 / matmul_precision: zero hits. Qwen3-0.6B/verify.py:13-19 matches the quote line-for-line (13 `import torch`, 14 `from transformers import ...`, 16 `from model import Qwen3ForCausalLM, Qwen3Config`, 19 `REPO = "Qwen/Qwen3-0.6B-Base"`); SmolLM2-134(base)/verify.py:13-19 is structurally identical with `from model_full import SmolLM2ForCausalLM, SmolLM2Config` at :16 and `REPO = "HuggingFaceTB/SmolLM2-135M"` at :19. Both scripts are pure @torch.no_grad() forward passes (verify.py:47 / :49), so seed absence is inconsequential for the CPU number — but see the refuted GPU fact: comparison_with_hf.md attributes the non-zero GPU deltas precisely to unpinned backend dispatch, which is what a determinism flag would have controlled. +``` + + +### 5.G Gaps — not determinable from disk + +- cuDNN version (live: 91900) and NVIDIA driver version (live: 580.142) are recorded NOWHERE in the repo. I grepped all *.md/*.json/*.toml/*.txt/*.log for 'cuDNN|cudnn_version|580.14|CUDA 13|CUDA 12' and the only hit was an unrelated prose mention in jax_vs_pytorch_tradeoffs.md:44. Version drift in these two cannot be detected from disk. +- The exact torch/transformers versions under which the Qwen3 verify.json (2026-06-08) and original_vs_repro.txt (2026-06-09) were produced are undeterminable. Qwen3-0.6B has no requirements.txt/pyproject.toml, no script stamps torch.__version__, verify.json has no env fields, and no ledger entry covers those runs. The only weak signal is original_eval_run2.log:1's torch_dtype deprecation warning, which bounds transformers from below but names no version. +- The exact transformers version behind SmolLM2's parity.log (2026-05-13) is likewise unstamped. parity.log:1's deprecation warning is consistent with the pinned transformers==5.8.0 but does not prove it. The sibling results.ipynb (same day) stamps torch 2.11.0+cu130 / python 3.12.11, which is strong circumstantial evidence for the same session, but parity.log itself carries no stamp. +- No GPU-side parity number exists for either model. compare_with_hf.py is written to run parity on cuda and to save results/comparison_with_hf.json (compare_with_hf.py:259-260), but that JSON is absent from disk (only comparison_with_hf.md is present). Whether the bit-exact 0.0 survives on GB10 GPU, under default TF32 settings, is untested and unrecorded. +- The SmolLM2 15.371 PPL and the Qwen3 13.400 PPL are on DIFFERENT corpora with DIFFERENT window semantics, dtypes, and devices (wikitext-2-raw-v1 / overlapping stride-512 / fp32 vs FineWeb-Edu / non-overlapping / bf16). Nothing on disk makes them comparable, and no single command recomputes both. +- There is no HuggingFace model-card file, no MODEL_CARD.md, and no exported HF repo for either reproduction anywhere under the repo root — so there is no existing Reproduce section on disk to check these commands against. (SmolLM2-134(base)/scripts/export_to_hf.py exists as an exporter, but results/lm_eval/ and any exported dir are absent.) +- No lockfile exists anywhere (find for *.lock returned only research/loop_state.json.lock, .claude/scheduled_tasks.lock, research/ledger/ledger.json.lock — all mutex files, not dependency locks). pyproject.toml:6-8 acknowledges this and gives the uv/pip-compile command to create one, but it was never run. + +--- + +## 6. Training details (the 1.19B-token runs) + +Audit dimension: Training details (from-scratch Qwen3-0.6B runs behind the ~1.19B-token figure, + SmolLM2) + +### 6.1 Which concrete runs are behind the ~1.19B-token figure? + +**Value** + +``` +Four 'Phase B' runs at 18,150 steps x 65,536 tok/step = 1,189,478,400 tokens each, launched sequentially by phase_b_driver.sh: (1) faithful baseline (train_qwen3.py, run_name=baseline2tpp), (2) IMU-1/NorMuon modernized (train_imu1.py, imu1_2tpp), (3) partial-RoPE 0.25 (train_partialrope.py, prope25_2tpp), (4) partial-RoPE 0.10 (prope10_2tpp). Run 4 died incomplete at step ~5450/18150. +``` + +**Evidence** — `Qwen3-0.6B/builds/phase_b_driver.sh:2-34` + +**Source quote** + +``` +# Phase B — 4 matched-compute runs @ 2 TPP (18,150 steps = 1.19B tokens each). +# Sequential (one GPU job at a time on the GB10). Best LR from Phase A = 2.4e-3. +S=18150; W=900; COMMON="--eval_every 2000 --ckpt_every 2000 --log_every 50" +cd "$FAITHFUL" && python train_qwen3.py --steps $S --peak_lr 2.4e-3 --end_lr 3.2e-4 \ + --warmup_steps $W $COMMON --run_name baseline2tpp +``` + +**Confidence** — measured from code + +**Caveat** — prope10 is incomplete — the comparison README itself says 'died incomplete at step 5450/18150 (~30%)' (Qwen3-0.6B/builds/comparison/README.md:10). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Verified verbatim. Qwen3-0.6B/builds/phase_b_driver.sh:2 ('Phase B — 4 matched-compute runs @ 2 TPP (18,150 steps = 1.19B tokens each).'), :3 (best LR 2.4e-3), :14 ('S=18150; W=900; COMMON=...'), :19-20 (faithful/baseline2tpp), :24 (train_imu1.py --run_name imu1_2tpp), :28-29 (--partial_rotary_factor 0.25 prope25_2tpp), :33-34 (0.10 prope10_2tpp). 18150*65536 = 1,189,478,400 exactly. prope10 incompleteness independently confirmed: qwen3_prope10_2tpp_train.log has 116 lines and its LAST line is '[22:04:32] step 5450/18150 ... tok/s 7,096'. Qwen3-0.6B/builds/comparison/README.md:10 quote is verbatim. +``` + + +### 6.2 GPU model and count + +**Value** + +``` +1 x NVIDIA GB10 (Grace Blackwell), unified ~119 GB CPU+GPU pool, no separate VRAM. Device string is measured (torch.cuda.get_device_name(0)). Count = 1 is NOT recorded as a measured value anywhere on disk — no file contains torch.cuda.device_count() output. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/throughput_probe.json:2` + +**Source quote** + +``` +"device": "NVIDIA GB10", +``` + +**Confidence** — measured from code + +**Caveat** — Single-GPU-ness is asserted in prose only: CLAUDE.md:3 ('single-box ML research repo on an NVIDIA GB10 ... unified ~119 GB CPU+GPU memory pool') and CLAUDE.md:9 ('ONE GPU job at a time on the GB10'). safe_cuda.guard(device=0) and get_device_name(0) only prove device 0 exists. Grep for 'device_count' across all .py/.json/.md returned zero hits. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +throughput_probe.json:2 is verbatim ('"device": "NVIDIA GB10",'). I re-ran the device_count grep repo-wide excluding .git: ZERO hits, so the 'count is not measured' claim survives. CLAUDE.md:3 and CLAUDE.md:9 quotes verified verbatim. The honest MEASURED/PROSE split here is correct and should be preserved on the card. +``` + + +### 6.3 Wall-clock hours — faithful baseline (the 1.19B headline run) + +**Value** + +``` +2,663.1 minutes = 44.4 h of training-loop time (excludes the ~26 min corpus stream+tokenize and the baseline eval). Sustained ~7,444-7,481 tok/s. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log:395` + +**Source quote** + +``` +[18:05:05] Training complete in 2663.1 min. +``` + +**Confidence** — measured from code + +**Caveat** — The corpus stream/tokenize step took an extra 1573.8 s (~26 min), logged separately at line 6 of the same file. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +2,663.1 min = 44.4 h (confirmed). Throughput sub-claim should read: cumulative tok/s ranged 7,414–7,483 over the run, ending at 7,444 — not '7,444–7,481'. +``` + +**Verifier note** + +``` +Hours CONFIRMED: qwen3_baseline2tpp_train.log:395 is verbatim '[18:05:05] Training complete in 2663.1 min.' 2663.1 min = 44.385 h. Scope confirmed by arithmetic: training started at :12 '[21:41:59] Training to 18,150 steps' and 21:41:59 + 2663.1 min lands exactly on 18:05:05, so the timer excludes both the stream (:6, 1573.8 s = 26.2 min) and the baseline eval (:11). REFUTED sub-claim: the max cumulative tok/s in the log is 7,483 (line 41, step 1450), not 7,481; the min is 7,414. The stated '7,444-7,481' band is wrong at both ends. +``` + + +### 6.4 Wall-clock hours — IMU-1/NorMuon and partial-RoPE arms + +**Value** + +``` +IMU-1 (imu1_2tpp): ~63.9 h DERIVED (no 'Training complete' line; final logged rate 5,172 tok/s, 1,189,478,400/5172 = 229,984 s). partial-RoPE 0.25: ~46.1 h DERIVED (final rate 7,168 tok/s). partial-RoPE 0.10: ~14 h before it stopped at step 5450 (7,096-7,104 tok/s). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log:114-115` + +**Source quote** + +``` +[09:58:25] step 18150/18150 ce 2.4688 z 0.00659 lr 0.00e+00 |grad| 0.06 mem 66.1GB tok/s 5,172 +[09:58:25] DONE +``` + +**Confidence** — measured from code + +**Caveat** — DERIVED, not logged: train_imu1.py and train_partialrope.py write no wall-clock summary line. The IMU-1 derivation is corroborated by checkpoint mtimes (step2000 Jun 12 02:10 -> step18000 Jun 14 10:27 = 56.28 h for 16,000 steps = 5,175 tok/s). NorMuon is ~31% slower per token than AdamW at the same shape. + +**Verdict — ❌ WRONG** + +**Corrected value** + +``` +Hours are right (IMU-1 ~63.9 h, pRoPE-0.25 ~46.1 h, pRoPE-0.10 ~14 h) but TWO errors must be fixed: (a) the evidence citation is qwen3_imu1_2tpp_train.log:385-386, NOT :114-115; (b) NorMuon is 30.5% LOWER THROUGHPUT, which is +43.9% wall-clock / +43.9% time per token — not '~31% slower per token'. +``` + +**Verifier note** + +``` +CITATION REFUTED: the file is 386 lines. Lines 114-115 are '[12:05:16] step 5100/18150 ... tok/s 5,160' and '[12:15:44] step 5150/18150 ... tok/s 5,160'. The quoted 'step 18150/18150 ... tok/s 5,172' + 'DONE' pair is at lines 385-386 (verified by grep -n). ARITHMETIC REFUTED: 5,172/7,444 = 0.6948, i.e. 30.5% lower throughput; the reciprocal is 1.439, so per-token time and wall-clock are 43.9% HIGHER. The fact's own derived hours prove it: 63.9 h / 44.4 h = 1.439. Derivations themselves check out: 1,189,478,400/5172 = 229,984 s = 63.88 h; checkpoint mtimes step2000 2026-06-12 02:10 -> step18000 2026-06-14 10:27 = 56.28 h for 16,000 steps = 5,175 tok/s. pRoPE-0.25 log clock 09:58:33 -> 08:05:04 next-next day = 46.11 h, final rate 7,168 (verified in tail). pRoPE-0.10 08:05:09 -> 22:04:32 same day = 13.99 h, rates 7,096-7,104 (verified in tail). +``` + + +### 6.5 GPU-hours / cost recorded in the ledger for these runs + +**Value** + +``` +NOT_FOUND — the four Phase-B training runs have NO ledger run entries at all. Only their downstream EVAL runs are in the ledger, and those carry cost.wall_clock_min=null, cost.gpu_hours=null. +``` + +**Evidence** — `research/ledger/ledger.json:451` + +**Source quote** + +``` +"artifact_sha256": "checkpoint_qwen3_baseline2tpp.pt@step18150" +``` + +**Confidence** — NOT FOUND + +**Caveat** — I enumerated all 29 ledger run_ids; the earliest is 2026-06-16_qwen3-faithful_eval-first (type=eval). The training runs themselves (Jun 9-16) predate ledger adoption. The only trace of baseline2tpp in ledger.json is the lineage artifact string above. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Independently re-enumerated all 29 ledger runs via ledger.json: earliest is 2026-06-16_qwen3-faithful_eval-first (type=eval), confirming the Phase-B trainers predate ledger adoption. grep for baseline2tpp|imu1_2tpp|prope25_2tpp|prope10_2tpp across ledger.json returns exactly ONE hit — research/ledger/ledger.json:451 '"artifact_sha256": "checkpoint_qwen3_baseline2tpp.pt@step18150"' — verbatim as quoted. All five 2026-06-16 eval entries carry cost.wall_clock_min=null and cost.gpu_hours=null. (Side note, not a refutation: 3 unrelated later runs DO carry costs — 2026-06-30_qwen3-0.6b_midtrain-anneal 2220 min / 36.7 gpu_h; 2026-07-19_hybrid-ssm-0.2b_pretrain-ssm-base-s0 990 / 16.5. So the null is specific to these runs, not a repo-wide absence.) +``` + + +### 6.6 Sequence length, micro-batch, grad-accum, global batch (sequences and tokens) + +**Value** + +``` +seq_len 4096; micro_batch 4; grad_accum 4; global batch = 16 sequences = 65,536 tokens/step. Identical across all four Phase-B arms. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:234-236` + +**Source quote** + +``` +ap.add_argument("--seq_len", type=int, default=4096) + ap.add_argument("--micro_batch", type=int, default=4, help="DO NOT raise; >=8 OOMs at seq 4096 (probe-verified)") + ap.add_argument("--grad_accum", type=int, default=4, help="effective batch = micro_batch * grad_accum seqs") +``` + +**Confidence** — measured from code + +**Caveat** — Confirmed in the run log itself: 'tok/step=65,536 steps=18,150 token_budget=1,189,478,400' (qwen3_baseline2tpp_train.log:3). train_imu1.py:93-95 and train_partialrope.py:37-39 carry the same 4096/4/4 defaults. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_qwen3.py:234-236 quote is verbatim at exactly those line numbers. Cross-arm identity independently verified: train_imu1.py:93 (--seq_len 4096), :94 (--micro_batch 4), :95 (--grad_accum 4); train_partialrope.py:37/:38/:39 same. Log line 3 verbatim '[21:14:18] tok/step=65,536 steps=18,150 token_budget=1,189,478,400'. Derivation in code at train_qwen3.py:275 'tok_per_step = args.seq_len * args.micro_batch * args.grad_accum'. +``` + + +### 6.7 Precision: bf16/fp16/fp32, autocast, master weights + +**Value** + +``` +FULL bf16 — model weights cast to torch.bfloat16 at construction; there is NO torch.autocast, NO GradScaler, and NO fp32 master-weight copy in any of the three trainers. AdamW moment buffers are therefore allocated in the parameter dtype (bf16). Only cross-entropy is upcast: each 8192-row chunk is .float()'d into an fp32 accumulator. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:293` + +**Source quote** + +``` +model = Qwen3ForCausalLM(cfg).to(device=device, dtype=dtype) +``` + +**Confidence** — measured from code + +**Caveat** — Verified by grep: 'autocast|GradScaler|master' returns zero hits in train_qwen3.py, train_imu1.py, train_partialrope.py. The fp32 CE path is train_qwen3.py:87 ('total = flat.new_zeros((), dtype=torch.float32)') and :91 ('flat[i:i + chunk].float()'). NOTE train_imu1.py:38-45 does NOT .float() its CE chunks (only its z-loss does), so the IMU-1 arm's CE is accumulated in bf16 — a real, unremarked difference between arms. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +All file-level claims CONFIRMED. Qualify one clause: 'AdamW moment buffers are allocated in bf16' is an INFERENCE from PyTorch's zeros_like(p) semantics — no file on disk records optimizer-state dtype. State it as inferred, not measured. +``` + +**Verifier note** + +``` +train_qwen3.py:293 verbatim. I re-ran the grep for 'autocast|GradScaler|master' across all three trainers: exit 1 (zero hits) on each — confirmed. fp32 CE path verified at train_qwen3.py:87 ('total = flat.new_zeros((), dtype=torch.float32)') and :91 ('flat[i:i + chunk].float()'). The IMU-1 asymmetry is REAL and correctly reported: train_imu1.py:41 initialises 'total, n = 0.0, flat_tgt.numel()' and :43-44 calls cross_entropy on flat_logits WITHOUT .float(), while chunked_z_loss:54 does use '.float()'. This is a genuine unremarked between-arm numerical difference and belongs on the card. +``` + + +### 6.8 Optimizer / LR / betas / eps / weight decay / warmup / schedule / grad clip — faithful baseline + +**Value** + +``` +AdamW, peak_lr 2.4e-3 (Phase-A winner; script default is 1.7e-3), end_lr 3.2e-4, betas (0.9, 0.95), eps 1e-8, weight_decay 0.01 applied ONLY to params with dim>=2 (dim<2 gets 0.0), warmup 900 steps linear, then cosine decay from peak to end_lr (floor = end_lr/peak_lr = 0.1333), grad_clip 1.0 (global L2 via clip_grad_norm_). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:296-304` + +**Source quote** + +``` +decay, no_decay = [], [] + for _, p in model.named_parameters(): + (no_decay if p.dim() < 2 else decay).append(p) + optim = AdamW( + [{"params": decay, "weight_decay": args.weight_decay}, + {"params": no_decay, "weight_decay": 0.0}], + lr=args.peak_lr, betas=(0.9, 0.95), eps=1e-8, + ) + sched = make_cosine_scheduler(optim, args.warmup_steps, args.steps, args.peak_lr, args.end_lr) +``` + +**Confidence** — measured from code + +**Caveat** — The run's own resolved args are echoed at qwen3_baseline2tpp_train.log:2: "'peak_lr': 0.0024, 'end_lr': 0.00032, 'warmup_steps': 900, 'weight_decay': 0.01, 'grad_clip': 1.0". Cosine shape at train_qwen3.py:96-107: floor + 0.5*(1-floor)*(1+cos(pi*prog)). The file's own docstring (line 6) still claims 'peak 1.7e-3' — stale vs the actual 1.19B run, which used 2.4e-3. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_qwen3.py:296-304 quote is verbatim at exactly those lines. Script default peak_lr=1.7e-3 confirmed at :237; driver overrides to 2.4e-3 at phase_b_driver.sh:19. Resolved args echoed verbatim at log line 2. Cosine shape verified at :96-107 ('floor = end_lr / peak_lr'; 'floor + 0.5 * (1.0 - floor) * (1.0 + math.cos(math.pi * prog))'); 3.2e-4/2.4e-3 = 0.13333. grad_clip via torch.nn.utils.clip_grad_norm_ confirmed at :391. The stale-docstring catch is real: train_qwen3.py:6 still reads 'cosine schedule, peak 1.7e-3 -> end 3.2e-4'. +``` + + +### 6.9 Optimizer for the IMU-1 / NorMuon 1.19B arm (the arm that won) + +**Value** + +``` +Hybrid split: 2D non-embedding matrices (224 params) -> NorMuon(lr=0.011, weight_decay=0.1, beta1=0.95, beta2=0.95); embeddings/norms/1D (198 params) -> AdamW(lr=0.006, betas=(0.9,0.95), eps=1e-8, weight_decay=0.0). Schedule = WSD (linear warmup 900, stable, then linear decay-to-ZERO over the final 20%), not cosine. grad_clip 1.0. Extra chunked z-loss weighted 1e-4 added to CE. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/train_imu1.py:82-87` + +**Source quote** + +``` +def build_optimizers(model, normuon_lr, adam_lr, wd): + n_params, a_params = split_params(model) + opt_n = NorMuon(n_params, lr=normuon_lr, weight_decay=wd, beta1=0.95, beta2=0.95) + opt_a = torch.optim.AdamW(a_params, lr=adam_lr, betas=(0.9, 0.95), eps=1e-8, weight_decay=0.0) +``` + +**Confidence** — measured from code + +**Caveat** — CONFOUND for any 'IMU-1 wins' claim: this arm changes optimizer AND schedule shape (WSD vs cosine) AND adds z-loss AND changes the model (value residuals + LayerNorm scaling + head gating, train_imu1.py:1-8) AND weight_decay 0.1 vs 0.01 — five variables at once, violating the repo's own one-variable rule. Param split (224/198) measured at qwen3_imu1_2tpp_train.log:2. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_imu1.py:82-85 quote verbatim at those lines. Defaults confirmed: :96 normuon_lr 0.011, :97 adam_lr 0.006, :98 weight_decay 0.1 ('# 2D only (NorMuon)'), :101 z_weight 1e-4, :102 grad_clip 1.0. WSD implementation at :59-66 (linear warmup, stable, 'max(0.0, (total - step) / max(1.0, total - decay_start))' with decay_frac 0.2). Param split 224/198 verbatim at qwen3_imu1_2tpp_train.log:2. The 5-variable-confound caveat is correct and load-bearing. ADD A SECOND CAVEAT the fact omits: the NorMuon advantage was later NULLED — ledger run 2026-07-05_qwen3-0.6b_scaling-persistence carries verdict 'null' and metrics.conclusion 'The NorMuon advantage CONVERGES toward 0 with budget — an early-training speedup'. Calling this 'the arm that won' without that is misleading on a public card. +``` + + +### 6.10 Total steps, total tokens, seeds + +**Value** + +``` +18,150 steps; 1,189,478,400 tokens (18150 x 65,536); seed = 0 for ALL arms. n=1 per arm — no seed replicates exist for any 1.19B run. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log:1-3` + +**Source quote** + +``` +[21:14:18] device=cuda dtype=torch.bfloat16 seed=0 +[21:14:18] args={'steps': 18150, 'seq_len': 4096, 'micro_batch': 4, 'grad_accum': 4, 'peak_lr': 0.0024, 'end_lr': 0.00032, 'warmup_steps': 900, 'weight_decay': 0.01, 'grad_clip': 1.0, 'mem_fraction': 0.85, 'seed': 0, 'dtype': 'bfloat16', ...} +[21:14:18] tok/step=65,536 steps=18,150 token_budget=1,189,478,400 +``` + +**Confidence** — measured from code + +**Caveat** — Seed 0 is the argparse default in all three trainers (train_qwen3.py:243, train_imu1.py:104, train_partialrope.py:47) and phase_b_driver.sh never passes --seed. Every 1.19B headline number is therefore n=1, single seed, no CI. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Log lines 1-3 quoted verbatim and verified. Seed defaults confirmed at train_qwen3.py:243, train_imu1.py:104, train_partialrope.py:47 (all '--seed', default=0); phase_b_driver.sh contains no '--seed' anywhere (read the full 37-line file). The 'n=1, single seed, no CI' framing is the honest one and must survive to the card. +``` + + +### 6.11 The ACTUAL training corpus behind the 1.19B tokens — HF dataset id, config, split + +**Value** + +``` +HuggingFaceFW/fineweb-edu, config 'sample-10BT', split 'train', streaming=True. Streamed once by the faithful run and cached to results/tokcache_1191478400_300000.pt; the other three arms LOADED THAT SAME CACHE (identical corpus, identical stream order). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:151` + +**Source quote** + +``` +ds = load_dataset("HuggingFaceFW/fineweb-edu", "sample-10BT", split="train", streaming=True) +``` + +**Confidence** — measured from code + +**Caveat** — No revision/sha is pinned — load_dataset is called without a revision arg, so the exact snapshot is unrecoverable. Cache reuse proven at qwen3_imu1_2tpp_train.log:3, qwen3_prope25_2tpp_train.log:2, qwen3_prope10_2tpp_train.log:2: 'loaded cached tokens from tokcache_1191478400_300000.pt (1,191,478,400 train + 300,000 val)'. There is NO prepare_*.py and NO dataset card under research/datasets/ for this corpus — that directory holds only dclm-edu, openr1-math-220k, math-eval-v1, grpo-math-prompts-v1 and hybridssm-fineweb-edu, none of which fed the 1.19B Qwen3 runs. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_qwen3.py:151 verbatim at that exact line. CRITICALLY, I also confirmed the load_dataset line is UNCHANGED in the pre-86e79f3 version that actually ran (it appears as unmodified context in `git diff e791875 86e79f3`), so the dataset id/config/split claim holds for the runs, not just for today's file. No revision= arg — confirmed by reading the call. Cache reuse verified verbatim at qwen3_imu1_2tpp_train.log:3, qwen3_prope25_2tpp_train.log:2, qwen3_prope10_2tpp_train.log:2. Cache file exists on disk: results/tokcache_1191478400_300000.pt, 9,534,229,373 bytes, mtime 2026-06-09 22:41. research/datasets/ listing verified: exactly data-selection-dclm-edu, grpo-math-prompts-v1, hybridssm-fineweb-edu, math-eval-v1, math-reasoning-openr1-math-220k — none is this corpus. +``` + + +### 6.12 How the 1.19B corpus was tokenized and packed + +**Value** + +``` +Tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen3-0.6B-Base') (vocab 151,936). Each doc encoded with add_special_tokens=False, one EOS id appended, all ids concatenated into one flat stream. Packing = contiguous non-overlapping 4096-token windows (window i = tokens[i*4096 : i*4096+4097], input=[:-1], label=[1:]). No cross-document attention masking — documents bleed across window boundaries. DataLoader shuffles WINDOW order (shuffle=True, drop_last=True). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:110-123` + +**Source quote** + +``` +class PackedTextDataset(torch.utils.data.Dataset): + """Pre-tokenized contiguous windows of seq_len (mirrors SmolLM2 train.py).""" + def __init__(self, token_ids: torch.Tensor, seq_len: int): + self.tokens = token_ids + self.seq_len = seq_len + self.n_windows = (len(token_ids) - 1) // seq_len +``` + +**Confidence** — measured from code + +**Caveat** — 290,888 windows were available and 18150*16 = 290,400 were consumed = 99.83% of exactly one epoch (single-pass, effectively no repetition). Log line 7: '290,888 train windows of 4096'. REPO id at train_qwen3.py:59, tokenizer load at :281, doc+EOS at :166. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +train_qwen3.py:110-123 quote verbatim at exactly those lines. REPO='Qwen/Qwen3-0.6B-Base' at :59, tokenizer load at :281, doc+EOS at :166, DataLoader(shuffle=True, drop_last=True) at :287. Epoch arithmetic checks: log line 7 '290,888 train windows of 4096'; 18150 steps x 16 seqs = 290,400 = 99.83% of one epoch, so effectively single-pass. The old (actually-executed) code did the same encode: `git show e791875` line 141 '(buf if len(buf) < n_train else val).extend(ids + [eos])' after `tokenizer.encode(text, add_special_tokens=False)`. +``` + + +### 6.13 Was the train/val split for the 1.19B runs document-disjoint / decontaminated? + +**Value** + +``` +NO. The 1.19B runs used the OLD stream_tokens, which cut the stream by token count so val was the sequential continuation of train. The document-disjoint + 13-gram-decontam version now on disk landed in commit 86e79f3 (2026-06-16), AFTER all four Phase-B runs finished. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:131-136` + +**Source quote** + +``` +* each whole document is routed to train or val by a seeded hash + (`is_val_doc`), so train/val never share a document and no document spans + the boundary (the old code cut the stream by token count — val was the + sequential continuation of train, leak-suspect); +``` + +**Confidence** — measured from code + +**Caveat** — Proof of ordering: (a) `git diff e791875 86e79f3 -- .../train_qwen3.py` shows the old body was `(buf if len(buf) < n_train else val).extend(ids + [eos])` with cache key `tokcache_{n_train}_{n_val}.pt`; (b) the cache the runs loaded is named `tokcache_1191478400_300000.pt` — the OLD key, no seed/tokenizer tag; (c) baseline2tpp log line 6 has no decontam sentence, unlike the current code's log call. So the 28.65 / 23.52 / 29.54 val-PPL headlines rest on a leak-suspect split by the repo's own later admission. The file you read today is NOT the file that produced those numbers. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +This is the strongest fact in the set and I could not break it. train_qwen3.py:131-136 quote verbatim. `git diff e791875 86e79f3` shows the removed lines '- cache = RESULTS / f"tokcache_{n_train}_{n_val}.pt"' and '- (buf if len(buf) < n_train else val).extend(ids + [eos])', replaced by the seeded is_val_doc routing + decontaminate_val + the seed/tokenizer-tagged cache key at :145. The cache the runs loaded is literally named tokcache_1191478400_300000.pt (old key, no seed tag) — on disk, verified. Commit timestamp 86e79f3 = 2026-06-16 21:57:33 +0000, after prope10 stopped. Corroborating detail the fact could have added: results/decontam_report.json DOES exist but its mtime is 2026-07-07 15:13, matching tokcache_422020224_300000_seed0_Qwen3-0.6B-Base.pt (same mtime) — i.e. it documents a LATER cache, not the 1.19B one, exactly as the fact asserts. +``` + + +### 6.14 Throughput (tokens/sec) for the 1.19B runs + +**Value** + +``` +faithful baseline 7,444-7,481 tok/s sustained (final 7,444); IMU-1/NorMuon 5,172 tok/s; partial-RoPE 0.25 7,168 tok/s; partial-RoPE 0.10 ~7,100 tok/s. Peak memory: 52.4 GB (faithful), 66.1 GB (IMU-1), 54.3 GB (partial-RoPE). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log:394` + +**Source quote** + +``` +[18:05:01] step 18150/18150 loss 3.2811 lr 3.20e-04 |grad| 0.13 mem 52.4GB tok/s 7,444 tok 1189.5M ETA 0.0min +``` + +**Confidence** — measured from code + +**Caveat** — These are CUMULATIVE averages (tok_seen / elapsed since t0), not instantaneous rates — train_qwen3.py:404-405 'tps = tok_seen / max(1e-9, dt)'. An independent standalone probe measured 7,167.4 tok/s compiled vs 3,787.5 uncompiled at the same shape (throughput_probe.json:26 and :12), i.e. torch.compile gives 1.89x. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +faithful cumulative tok/s spans 7,414–7,483 (max at log line 41, step 1450), ending 7,444 — the stated 7,444–7,481 band is wrong at both ends. All other figures confirmed. +``` + +**Verifier note** + +``` +Log line 394 quote verbatim (mem 52.4GB, tok/s 7,444). IMU-1 5,172 + mem 66.1GB confirmed at log:385; pRoPE-0.25 7,168 + 54.3GB at its tail; pRoPE-0.10 7,096-7,104 + 54.3GB at its tail. Cumulative-not-instantaneous caveat CONFIRMED verbatim at train_qwen3.py:404-405 ('dt = time.time() - t0' / 'tps = tok_seen / max(1e-9, dt)'). Probe figures confirmed: throughput_probe.json:12 = 3787.5 (uncompiled), :26 = 7167.4 (compiled), :34 '"speedup_compile_vs_baseline": 1.89'. +``` + + +### 6.15 MFU for the 1.19B runs + +**Value** + +``` +NOT_FOUND. No MFU number was ever computed or stored for these runs. mfu_meter.py exists but has no JSON output anywhere in the repo (find '*mfu*' returns only the module, its pycache, and its unit test). No README claims an MFU for the three builds. +``` + +**Evidence** — `mfu_meter.py:63-66` + +**Source quote** + +``` +"gb10": {"bf16_dense_tflops": 125.0, "estimated": True, + "GB10; BF16-dense derived (~FP4/8) — treat MFU as approximate"}, +``` + +**Confidence** — NOT FOUND + +**Caveat** — Even if computed post-hoc, the GB10 peak entry is estimated=True (NVIDIA publishes only a sparse-FP4 ~1 PFLOP figure), so per CLAUDE.md:19 a GB10 MFU must never be quoted as exact. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Scoped claim survives: no MFU exists for the four 1.19B Phase-B runs. But 'no JSON output anywhere in the repo' is FALSE, and the companion gap line 'MFU was never computed for any Qwen3 run' is FLATLY WRONG. +``` + +**Verifier note** + +``` +mfu_meter.py:63-66 quote is verbatim. But the find '*mfu*' filename search was the wrong instrument — MFU output is embedded inside other JSONs. `grep -rln '\"mfu\"' --include=*.json` returns TWO real hits: (1) research/ledger/ledger.json:503 '\"mfu\": 0.2909' inside run 2026-06-16_qwen3_normuon-vs-adamw, alongside mfu_normuon 0.2907, achieved_tflops 36.36, device_peak_tflops 125.0, peak_is_estimated true; (2) Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/c5_evidence.json:11-14, a full block with mfu 0.3209, hfu 0.3209, achieved_tflops 40.11, tokens_per_sec 7344, formula 'MFU = (6N + 12*L*H*Q*T) * tok/s / peak_bf16_dense'. Neither is a 1.19B run, so the narrow claim holds — but the card must not say the repo has never computed MFU. The estimated=True caveat is correct and mandatory (CLAUDE.md:19 verified). +``` + + +### 6.16 How was the reported val PPL (28.65 etc.) measured? + +**Value** + +``` +In-trainer evaluate(): non-overlapping windows of 4096 tokens (stride = seq_len = 4096), capped at max_windows=50 => 204,800 target tokens, chunked fp32 CE, exp(mean NLL), on the FineWeb-Edu val slice with the model's own Qwen3 tokenizer. NOT the eval-harness. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:194-207` + +**Source quote** + +``` +def evaluate(model, val_tokens, device, seq_len: int, max_windows: int = 50): + for begin in range(0, min(len(val_tokens) - seq_len, max_windows * seq_len), seq_len): + ids = val_tokens[begin:begin + seq_len + 1].unsqueeze(0).to(device) + logits = model(ids[:, :-1])["logits"] +``` + +**Confidence** — measured from code + +**Caveat** — Log line 342-equivalent confirms the count: 'baseline val PPL=185810.49 (204,800 tokens)'. These in-trainer PPLs are not cross-run-comparable under the repo's own §C10; the comparable numbers are the separate text-lm-v2 eval-harness ledger entries: faithful wikitext2 ppl 37.01 / bpb 1.2256, modernized 27.8, prope25 38.08, prope10 69.63 — the last from a step-4000 checkpoint of the incomplete run (ledger note: 'run stopped early at ~step 5400 (undertrained vs 18150-step peers)'). + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Substance confirmed, but three fixes: (a) 'def evaluate' is at train_qwen3.py:195, not 194; (b) the 204,800-token line is qwen3_baseline2tpp_train.log:11, not 'line 342-equivalent'; (c) the modernized 27.8 / 23.52 were scored on checkpoint_imu1_2tpp_step18000.pt — step 18,000 (1.180B tok), NOT the step-18150 endpoint, so it is 0.8% short of iso-token vs the faithful number. +``` + +**Verifier note** + +``` +evaluate() body verified: :199 'for begin in range(0, min(len(val_tokens) - seq_len, max_windows * seq_len), seq_len)' — stride == seq_len, 50*4096 = 204,800. fp32 CE via chunked_cross_entropy:87/:91. Log line 11 verbatim: '[21:41:59] baseline val PPL=185810.49 (204,800 tokens) — expect ~vocab_size (151,936) at init' (grep confirms it is the ONLY '(204,800 tokens)' line in the file). All four ledger eval numbers verified exactly: 37.0101/1.2256, 27.8, 38.08, 69.63, all suite_version 'text-lm-v2'. prope10 note verbatim: 'step-4000 checkpoint; run stopped early at ~step 5400 (undertrained vs 18150-step peers)'. Checkpoint (c) verified at Qwen3-0.6B/experiments/2026-06-16_qwen3-0.6b_eval-modernized/eval_suite.py:55; no step-18150 IMU-1 checkpoint exists on disk (last '[ckpt @ 18000]' at imu1 log:382). Dataset-config check (the flagged failure mode) PASSES: eval_suite.py:162-163 says load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="validation", revision=WIKITEXT_REV) — the '-raw-' variant is genuinely what the code names. +``` + + +### 6.17 Was the 1.19B-token budget iso-FLOP across arms? + +**Value** + +``` +Iso-TOKEN, not iso-FLOP or iso-wall-clock. All four arms ran the same 18,150 steps / 1,189,478,400 tokens, but IMU-1 took ~31% more wall-clock (5,172 vs 7,444 tok/s) and adds a z-loss plus three architecture changes. +``` + +**Evidence** — `Qwen3-0.6B/README.md:80` + +**Source quote** + +``` +matched **tokens** (1.19B); IMU-1 also ran NorMuon (~30% more wall-clock, uncounted by the 6ND +``` + +**Confidence** — PROSE ONLY + +**Caveat** — The prose claim matches the tok/s I read in the logs (7,444 vs 5,172 = 30.5% slower), so it is corroborated by measurement — but no train_flops artifact exists for these runs, so the §C18 '<=5% train_flops match' gate was never actually evaluated on disk. + +**Verdict — ❌ WRONG** + +**Corrected value** + +``` +IMU-1 took ~44% more wall-clock (~63.9 h vs 44.4 h), not ~31%. 30.5% is the tok/s DROP; the wall-clock inflation is its reciprocal, 1/0.695 = 1.439. +``` + +**Verifier note** + +``` +Qwen3-0.6B/README.md:80 quote is verbatim, but the fact then endorses the prose as 'corroborated by measurement' — it is not. 7,444 vs 5,172 tok/s is a 30.5% throughput reduction, which is +43.9% wall-clock, and the fact set's OWN derived hours (63.9 h vs 44.4 h = 1.439x) prove it. The repo's README:80 prose ('~30% more wall-clock') is itself the original error and must not be copied onto a card. Two further points: (a) the iso-TOKEN-not-iso-FLOP core claim is CONFIRMED — grep for 'train_flops' across Qwen3-0.6B/builds/ returns zero files, so the §C18 <=5% gate was never evaluated for these four runs; (b) the fact omits README.md:81, which does assert 'params are iso-FLOP at 1.00043' — a parameter-count-based claim, not a train_flops artifact, and it should be characterised as such rather than ignored. +``` + + +### 6.18 Chinchilla ratio / degree of under-training + +**Value** + +``` +~2 tokens per parameter (1.19B tokens / 596M params). The repo states Chinchilla-optimal for 596M is ~12B tokens, so Phase B is ~10x under-trained, and ~30,000x less data than the real Qwen3 run (36T). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md:20` + +**Source quote** + +``` +- **The unavoidable deviation = the token budget.** The paper used ~36T tokens; we use 131M (Phase A) to 1.19B (Phase B). ... Chinchilla-optimal for 596M is ~12B tokens (20 tok/param); even Phase B is ~10× under-trained. +``` + +**Confidence** — PROSE ONLY + +**Caveat** — '36T tokens' is COPIED from the Qwen3 paper; the 'original PPL 13.40' comparator is a measured eval of the released HF model, not a run this repo trained. The 596M param count is measured (throughput_probe.json:3 'model_params': '596M'). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +faithful README.md:20 quote verbatim, including '~36T tokens', 'we use 131M (Phase A) to 1.19B (Phase B)' and 'Chinchilla-optimal for 596M is ~12B tokens (20 tok/param); even Phase B is ~10x under-trained'. The '~30,000x' is DERIVED arithmetic (36T/1.19B = 30,252) but is independently corroborated in prose at research/brutal_scorecard.md:57 and research/brutal_scorecard_core.md:35 ('~30,000x under-trained'). The COPIED-vs-MEASURED split is right: 36T is from the Qwen3 report; 13.40 is genuinely MEASURED on this box — Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/original_vs_repro.txt:2 'ORIGINAL Qwen3-0.6B-Base (36T tok) val PPL = 13.400 (204,800 tok, 21s)'. 596M confirmed at throughput_probe.json:3. (The '275,000x' at Qwen3-0.6B/README.md:74 is the Phase-A comparison, 36T/131M — do not conflate the two on the card.) +``` + + +### 6.19 The other Qwen3 from-scratch family: the scaling-persistence ladder + +**Value** + +``` +Separate runs at the SAME 596M params but SMALLER token budgets: 2,564 steps = 168M tokens (3 seeds/arm) and 6,409 steps = 420M tokens (2-3 seeds/arm), arms = adamw vs normuon, same 4096/4/4 = 65,536 tok/step, bf16, peak_lr 2.4e-3 (AdamW) / normuon_lr 0.011, wd 0.1 on 2D for BOTH arms, warmup 50, cosine to 10% of peak, grad_clip 1.0, fixed data split seed 0 across all cells. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/train_ablation.py:51-52` + +**Source quote** + +``` +SEQ_LEN, MICRO_BATCH, GRAD_ACCUM = 4096, 4, 4 +WARMUP, END_RATIO, SPLIT_SEED = 50, 0.1, 0 # fixed data split across cells +``` + +**Confidence** — measured from code + +**Caveat** — Cell list at Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder.sh:49-53 ('2564 adamw 0' ... '6409 normuon 1'). The '168M'/'420M' in the run tags are TOKEN budgets (bM=steps*65536/1e6), NOT param counts — easy to misread. This ladder DOES carry 3 seeds/arm and per-cell sentinel logs; the 1.19B three-build runs do not. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Hyperparameters all confirmed, but the seed count must be stated precisely (420M is n=3 per arm FINAL, via a separate extension script) and the ladder's TERMINAL VERDICT — 'null', NorMuon's advantage CONVERGES to 0 with budget — is a material missing caveat. +``` + +**Verifier note** + +``` +train_ablation.py:51-52 quote verbatim. Defaults verified at :85 (--peak_lr 2.4e-3), :86 (--normuon_lr 0.011), :87 (--weight_decay 0.1 '# on 2D, BOTH arms (held equal)'), :88 (--grad_clip 1.0); END_RATIO 0.1 and WARMUP 50 at :52; cosine at :56-60. 2564*65536 = 168,034,304; 6409*65536 = 420,020,224 — the 'token budget not param count' warning is correct (run_ladder.sh:57 'budgetM=$(( steps * 65536 / 1000000 ))'). SEED PRECISION: run_ladder.sh:49 states '168M=2564 steps (3 seeds/arm), 420M=6409 steps (2 seeds/arm)' and CELLS at :50-53 list only s0/s1 at 6409; the third 420M seed came from run_ladder_scale_ext.sh:593-594 (CELLS=("6409 adamw 2" "6409 normuon 2")) under ledger run 2026-07-23_qwen3-0.6b_normuon-at-scale. Final ledger metrics for 2026-07-05_qwen3-0.6b_scaling-persistence: top_budget_seeds [3,3], trend_verdict_wikitext 'CONVERGES', significance_verdict 'null', c25_complete false, c25_missing_hard [log_rmse_r2, holdout_extrapolation_pctdev, bootstrap_forecast_ci]. Any card mentioning NorMuon must carry this. +``` + + +### 6.20 SmolLM2 — is there a from-scratch or continued-pretrain config/run? + +**Value** + +``` +Both exist, but neither is a real pretrain. (a) FROM-SCRATCH: train.py on Salesforce/wikitext config 'wikitext-103-raw-v1' split 'train', seq_len 2048, micro 2 x accum 8 = 16 seqs = 32,768 tok/step, AdamW betas (0.9,0.95) eps 1e-8, peak lr 3.0e-3, wd 0.01 (dim>=2 only), WSD warmup 20 / 20% linear decay-to-zero, grad_clip 1.0, bf16 — the only recorded run is a 150-step DEMO (~4.9M tokens). (b) CONTINUED-PRETRAIN: train_tinystories.py from official HF SmolLM2-135M weights. +``` + +**Evidence** — `SmolLM2-134(base)/train.py:101-116` + +**Source quote** + +``` +ap.add_argument("--steps", type=int, default=200, help="total optimizer steps") + ap.add_argument("--seq_len", type=int, default=2048) + ap.add_argument("--batch_size", type=int, default=2, help="micro-batch (per accum step)") + ap.add_argument("--grad_accum", type=int, default=8) + ap.add_argument("--lr", type=float, default=3.0e-3, help="peak LR; paper §6") +``` + +**Confidence** — measured from code + +**Caveat** — Corpus at train.py:78 ('load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train")'); betas/eps at train.py:152. The from-scratch run's scale is recorded ONLY as 'Demo-run steps': '150 (warmup 20, decay 20%)' and 'Demo-run final loss': '6.288 (start 11.254, baseline ln(V)=10.803)' in SmolLM2-134(base)/results/summary.json. train.py:13-19 calls this a single-GPU starter, not a reproduction. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +All named hyperparameters and the dataset config are CONFIRMED. Qualify the token figure: '~4.9M tokens' is DERIVED (150 x 32,768) from argparse defaults — nothing on disk records the demo run's batch shape. +``` + +**Verifier note** + +``` +train.py:101-105 quote verbatim at those lines (:101 steps default 200, :102 seq_len 2048, :103 batch_size 2, :104 grad_accum 8, :105 lr 3.0e-3). Dataset config check (the flagged failure mode) PASSES: train.py:78 is literally load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train") — the '-raw-' variant and the 103 (not 2) are exactly as claimed. betas/eps at :152, wd split at :145-151, WSD at :154, warmup 20 at :106, wd 0.01 at :108, grad_clip 1.0 at :110, bf16 at :115. Demo scale corroborated beyond summary.json:11-12: results/loss_curve.csv has 151 lines (header + steps 0-149), step 0 loss 11.254480, step 149 loss 6.288341 — matching summary.json exactly. But that CSV records only step/loss/lr, so the token count remains an inference from defaults. 'single-GPU starter, not a reproduction' verified at train.py:13-18. +``` + + +### 6.21 SmolLM2 continued-pretrain run details + +**Value** + +``` +roneneldan/TinyStories (split 'train' for training, 'validation' for eval), 102,000,116 tokens packed, seq_len 1024, micro_batch 4, grad_accum 1 => 4,096 tok/step, 24,414 steps, 100,000,000-token budget, AdamW betas (0.9,0.95) eps 1e-8 peak_lr 3e-4, wd 0.01 (dim>=2 only), grad_clip 1.0, WSD warmup 200 / 20% decay, bf16, seed 0, 116.1 min wall-clock, ~14,356 tok/s. PPL 6.895 -> 3.790. +``` + +**Evidence** — `SmolLM2-134(base)/results/tinystories_train.log:1-17` + +**Source quote** + +``` +[21:23:28] Device: cuda dtype: torch.bfloat16 +[21:23:28] Loading tokenizer + official SmolLM2-135M weights... +[21:25:09] packed 102,000,116 train tokens in 97.9s +[21:25:09] tok/step = 4,096 total_steps = 24,414 +[21:25:16] Training 24,414 steps to 100,000,000 tokens... +``` + +**Confidence** — measured from code + +**Caveat** — Starts from OFFICIAL HF weights (train_tinystories.py:2-3), so NOT a from-scratch result. Wall-clock at line 506 ('Training complete in 116.1 min.'), final PPL at line 508. Dataset ids at train_tinystories.py:154-155; hyperparams at :105-122 and :213-217. The 100M budget is ~1/4 epoch of TinyStories per the docstring. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +tinystories_train.log:1-17 quote verbatim (verified line-by-line: :1 device/bf16, :2 official weights, :5 'packed 102,000,116 train tokens in 97.9s', :8 'tok/step = 4,096 total_steps = 24,414', :17 'Training 24,414 steps to 100,000,000 tokens...'). Wall-clock at :506 'Training complete in 116.1 min.'; PPL at :508 'AFTER PPL = 3.790 (BEFORE was 6.895; improvement +3.105 = +45.0%)'; 14,356 tok/s at :505. Hyperparams verified at train_tinystories.py:105-122 and optimizer at :213-217; dataset ids at :154-155. Two nits worth carrying: (1) 'seed 0' is the argparse default (:120) — the log never echoes resolved args, so it is inferred-from-default, though 24,414 = floor(100,000,000/4,096) is consistent with defaults throughout; (2) 14,356 tok/s is a cumulative average, not instantaneous. The 'starts from official HF weights, NOT from-scratch' caveat is correct — train_tinystories.py:4 'Starts from the official HF safetensors loaded into our SmolLM2ForCausalLM' (line 4, the fact cited :2-3). +``` + + +### 6.22 SmolLM2 — is the nanotron reference config MEASURED here or COPIED from the publication? + +**Value** + +``` +COPIED, and explicitly labelled as such: global_batch_size 512, tokens_per_step 1,048,576, 2,000,000 steps, ~2.1T tokens, implied data-parallel 64 (i.e. 64 GPUs), warmup 2000, decay 400,000 steps, clip_grad 1.0, bf16. None of this was run on this box. +``` + +**Evidence** — `SmolLM2-134(base)/results/training_recipe_resolved.json:2-3` + +**Source quote** + +``` +"source": "https://github.com/huggingface/smollm/blob/main/text/pretraining/smollm2/config_smollm2_135M.yaml", + "fetched": "2026-05-13", +``` + +**Confidence** — measured from code + +**Caveat** — This file is the clean case where COPIED vs MEASURED is unambiguous — it names its source URL and fetch date. It also records a real bug fix in its notes: 'Previous train.py default weight_decay=0.1 was 10× too high — corrected to 0.01'. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Every value CONFIRMED verbatim, but two labelling fixes: (a) the fact's confidence tag says 'measured-from-code' while the value itself says COPIED — for a model card it must be tagged COPIED/external; (b) 'implied data-parallel 64 (i.e. 64 GPUs)' — the JSON key is literally 'implied_data_parallel', a DERIVED quantity (512 / (8 x 1)); the file never says 'GPUs'. +``` + +**Verifier note** + +``` +training_recipe_resolved.json read in full (40 lines). Verified: :2 source URL (huggingface/smollm .../config_smollm2_135M.yaml), :3 '"fetched": "2026-05-13"', :15 warmup_steps 2000, :17 decay_steps 400000, :22 sequence_length 2048, :23 micro_batch_size 8, :24 batch_accumulation_per_replica 1, :25 implied_data_parallel 64, :26 global_batch_size 512, :27 tokens_per_step 1048576, :30 total_steps 2000000, :31 total_tokens_approx 2097152000000 (= 2.097T, i.e. '~2.1T'), :32 clip_grad 1.0, :33 precision bf16. The bug-fix note is verbatim at :38: 'Previous train.py default weight_decay=0.1 was 10x too high — corrected to 0.01'. This file is indeed the cleanest COPIED-vs-MEASURED case in the repo. +``` + + +### 6.G Gaps — not determinable from disk + +- GPU COUNT is not measured anywhere on disk. No file records torch.cuda.device_count(); only get_device_name(0) and safe_cuda.guard(device=0). 'Single GPU' rests on CLAUDE.md prose (lines 3, 9) and contract §C4.5. +- MFU was never computed for any Qwen3 run. mfu_meter.py has no output artifact anywhere in the repo, and no README quotes an MFU for the three builds. The GB10 peak in mfu_meter.py is estimated=True anyway. +- No train_flops / FLOP-accounting artifact exists for the 1.19B runs, so the §C18 iso-FLOP (<=5%) gate was never evaluated on disk for the three-build comparison — it is iso-TOKEN only, and the arms differ by ~31% in wall-clock. +- The Phase-B training runs have NO ledger entries (only their downstream evals do), so there is no ledger-recorded wall_clock_min, gpu_hours, git_commit, c5_evidence.json, or verdict.json for them. They predate ledger adoption. +- No HF dataset revision/sha is pinned for HuggingFaceFW/fineweb-edu sample-10BT — load_dataset is called with no revision arg, so the exact corpus snapshot behind the 1.19B tokens is unrecoverable. +- Wall-clock for the IMU-1 and partial-RoPE arms is NOT logged (those trainers write no completion line); I could only derive it from logged cumulative tok/s and corroborate against checkpoint mtimes. +- train_qwen3.py on disk today is NOT the version that produced the 1.19B numbers — the document-disjoint/decontam split landed in commit 86e79f3 after the runs. The pre-86e79f3 behaviour is recoverable from git, but no decontam_report.json exists for the tokcache_1191478400_300000.pt cache those runs actually consumed. +- No smoke-test artifact or c5_evidence.json exists for the Phase-B launches (the §C5 contract postdates them). +- I did not open the .pt checkpoints, which embed a 'training_recipe' dict written by save_ckpt (train_qwen3.py:358-367). That in-checkpoint record is therefore unverified — everything above comes from the scripts, the driver, and the logs. Loading a 1.2 GB checkpoint would need torch and I avoided touching the GPU path. + +--- + +## 7. Loader API (real code) + +Audit dimension: Real loader API of Qwen3-0.6B/model.py and SmolLM2-134(base)/model_full.py (classes, config dataclasses, checkpoint loading, forward contract, tokenizer, faithful end-to-end snippet, safe_cuda dependency) + +### 7.1 Qwen3: exact top-level model class name and __init__ signature + +**Value** + +``` +class Qwen3ForCausalLM(nn.Module) with __init__(self, cfg: Qwen3Config). Single positional arg named `cfg`, no kwargs, no from_pretrained classmethod. Inner module is Qwen3Model(cfg) exposed as .model; .lm_head is nn.Linear(hidden_size, vocab_size, bias=False), tied to .model.embed_tokens.weight when cfg.tie_word_embeddings. +``` + +**Evidence** — `Qwen3-0.6B/model.py:237-246` + +**Source quote** + +``` +class Qwen3ForCausalLM(nn.Module): + def __init__(self, cfg: Qwen3Config): + super().__init__() + self.cfg = cfg + self.model = Qwen3Model(cfg) + self.lm_head = nn.Linear(cfg.hidden_size, cfg.vocab_size, bias=False) + if cfg.tie_word_embeddings: + # lm_head.weight aliases embed_tokens.weight (same storage). + self.lm_head.weight = self.model.embed_tokens.weight + self.apply(self._init_weights) +``` + +**Confidence** — measured from code + +**Caveat** — There is NO from_pretrained classmethod on this class — grep for 'from_pretrained' in Qwen3-0.6B/model.py matches only a comment on line 33. Weight loading is always external (see the load_official_weights_into_ours fact). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Verbatim match. Qwen3-0.6B/model.py:237 `class Qwen3ForCausalLM(nn.Module):`, :238 `def __init__(self, cfg: Qwen3Config):`, :241 `self.model = Qwen3Model(cfg)`, :242 lm_head nn.Linear(cfg.hidden_size, cfg.vocab_size, bias=False), :243-245 tie branch aliasing embed_tokens.weight, :246 `self.apply(self._init_weights)`. Quoted block = lines 237-246 exactly. `grep -n from_pretrained Qwen3-0.6B/model.py` returns exactly one hit, line 33 (a comment) — caveat CONFIRMED. Qwen3Model is defined at model.py:214 and .model is that instance. +``` + + +### 7.2 Qwen3: config dataclass name and exact fields with defaults + +**Value** + +``` +@dataclass Qwen3Config — 14 fields, all with defaults, so Qwen3Config() is valid with zero args. +``` + +**Evidence** — `Qwen3-0.6B/model.py:35-51` + +**Source quote** + +``` +@dataclass +class Qwen3Config: + vocab_size: int = 151_936 # config.json: vocab_size + hidden_size: int = 1024 # config.json: hidden_size + intermediate_size: int = 3072 # config.json: intermediate_size + num_hidden_layers: int = 28 # config.json: num_hidden_layers + num_attention_heads: int = 16 # config.json: num_attention_heads + num_key_value_heads: int = 8 # config.json: num_key_value_heads (GQA 16/8 = 2:1) + head_dim: int = 128 # config.json: head_dim — INDEPENDENT field, not hidden/n_heads + max_position_embeddings: int = 40_960 # config.json: max_position_embeddings + rope_theta: float = 1_000_000.0 # config.json: rope_theta + rms_norm_eps: float = 1e-6 # config.json: rms_norm_eps + initializer_range: float = 0.02 # config.json: initializer_range + tie_word_embeddings: bool = True # config.json: tie_word_embeddings + attention_bias: bool = False # config.json: attention_bias + attention_dropout: float = 0.0 # config.json: attention_dropout + # hidden_act = "silu" → SwiGLU(silu(gate) * up). Hardcoded in MLP below. +``` + +**Confidence** — measured from code + +**Caveat** — head_dim is a real dataclass FIELD here (128), unlike SmolLM2 where it is a derived @property. The inline comments claim each default matches Qwen/Qwen3-0.6B-Base config.json 'pulled via AutoConfig.from_pretrained on 2026-06-08' (model.py:33) — that provenance claim is a code comment, NOT something I re-verified against a config.json on disk. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Verifier note** + +``` +FIELD COUNT AND DEFAULTS CONFIRMED: AST parse of Qwen3-0.6B/model.py yields exactly 14 AnnAssign fields ['vocab_size','hidden_size','intermediate_size','num_hidden_layers','num_attention_heads','num_key_value_heads','head_dim','max_position_embeddings','rope_theta','rms_norm_eps','initializer_range','tie_word_embeddings','attention_bias','attention_dropout'], all defaulted; quoted block = model.py:35-51 exactly; head_dim IS a real field at model.py:43. + +MATERIAL QUALIFIER THE ORIGINAL AGENT DID NOT CATCH (it declared the provenance 'not re-verified' — I verified it and it FAILS on one field): the file header comment model.py:32-33 says 'every default matches config.json at the Qwen3-0.6B-Base repo HEAD (pulled via AutoConfig.from_pretrained on 2026-06-08)'. The actual cached config.json for that repo is on this box at HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache/hub/models--Qwen--Qwen3-0.6B-Base/snapshots/da87bfb608c14b7cf20ba1ce41287e8de496c0cd/config.json (snapshot dir mtime 2026-06-08 14:36 — the very date the comment cites). It states "max_position_embeddings": 32768, whereas Qwen3-0.6B/model.py:44 hardcodes `max_position_embeddings: int = 40_960 # config.json: max_position_embeddings`. 13 of 14 defaults match that config.json (vocab 151936, hidden 1024, inter 3072, layers 28, heads 16, kv 8, head_dim 128, rope_theta 1e6, eps 1e-6, init 0.02, tie true, attn_bias false, attn_dropout 0.0); max_position_embeddings does NOT. 40960 is the Qwen3-0.6B instruct/thinking value, not the -Base value in this snapshot. I did NOT fetch the live HF HEAD, so I can only assert the mismatch against the on-disk snapshot. +Impact: harmless for weight loading (RoPE tables are built at model.py:222 and registered with persistent=False, so they never enter the state_dict — verify.py still passes), but a published model card MUST NOT repeat 'every default matches the Base config.json', and must not print 40,960 as the Base context length. +``` + + +### 7.3 Qwen3: how are HF weights mapped in — is there a named converter/loader function? + +**Value** + +``` +Yes: load_official_weights_into_ours(ours: Qwen3ForCausalLM, hf_state_dict: dict) defined in Qwen3-0.6B/verify.py:22. It does NO key remapping — module names were written to mirror HF exactly, so it is a plain load_state_dict(strict=False) that then asserts the only missing key is the tied lm_head.weight and that nothing is unexpected. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:22,39-44` + +**Source quote** + +``` +def load_official_weights_into_ours(ours: Qwen3ForCausalLM, hf_state_dict: dict): +... + missing, unexpected = ours.load_state_dict(hf_state_dict, strict=False) + missing = [k for k in missing if k != "lm_head.weight"] + if missing: + raise RuntimeError(f"Unexpected missing keys: {missing}") + if unexpected: + raise RuntimeError(f"Unexpected keys: {unexpected}") +``` + +**Confidence** — measured from code + +**Caveat** — The function lives in verify.py, not model.py. Importing it (`from verify import load_official_weights_into_ours, REPO`) is safe because verify.py's main() is guarded by `if __name__ == "__main__":` at Qwen3-0.6B/verify.py:86. The SmolLM2 side proves this import pattern is the repo's own idiom (SmolLM2-134(base)/generate.py:11). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/verify.py:22 signature exact. Body lines 39-44 match the quote verbatim (39 load_state_dict strict=False, 40 filter lm_head.weight, 41-42 raise on missing, 43-44 raise on unexpected). Import-safety caveat CONFIRMED: verify.py:86 is `if __name__ == "__main__":` and :87 `main()`, so importing verify.py executes only lines 13-19 (torch, transformers, `from model import ...`, REPO). SmolLM2-134(base)/generate.py:11 is indeed `from verify import load_official_weights_into_ours, REPO`. +``` + + +### 7.4 Qwen3: real call site that constructs the model and loads OFFICIAL HF weights + +**Value** + +``` +Qwen3-0.6B/verify.py main() — the canonical parity gate. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:49-64` + +**Source quote** + +``` +print(f"Loading official {REPO} ...") + tokenizer = AutoTokenizer.from_pretrained(REPO) + hf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) + hf_model.eval() + + print("Building our model and copying weights ...") + ours = Qwen3ForCausalLM(Qwen3Config()) + load_official_weights_into_ours(ours, hf_model.state_dict()) + ours.eval() + + # Same prompt, same dtype, same device. + text = "The capital of France is" + input_ids = tokenizer(text, return_tensors="pt").input_ids + + hf_out = hf_model(input_ids).logits # (1, T, V) + our_out = ours(input_ids)["logits"] +``` + +**Confidence** — measured from code + +**Caveat** — Note `dtype=torch.float32` here, whereas the SmolLM2 equivalent uses the older `torch_dtype=torch.float32` (SmolLM2-134(base)/verify.py:53). The two repro folders are inconsistent on this transformers kwarg — pick one deliberately for a published snippet. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Quoted block is byte-for-byte lines 49-64 of Qwen3-0.6B/verify.py (49 print, 50 AutoTokenizer, 51 `AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32)`, 55 `ours = Qwen3ForCausalLM(Qwen3Config())`, 56 load_official_weights_into_ours, 57 ours.eval(), 60 text, 61 input_ids, 63 hf_out, 64 our_out). The transformers-kwarg divergence caveat is CONFIRMED: Qwen3-0.6B/verify.py:51 uses `dtype=`, SmolLM2-134(base)/verify.py:53 uses `torch_dtype=`. +``` + + +### 7.5 Qwen3: real call site that loads a TRAINED .pt checkpoint (not HF weights) + +**Value** + +``` +The eval-harness-constructed suite script. It handles both bare state_dicts and {'model': sd} wrappers, and strips the torch.compile `_orig_mod.` prefix, then load_state_dict(..., strict=True). +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3-0.6b_eval-faithful/eval_suite.py:97-114` + +**Source quote** + +``` +def load_checkpoint_sd(path: str) -> dict: + ck = torch.load(path, map_location="cpu", weights_only=False) + sd = ck["model"] if isinstance(ck, dict) and "model" in ck else ck + # torch.compile-trained checkpoints carry an _orig_mod. prefix (e.g. the + # Qwen3 modernized build) — strip per finance-research-loop SKILL.md. + return {k.removeprefix("_orig_mod."): v for k, v in sd.items()} + + +def build_model(mod): + # ADJUST AT CONSTRUCTION if the build's own train/verify script constructs + # its config differently (extra kwargs, config loaded from the ckpt, ...). + return getattr(mod, MODEL_CLASS)(getattr(mod, CONFIG_CLASS)()) + + +def load_model(ckpt_path: str): + model = build_model(load_model_module()) + model.load_state_dict(load_checkpoint_sd(ckpt_path), strict=True) # strict, always + return model.to(device=DEVICE, dtype=DTYPE).eval() +``` + +**Confidence** — measured from code + +**Caveat** — build_model uses getattr indirection driven by module constants MODEL_CLASS="Qwen3ForCausalLM" / CONFIG_CLASS="Qwen3Config" (eval_suite.py:52-53). For a published snippet, inline the direct call `Qwen3ForCausalLM(Qwen3Config())` — that is exactly what the getattr resolves to. DEVICE/DTYPE are cuda+bfloat16 when CUDA is available (eval_suite.py:80-81). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/experiments/2026-06-16_qwen3-0.6b_eval-faithful/eval_suite.py:97 `def load_checkpoint_sd`, :98 torch.load(weights_only=False), :99 `sd = ck["model"] if isinstance(ck, dict) and "model" in ck else ck`, :102 removeprefix("_orig_mod."), :105 build_model, :108 getattr(mod, MODEL_CLASS)(getattr(mod, CONFIG_CLASS)()), :113 load_state_dict(..., strict=True), :114 .to(DEVICE, DTYPE).eval() — quote matches 97-114 verbatim. Caveat CONFIRMED: MODEL_CLASS="Qwen3ForCausalLM" at eval_suite.py:52, CONFIG_CLASS="Qwen3Config" at :53; DEVICE at :80 (`cuda` if available), DTYPE at :81 (bfloat16 if cuda else float32). +``` + + +### 7.6 Qwen3: training-script construction + resume path (the from-scratch build) + +**Value** + +``` +cfg = Qwen3Config(); model = Qwen3ForCausalLM(cfg).to(device=device, dtype=dtype); resume via torch.load(...)['model'] into model.load_state_dict. Checkpoints are saved as a dict with keys model/config/step/tok_seen/baseline_ppl/trained_ppl/training_recipe/optim/sched/rng_*. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:292-293,311-312,350-356` + +**Source quote** + +``` +cfg = Qwen3Config() + model = Qwen3ForCausalLM(cfg).to(device=device, dtype=dtype) +... + ck = torch.load(args.resume, map_location="cpu", weights_only=False) + model.load_state_dict(ck["model"]) +... + def save_ckpt(step: int, tok_seen: int, trained_ppl=None): + torch.save({ + "model": model.state_dict(), + "config": cfg.__dict__, + "step": step, + "tok_seen": tok_seen, + "baseline_ppl": base_ppl, +``` + +**Confidence** — measured from code + +**Caveat** — The saved 'config' is cfg.__dict__ (a plain dict), NOT a pickled Qwen3Config. No loader in the repo reconstructs the config from the checkpoint — every load site calls Qwen3Config() with defaults instead. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:292 `cfg = Qwen3Config()`, :293 `model = Qwen3ForCausalLM(cfg).to(device=device, dtype=dtype)`; :311 `ck = torch.load(args.resume, map_location="cpu", weights_only=False)`, :312 `model.load_state_dict(ck["model"])`; :350 `def save_ckpt(...)`, :351 torch.save({, :352 model, :353 `"config": cfg.__dict__,`, :354 step, :355 tok_seen, :356 baseline_ppl — quote matches exactly. Remaining keys verified: trained_ppl :357, training_recipe :358, optim/sched :368, rng_torch :369, rng_cuda :370, rng_numpy/rng_python :371. Caveat CONFIRMED: `cfg.__dict__` is a plain dict and no loader in the repo reads it back. +``` + + +### 7.7 Qwen3: forward() signature and return type/shapes + +**Value** + +``` +forward(self, input_ids, labels=None, attention_mask=None) -> dict with keys 'logits' and 'loss'. logits shape (B, T, vocab_size); loss is a scalar tensor when labels is passed, else None. Not a tuple, not a HF ModelOutput — a plain Python dict, so call sites index it as model(x)["logits"]. +``` + +**Evidence** — `Qwen3-0.6B/model.py:259-274` + +**Source quote** + +``` +def forward(self, input_ids: torch.Tensor, + labels: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None): + hidden = self.model(input_ids, attention_mask) + logits = self.lm_head(hidden) + + loss = None + if labels is not None: + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + loss = F.cross_entropy( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1), + ignore_index=-100, + ) + return {"logits": logits, "loss": loss} +``` + +**Confidence** — measured from code + +**Caveat** — The labels path materializes a full (B*T, 151936) fp32 CE — this is exactly the pattern CLAUDE.md §C1 warns about for large vocab. The model file itself does NOT chunk cross-entropy. Shapes confirmed by the module's own __main__ demo at model.py:307-309 (x = torch.randint(0, cfg.vocab_size, (2, 16)); out = m(x, labels=x)). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/model.py:259-274 matches the quote verbatim (259-261 signature, 262 hidden, 263 logits, 265 loss=None, 267-272 shift + F.cross_entropy over view(-1, 151936), 274 `return {"logits": logits, "loss": loss}`). Shape demo caveat CONFIRMED: model.py:307 `x = torch.randint(0, cfg.vocab_size, (2, 16))`, :308 `out = m(x, labels=x)`, :309 prints logits shape + loss. §C1 chunked-CE caveat is a correct reading — there is no chunking anywhere in model.py. +``` + + +### 7.8 Qwen3: generate() signature and semantics + +**Value** + +``` +@torch.no_grad() generate(self, input_ids, max_new_tokens=64, temperature=0.8, top_k=50) -> torch.Tensor of token ids (prompt + continuation concatenated). temperature<=0 means greedy argmax. No KV cache — the prefix is recomputed each step. +``` + +**Evidence** — `Qwen3-0.6B/model.py:276-295` + +**Source quote** + +``` +@torch.no_grad() + def generate(self, input_ids: torch.Tensor, max_new_tokens: int = 64, + temperature: float = 0.8, top_k: int | None = 50) -> torch.Tensor: + """Greedy/top-k sampling. No KV cache — recomputes the prefix each step. + Defaults mirror SmolLM2's generate() so the harness stays consistent.""" + self.eval() +``` + +**Confidence** — measured from code + +**Caveat** — generate() calls self.eval() internally (model.py:281), so an explicit .eval() before it is redundant though harmless. Real call site: eval_suite.py:190-192 uses max_new_tokens=60, temperature=0.7, top_k=40 (suite constants at eval_suite.py:68), NOT the class defaults. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/model.py:276 `@torch.no_grad()`, :277-278 signature with those exact defaults, :279-280 docstring 'No KV cache — recomputes the prefix each step', :281 `self.eval()`, :285 `if temperature <= 0:` greedy argmax, :295 `return input_ids` after torch.cat at :294 — so the return is prompt+continuation. Caveats CONFIRMED: self.eval() at :281; real call site eval_suite.py:190-191 `model.generate(ids, max_new_tokens=GEN_MAXNEW, temperature=GEN_TEMP, top_k=GEN_TOPK)` with `GEN_SEED, GEN_TEMP, GEN_TOPK, GEN_MAXNEW = 42, 0.7, 40, 60` at eval_suite.py:68. +``` + + +### 7.9 Qwen3: how is the tokenizer obtained, and with which repo id? + +**Value** + +``` +AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B-Base"). The repo id is the module constant REPO in verify.py and is re-declared as REPO in the train script and as TOKENIZER_REPO in the eval suite. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:19,50` + +**Source quote** + +``` +REPO = "Qwen/Qwen3-0.6B-Base" +... + tokenizer = AutoTokenizer.from_pretrained(REPO) +``` + +**Confidence** — measured from code + +**Caveat** — Same id independently at Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:59 (REPO = "Qwen/Qwen3-0.6B-Base") and Qwen3-0.6B/experiments/2026-06-16_qwen3-0.6b_eval-faithful/eval_suite.py:54 (TOKENIZER_REPO = "Qwen/Qwen3-0.6B-Base" # REPO from /verify.py — own tokenizer ONLY). Note it is the -Base repo, not Qwen/Qwen3-0.6B. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/verify.py:19 `REPO = "Qwen/Qwen3-0.6B-Base"`, :50 `tokenizer = AutoTokenizer.from_pretrained(REPO)`. Independent restatements confirmed: train_qwen3.py:59 `REPO = "Qwen/Qwen3-0.6B-Base"` and eval_suite.py:54 `TOKENIZER_REPO = "Qwen/Qwen3-0.6B-Base" # REPO from /verify.py — own tokenizer ONLY`. The -Base (not -Instruct) distinction is correct. +``` + + +### 7.10 SmolLM2: exact top-level model class name and __init__ signature + +**Value** + +``` +class SmolLM2ForCausalLM(nn.Module) with __init__(self, cfg: SmolLM2Config). Structurally identical to the Qwen3 class: one positional `cfg`, .model = SmolLM2Model(cfg), tied .lm_head. +``` + +**Evidence** — `SmolLM2-134(base)/model_full.py:238-248` + +**Source quote** + +``` +class SmolLM2ForCausalLM(nn.Module): + def __init__(self, cfg: SmolLM2Config): + super().__init__() + self.cfg = cfg + self.model = SmolLM2Model(cfg) + self.lm_head = nn.Linear(cfg.hidden_size, cfg.vocab_size, bias=False) + if cfg.tie_word_embeddings: + # Weight tying: lm_head.weight IS embed_tokens.weight (same storage). + # config.json: tie_word_embeddings = true. + self.lm_head.weight = self.model.embed_tokens.weight + self.apply(self._init_weights) +``` + +**Confidence** — measured from code + +**Caveat** — No from_pretrained classmethod (grep for 'from_pretrained' in model_full.py returns nothing). + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +SmolLM2-134(base)/model_full.py:238-248 matches the quote verbatim (238 class, 239 __init__, 242 self.model = SmolLM2Model(cfg), 243 lm_head, 244-246 tie comment+alias, 248 self.apply). `grep -n from_pretrained "SmolLM2-134(base)/model_full.py"` exits 1 (zero hits) — caveat CONFIRMED. +``` + + +### 7.11 SmolLM2: config dataclass name and exact fields with defaults + +**Value** + +``` +@dataclass SmolLM2Config — 12 fields, all defaulted, plus a derived head_dim @property (576//9 = 64). SmolLM2Config() is valid with zero args. +``` + +**Evidence** — `SmolLM2-134(base)/model_full.py:28-49` + +**Source quote** + +``` +@dataclass +class SmolLM2Config: + vocab_size: int = 49152 # config.json: vocab_size + hidden_size: int = 576 # config.json: hidden_size + intermediate_size: int = 1536 # config.json: intermediate_size + num_hidden_layers: int = 30 # config.json: num_hidden_layers + num_attention_heads: int = 9 # config.json: num_attention_heads + num_key_value_heads: int = 3 # config.json: num_key_value_heads (GQA: 9 Q / 3 KV) + max_position_embeddings: int = 8192 # config.json: max_position_embeddings + rope_theta: float = 100_000.0 # config.json: rope_theta (note: v2 uses 100k, v1 was 10k) + rms_norm_eps: float = 1e-5 # config.json: rms_norm_eps + initializer_range: float = 1.0 / math.sqrt(576) # config.json: initializer_range = 0.041666... = 1/sqrt(576) + tie_word_embeddings: bool = True # config.json: tie_word_embeddings + attention_bias: bool = False # config.json: attention_bias + attention_dropout: float = 0.0 # config.json: attention_dropout + # hidden_act = "silu" → SwiGLU(silu(gate) * up), per HF LlamaMLP. Hardcoded below. + + @property + def head_dim(self) -> int: + # Llama convention: head_dim = hidden_size // num_attention_heads. + # 576 / 9 = 64. + return self.hidden_size // self.num_attention_heads +``` + +**Confidence** — measured from code + +**Caveat** — KEY ASYMMETRY vs Qwen3: head_dim here is a read-only @property derived from hidden_size//num_attention_heads, NOT a settable dataclass field. `SmolLM2Config(head_dim=64)` would raise TypeError. Also initializer_range hardcodes sqrt(576) rather than sqrt(hidden_size), so overriding hidden_size silently leaves the old init std. + +**Verdict — ❌ WRONG** + +**Corrected value** + +``` +13 fields (not 12): vocab_size, hidden_size, intermediate_size, num_hidden_layers, num_attention_heads, num_key_value_heads, max_position_embeddings, rope_theta, rms_norm_eps, initializer_range, tie_word_embeddings, attention_bias, attention_dropout — plus the derived head_dim @property. +``` + +**Verifier note** + +``` +FIELD COUNT IS WRONG. AST parse of SmolLM2-134(base)/model_full.py returns 13 AnnAssign fields in SmolLM2Config, not 12. Cross-check: Qwen3Config = 14 = the same 13 + head_dim, which the fact list itself asserts, so 12 is internally inconsistent with its own sibling fact. Everything else in this fact is CONFIRMED: quoted block = model_full.py:28-49 verbatim; attention_dropout at :42 (grep-confirmed); @property head_dim at :45-49 returning hidden_size // num_attention_heads = 576//9 = 64; initializer_range at :39 literally `1.0 / math.sqrt(576)` (hardcoded 576, not hidden_size); all 13 fields defaulted so SmolLM2Config() is valid with zero args, and head_dim is not an accepted kwarg. BONUS VERIFICATION the original agent did not do: all 13 defaults DO match the cached HuggingFaceTB/SmolLM2-135M config.json under HF_HOME (/home/yashb98/projects/qwen-distill/hf_cache/hub/models--HuggingFaceTB--SmolLM2-135M/.../config.json: vocab 49152, hidden 576, inter 1536, layers 30, heads 9, kv 3, max_pos 8192, rope_theta 100000, eps 1e-05, initializer_range 0.041666666666666664, tie true, attention_bias false, attention_dropout 0.0) — unlike Qwen3, this config has no mismatch. +``` + + +### 7.12 SmolLM2: how are HF weights mapped in — named converter function? + +**Value** + +``` +Yes: load_official_weights_into_ours(ours: SmolLM2ForCausalLM, hf_state_dict: dict) in SmolLM2-134(base)/verify.py:22. Same shape as the Qwen3 one — no key remapping, strict=False load then assert only the tied lm_head.weight is missing. +``` + +**Evidence** — `SmolLM2-134(base)/verify.py:22,39-46` + +**Source quote** + +``` +def load_official_weights_into_ours(ours: SmolLM2ForCausalLM, hf_state_dict: dict): +... + # Filter strict=False to ignore the absent lm_head.weight (it's tied). + missing, unexpected = ours.load_state_dict(hf_state_dict, strict=False) + # We expect lm_head.weight to be "missing" (tied), and nothing unexpected. + missing = [k for k in missing if k != "lm_head.weight"] + if missing: + raise RuntimeError(f"Unexpected missing keys: {missing}") + if unexpected: + raise RuntimeError(f"Unexpected keys: {unexpected}") +``` + +**Confidence** — measured from code + +**Caveat** — Re-used by import in at least four call sites: generate.py:11, compare_with_hf.py:28, eval_after_vs_base.py:25, tests/test_parity.py:33. That makes `from verify import load_official_weights_into_ours, REPO` the repo's blessed public loader idiom. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +verify.py:22 signature exact; lines 39-46 match the quote verbatim (39 comment, 40 load_state_dict strict=False, 41 comment, 42 filter, 43-44 missing raise, 45-46 unexpected raise). Four re-use sites CONFIRMED at the cited lines: generate.py:11, compare_with_hf.py:28, eval_after_vs_base.py:25, tests/test_parity.py:33. Minor: three of the four write the names in the opposite order (`from verify import REPO, load_official_weights_into_ours`); only generate.py:11 uses the order the fact quotes. Semantically irrelevant, but the 'blessed idiom' wording overstates uniformity. +``` + + +### 7.13 SmolLM2: real call sites constructing the model and loading weights (HF and trained .pt) + +**Value** + +``` +HF path: verify.py:57-58 and generate.py:18-19. Trained-checkpoint path: eval_after_vs_base.py:42-44 (torch.load(...) then load_state_dict(ckpt['model'])). +``` + +**Evidence** — `SmolLM2-134(base)/eval_after_vs_base.py:34-45` + +**Source quote** + +``` +print("Loading BASE (official) model...") +hf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) +base = SmolLM2ForCausalLM(SmolLM2Config()) +load_official_weights_into_ours(base, hf.state_dict()) +base = base.to(device=device, dtype=dtype).eval() +del hf + +print("Loading TRAINED (TinyStories continued-pretrained) model...") +trained = SmolLM2ForCausalLM(SmolLM2Config()) +ckpt = torch.load("checkpoint_tinystories.pt", map_location="cpu", weights_only=False) +trained.load_state_dict(ckpt["model"]) +trained = trained.to(device=device, dtype=dtype).eval() +``` + +**Confidence** — measured from code + +**Caveat** — eval_after_vs_base.py:43 uses a RELATIVE path 'checkpoint_tinystories.pt' — it only works with cwd = SmolLM2-134(base)/. That file does exist on disk (269144681 bytes), as does checkpoint.pt (538173921 bytes). Unlike the Qwen3 eval suite, this site does NOT strip an `_orig_mod.` prefix, so it would break on a torch.compile-saved checkpoint. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +eval_after_vs_base.py:34-45 matches the quote verbatim: :35 `hf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32)`, :36 base build, :37 loader, :38 .to().eval(), :39 del hf, :42 `trained = SmolLM2ForCausalLM(SmolLM2Config())`, :43 `ckpt = torch.load("checkpoint_tinystories.pt", map_location="cpu", weights_only=False)`, :44 `trained.load_state_dict(ckpt["model"])`, :45 .to().eval(). HF path confirmed at verify.py:57-58 and generate.py:18-19. Caveats CONFIRMED by `ls -la`: checkpoint_tinystories.pt = 269144681 bytes, checkpoint.pt = 538173921 bytes, both present; the relative path at :43 does require cwd = SmolLM2-134(base)/; and there is no _orig_mod. stripping at :44 (default strict=True). +``` + + +### 7.14 SmolLM2: forward() signature and return type/shapes + +**Value** + +``` +Identical contract to Qwen3: forward(self, input_ids, labels=None, attention_mask=None) -> {"logits": (B,T,49152), "loss": scalar-or-None}. Plain dict. +``` + +**Evidence** — `SmolLM2-134(base)/model_full.py:263-279` + +**Source quote** + +``` +def forward(self, input_ids: torch.Tensor, + labels: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None): + hidden = self.model(input_ids, attention_mask) + logits = self.lm_head(hidden) + + loss = None + if labels is not None: + # Standard causal LM shift: predict token t+1 from positions ≤ t. + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + loss = F.cross_entropy( + shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1), + ignore_index=-100, + ) + return {"logits": logits, "loss": loss} +``` + +**Confidence** — measured from code + +**Caveat** — attention_mask is forwarded straight into F.scaled_dot_product_attention as attn_mask, and is_causal flips to False whenever a mask is supplied (model_full.py:156-161; identically Qwen3-0.6B/model.py:162-167). So passing an HF-style 2-D padding mask would SILENTLY DISABLE causal masking — a published snippet should not pass attention_mask. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +model_full.py:263-279 matches the quote verbatim (263-265 signature, 266 hidden, 267 logits, 269 loss=None, 271-277 shift + F.cross_entropy, 279 return dict). The attention_mask caveat is the most valuable item in this dimension and is CONFIRMED at both files: model_full.py:156-161 and Qwen3-0.6B/model.py:162-167 are the identical `F.scaled_dot_product_attention(q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=(attention_mask is None))` — passing any 2-D HF padding mask does silently disable causal masking. SmolLM2's own inline comment at model_full.py:154-155 acknowledges this. +``` + + +### 7.15 SmolLM2: how is the tokenizer obtained, and with which repo id? + +**Value** + +``` +AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M"), via the module constant REPO in verify.py. +``` + +**Evidence** — `SmolLM2-134(base)/verify.py:19,52` + +**Source quote** + +``` +REPO = "HuggingFaceTB/SmolLM2-135M" +... + tokenizer = AutoTokenizer.from_pretrained(REPO) +``` + +**Confidence** — measured from code + +**Caveat** — Also hardcoded literally (not via REPO) at SmolLM2-134(base)/train.py:134: `tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M")`. Note the folder is named SmolLM2-134(base) but the HF repo id says 135M. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +SmolLM2-134(base)/verify.py:19 `REPO = "HuggingFaceTB/SmolLM2-135M"`, :52 `tokenizer = AutoTokenizer.from_pretrained(REPO)`. Caveat CONFIRMED: train.py:134 hardcodes the literal `tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-135M")` rather than importing REPO. Folder-vs-repo-name mismatch (134 vs 135M) is real. +``` + + +### 7.16 Does model.py / model_full.py itself depend on safe_cuda / guard() being imported first? + +**Value** + +``` +NO. Neither model file imports safe_cuda — `grep -rn safe_cuda Qwen3-0.6B/model.py Qwen3-0.6B/verify.py` exits 1 (no match), and `grep -rn safe_cuda` over the ENTIRE SmolLM2-134(base)/ tree exits 1 (zero matches anywhere, including train.py). The model files import only torch/torch.nn/torch.nn.functional (+ dataclasses, and math for SmolLM2). safe_cuda is a CALLER-SIDE obligation imposed by CLAUDE.md §C1, honored by the research-loop-constructed scripts, not by the model modules. +``` + +**Evidence** — `Qwen3-0.6B/model.py:22-28` + +**Source quote** + +``` +from __future__ import annotations + +from dataclasses import dataclass + +import torch +import torch.nn as nn +import torch.nn.functional as F +``` + +**Confidence** — measured from code + +**Caveat** — IMPORTANT for a published snippet: Qwen3-0.6B/verify.py — the repo's own canonical usage script — does NOT import safe_cuda either (grep exit 1). The GPU-touching research-loop scripts DO: eval_suite.py:38-45 imports safe_cuda before torch then calls safe_cuda.guard(0.85) at line 202; train_qwen3.py:42 imports safe_cuda before torch and calls safe_cuda.guard(args.mem_fraction) at line 269. safe_cuda.guard's real signature is `def guard(fraction: float = 0.85, device: int = 0) -> None` (safe_cuda.py:47) and it no-ops when CUDA is unavailable (safe_cuda.py:51-52). So: a CPU-only snippet needs no guard; any snippet that moves the model to CUDA should include the two-line safe_cuda header to be repo-compliant. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Reproduced every grep: `grep -n safe_cuda Qwen3-0.6B/model.py Qwen3-0.6B/verify.py` exits 1 (no matches); `grep -rn safe_cuda "SmolLM2-134(base)/"` exits 1 (zero matches over the whole tree incl. train.py, tests/, scripts/). Qwen3-0.6B/model.py:22-28 imports exactly what the quote shows. Caveat lines all CONFIRMED: eval_suite.py:38 safe_cuda header comment, :43 `import safe_cuda`, :45 `import torch` (so 38-45 is the correct span), `safe_cuda.guard(0.85)` at :202 (also a second call at :315, not mentioned); train_qwen3.py:42 `import safe_cuda` before torch at :47, `safe_cuda.guard(args.mem_fraction)` at :269 with the flag defined at :242 (default 0.85). safe_cuda.py:47 `def guard(fraction: float = 0.85, device: int = 0) -> None:` exact; :51-52 `if not torch.cuda.is_available(): return` exact. Additional un-noted guard behavior: safe_cuda.py:53-56 raises ValueError unless 0.0 < fraction <= 0.95. +``` + + +### 7.17 SmolLM2: is there an existing end-to-end usage script I can copy verbatim? + +**Value** + +``` +YES — SmolLM2-134(base)/generate.py is a complete import -> build config -> build model -> load weights -> tokenize -> generate -> decode script in 31 lines. This is the highest-fidelity source for a published usage example on the SmolLM2 side; nothing needs to be invented. +``` + +**Evidence** — `SmolLM2-134(base)/generate.py:6-25` + +**Source quote** + +``` +import sys +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM + +from model_full import SmolLM2ForCausalLM, SmolLM2Config +from verify import load_official_weights_into_ours, REPO + + +def main(prompt: str, max_new_tokens: int = 64): + tokenizer = AutoTokenizer.from_pretrained(REPO) + hf = AutoModelForCausalLM.from_pretrained(REPO, torch_dtype=torch.float32) + + model = SmolLM2ForCausalLM(SmolLM2Config()) + load_official_weights_into_ours(model, hf.state_dict()) + del hf + model.eval() + + input_ids = tokenizer(prompt, return_tensors="pt").input_ids + out = model.generate(input_ids, max_new_tokens=max_new_tokens, temperature=0.8, top_k=50) + print(tokenizer.decode(out[0], skip_special_tokens=True)) +``` + +**Confidence** — measured from code + +**Caveat** — generate.py:16 uses `torch_dtype=` which is deprecated in current transformers; three other SmolLM2 call sites in the same folder use `dtype=` (compare_with_hf.py:45, eval_after_vs_base.py:35, tests/test_parity.py:39), as does Qwen3-0.6B/verify.py:51. If publishing, prefer `dtype=` and note the divergence rather than silently 'fixing' generate.py. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Verifier note** + +``` +SUBSTANCE CONFIRMED: generate.py:6-25 matches the quote verbatim and is a complete import->config->model->load->tokenize->generate->decode path. LINE COUNT WRONG: the file is 30 lines, not 31 (`wc -l generate.py` = 30; bytes=980; endswith newline=True; len(splitlines())=30). Deprecation caveat CONFIRMED: generate.py:16 uses `torch_dtype=`, while compare_with_hf.py:45, eval_after_vs_base.py:35, tests/test_parity.py:39 and Qwen3-0.6B/verify.py:51 all use `dtype=`. +``` + + +### 7.18 FAITHFUL end-to-end usage snippet — SmolLM2 (every line traced to a repo line) + +**Value** + +``` +import torch # generate.py:7\nfrom transformers import AutoTokenizer, AutoModelForCausalLM # generate.py:8\nfrom model_full import SmolLM2ForCausalLM, SmolLM2Config # generate.py:10\nfrom verify import load_official_weights_into_ours, REPO # generate.py:11\n\ntokenizer = AutoTokenizer.from_pretrained(REPO) # generate.py:15 (REPO = "HuggingFaceTB/SmolLM2-135M", verify.py:19)\nhf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) # eval_after_vs_base.py:35 (generate.py:16 is the same call with the deprecated torch_dtype=)\nmodel = SmolLM2ForCausalLM(SmolLM2Config()) # generate.py:18\nload_official_weights_into_ours(model, hf.state_dict()) # generate.py:19\ndel hf # generate.py:20\nmodel.eval() # generate.py:21\n\ninput_ids = tokenizer("The capital of France is", return_tensors="pt").input_ids # call = generate.py:23; prompt string = verify.py:62\nlogits = model(input_ids)["logits"] # verify.py:66 (`our_out = ours(input_ids)["logits"]`)\nnext_id = logits[0, -1].argmax().item() # verify.py:80 (`our_next = our_out[0, -1].argmax().item()`)\nprint(tokenizer.decode([next_id])) # decode-a-single-id form from verify.py:82; skip_special_tokens form from generate.py:25\n\nout = model.generate(input_ids, max_new_tokens=64, temperature=0.8, top_k=50) # generate.py:24 with its own defaults (generate.py:14, model_full.py:282-283)\nprint(tokenizer.decode(out[0], skip_special_tokens=True)) # generate.py:25 +``` + +**Evidence** — `SmolLM2-134(base)/generate.py:6-25` + +**Source quote** + +``` +input_ids = tokenizer(prompt, return_tensors="pt").input_ids + out = model.generate(input_ids, max_new_tokens=max_new_tokens, temperature=0.8, top_k=50) + print(tokenizer.decode(out[0], skip_special_tokens=True)) +``` + +**Confidence** — measured from code + +**Caveat** — LINES I WROTE MYSELF (not verbatim in any single repo file): (a) the blank-line/ordering assembly — generate.py wraps these in `def main(prompt, max_new_tokens=64)`, I unwrapped to module scope; (b) inlining the literal "The capital of France is" in place of the `prompt` parameter — the literal is real (verify.py:62) but appears there as a separate `text = ...` binding; (c) `print(tokenizer.decode([next_id]))` — verify.py:82 is `print(f"Ours next : {tokenizer.decode([our_next])!r}")`, so the decode call is real but the print wrapper is simplified; (d) swapping torch_dtype= for dtype= as noted. Everything else is verbatim. REQUIRES cwd = SmolLM2-134(base)/ (or that dir on sys.path) because `from model_full import ...` and `from verify import ...` are flat-module imports — there is no __init__.py in that folder. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Verifier note** + +``` +EVERY CITATION IN THE SNIPPET VERIFIED CORRECT, line by line: generate.py:7 import torch; :8 transformers import; :10 model_full import; :11 verify import; :14 default max_new_tokens=64; :15 tokenizer; :18 model build; :19 loader; :20 del hf; :21 model.eval(); :23 input_ids; :24 generate(temperature=0.8, top_k=50); :25 decode(skip_special_tokens=True). verify.py:19 REPO; :62 `text = "The capital of France is"`; :66 `our_out = ours(input_ids)["logits"]`; :80 `our_next = our_out[0, -1].argmax().item()`; :82 the decode print. eval_after_vs_base.py:35 the `dtype=` form. model_full.py:282-283 the generate defaults. The self-declared 'lines I wrote myself' list is honest and complete. +MISSING MATERIAL CAVEAT (asymmetric with the Qwen3 sibling fact, which does flag it): the repo runs this whole sequence under `@torch.no_grad()` (SmolLM2-134(base)/verify.py:49 decorates main(); generate.py has no forward call outside model.generate, which is itself @torch.no_grad() at model_full.py:281). The composed snippet calls `model(input_ids)["logits"]` at module scope with grad tracking ON. A published snippet should keep `with torch.no_grad():` around the forward. +Second qualifier: the snippet needs the official weights. They are NOT in ~/.cache/huggingface (that dir holds only models--Qwen--Qwen3.5-9B); they are at HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache/hub/models--HuggingFaceTB--SmolLM2-135M. First run on a clean machine downloads from the Hub. +``` + + +### 7.19 FAITHFUL end-to-end usage snippet — Qwen3 (every line traced to a repo line) + +**Value** + +``` +import torch # verify.py:13\nfrom transformers import AutoModelForCausalLM, AutoTokenizer # verify.py:14\nfrom model import Qwen3ForCausalLM, Qwen3Config # verify.py:16\nfrom verify import load_official_weights_into_ours, REPO # WRITTEN BY ME (import form copied from SmolLM2-134(base)/generate.py:11); both names are real at Qwen3-0.6B/verify.py:22 and :19\n\ntokenizer = AutoTokenizer.from_pretrained(REPO) # verify.py:50 (REPO = "Qwen/Qwen3-0.6B-Base", verify.py:19)\nhf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) # verify.py:51\nours = Qwen3ForCausalLM(Qwen3Config()) # verify.py:55\nload_official_weights_into_ours(ours, hf_model.state_dict()) # verify.py:56\nours.eval() # verify.py:57\n\ntext = "The capital of France is" # verify.py:60\ninput_ids = tokenizer(text, return_tensors="pt").input_ids # verify.py:61\nour_out = ours(input_ids)["logits"] # verify.py:64\nour_next = our_out[0, -1].argmax().item() # verify.py:78\nprint(tokenizer.decode([our_next])) # simplified from verify.py:80\n\nout = ours.generate(input_ids, max_new_tokens=60, temperature=0.7, top_k=40) # call form + exact args from eval_suite.py:190-191\nprint(tokenizer.decode(out[0], skip_special_tokens=True)) # eval_suite.py:192 (`return tokenizer.decode(out[0], skip_special_tokens=True)`) +``` + +**Evidence** — `Qwen3-0.6B/verify.py:49-64` + +**Source quote** + +``` +tokenizer = AutoTokenizer.from_pretrained(REPO) + hf_model = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) + hf_model.eval() + + print("Building our model and copying weights ...") + ours = Qwen3ForCausalLM(Qwen3Config()) + load_official_weights_into_ours(ours, hf_model.state_dict()) + ours.eval() + + # Same prompt, same dtype, same device. + text = "The capital of France is" + input_ids = tokenizer(text, return_tensors="pt").input_ids + + hf_out = hf_model(input_ids).logits # (1, T, V) + our_out = ours(input_ids)["logits"] +``` + +**Confidence** — measured from code + +**Caveat** — LINES I WROTE MYSELF: (a) `from verify import load_official_weights_into_ours, REPO` — this exact line does NOT exist in Qwen3-0.6B/ (it exists only as the SmolLM2 analogue at generate.py:11). Both imported names are real module-scope objects in Qwen3-0.6B/verify.py (lines 22 and 19) and verify.py's main() is __main__-guarded (line 86), so the import is sound, but it is my composition, not copied text. (b) unwrapping verify.py's `@torch.no_grad() def main()` (lines 47-48) to module scope — the repo runs this whole block under torch.no_grad(); a published snippet should keep the decorator or wrap in `with torch.no_grad():`. (c) the simplified print. (d) the two generate lines come from a DIFFERENT file (the eval suite) than the rest. REQUIRES cwd = Qwen3-0.6B/ or sys.path insertion — the repo's own idiom for that is train_qwen3.py:55-57: `MODEL_DIR = pathlib.Path(__file__).resolve().parents[2]; sys.path.insert(0, str(MODEL_DIR)); from model import Qwen3Config, Qwen3ForCausalLM`. Also: this snippet loads the FULL fp32 HF model plus a second full copy of the weights — on the GB10 that is ~2x596M x 4B = ~4.8 GB, CPU-only here (nothing is moved to CUDA), so no safe_cuda.guard is required, matching verify.py which has none. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Verifier note** + +``` +ALL VERIFY.PY CITATIONS EXACT: :13 import torch; :14 `from transformers import AutoModelForCausalLM, AutoTokenizer`; :16 `from model import Qwen3ForCausalLM, Qwen3Config`; :19 REPO; :22 loader def; :47-48 `@torch.no_grad()` + `def main():`; :50 tokenizer; :51 `dtype=torch.float32`; :55 model build; :56 loader call; :57 ours.eval(); :60 text; :61 input_ids; :64 logits; :78 argmax; :80 the decode print; :86 the __main__ guard. eval_suite.py:192 is `return tokenizer.decode(out[0], skip_special_tokens=True)`. The sys.path idiom quoted from train_qwen3.py:55-57 is verbatim correct (55 MODEL_DIR parents[2], 56 sys.path.insert, 57 `from model import Qwen3Config, Qwen3ForCausalLM`). The memory math checks out: 596,049,920 x 4 B = 2.38 GB per copy, ~4.8 GB for two, CPU-only, no safe_cuda needed. +QUALIFIER 1 (citation precision): `max_new_tokens=60, temperature=0.7, top_k=40` is annotated 'exact args from eval_suite.py:190-191', but lines 190-191 pass the NAMES GEN_MAXNEW/GEN_TEMP/GEN_TOPK; the literals 60/0.7/40 live at eval_suite.py:68 (`GEN_SEED, GEN_TEMP, GEN_TOPK, GEN_MAXNEW = 42, 0.7, 40, 60`). Re-anchor to :68. +QUALIFIER 2 (not stated): with no KV cache (model.py:279-280) 60 new tokens = 60 full fp32 CPU forward passes of a 596M model — this snippet is minutes-slow on CPU, unlike the 135M SmolLM2 one. +QUALIFIER 3: as with SmolLM2, the weights come from HF_HOME=/home/yashb98/projects/qwen-distill/hf_cache (snapshot da87bfb608c14b7cf20ba1ce41287e8de496c0cd), not ~/.cache/huggingface. The self-flagged 'written by me' items (the `from verify import ...` line, the no_grad unwrap, the simplified print, the cross-file generate lines) are all accurately declared. +``` + + +### 7.20 Is there a public helper for parameter counting? + +**Value** + +``` +Yes, in BOTH files, identically: def num_params(model: nn.Module, only_trainable: bool = False) -> int. Both module __main__ blocks print it against a hardcoded expected value. +``` + +**Evidence** — `Qwen3-0.6B/model.py:298-299` + +**Source quote** + +``` +def num_params(model: nn.Module, only_trainable: bool = False) -> int: + return sum(p.numel() for p in model.parameters() if (p.requires_grad or not only_trainable)) +``` + +**Confidence** — measured from code + +**Caveat** — The 'expected' counts printed in the __main__ demos are prose-in-code, NOT computed at read time: Qwen3-0.6B/model.py:306 prints 'Expected: ~596,049,920 (596M-branded, "0.6B")' and SmolLM2-134(base)/model_full.py:314 prints 'Expected: ~134,515,008 (135M-branded)'. I did not execute either module, so I am reporting those as literal source strings, not as verified parameter counts. Also note the `only_trainable` logic reads `p.requires_grad or not only_trainable`, which counts ALL params when only_trainable=False and only requires_grad ones when True — correct, but the condition is inverted-looking. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Qwen3-0.6B/model.py:298-299 and SmolLM2-134(base)/model_full.py:304-305 are byte-identical two-line definitions matching the quote. Hardcoded expected strings CONFIRMED as literal source text: model.py:306 `print(f"Expected: ~596,049,920 (596M-branded, '0.6B')")` (the fact's caveat renders the inner quotes as double — the source uses single quotes; cosmetic only) and model_full.py:314 `print(f"Expected: ~134,515,008 (135M-branded)")`. The caveat's honesty about not having executed either module is correct and should be preserved on any card: those two counts are prose-in-code, not values I re-computed. +``` + + +### 7.21 Config fields that are declared but never read (traps for a published snippet) + +**Value** + +``` +attention_dropout is declared in BOTH configs (Qwen3-0.6B/model.py:50, SmolLM2-134(base)/model_full.py:42) but is never referenced anywhere else in either file — attention hardcodes dropout_p=0.0. Setting it has NO effect. +``` + +**Evidence** — `Qwen3-0.6B/model.py:162-167` + +**Source quote** + +``` +out = F.scaled_dot_product_attention( + q, k, v, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=(attention_mask is None), + ) +``` + +**Confidence** — measured from code + +**Caveat** — Verified by `grep -n "attention_dropout" Qwen3-0.6B/model.py "SmolLM2-134(base)/model_full.py"` — the only hits are the two dataclass declaration lines; there is no read site. Same conclusion for both models. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +`grep -n attention_dropout Qwen3-0.6B/model.py "SmolLM2-134(base)/model_full.py"` returns exactly two hits total — Qwen3-0.6B/model.py:50 and SmolLM2-134(base)/model_full.py:42, both dataclass declarations, zero read sites. Quoted SDPA block = Qwen3-0.6B/model.py:162-167 verbatim, with `dropout_p=0.0` hardcoded at :165; the SmolLM2 twin is model_full.py:156-161. Setting attention_dropout has no effect in either model. +``` + + +### 7.V Additional verifier findings (no 1:1 extracted fact) + +**7.V1 — ⚠️ NEEDS QUALIFIER** · [GAP CHECK] Qwen3-0.6B has no standalone end-to-end usage script; top-level listing + +**Checked against** + +``` +ls Qwen3-0.6B/ shows only model.py, verify.py, make_phase_plots.py at top level +``` + +**Verifier note** + +``` +The load-bearing part is CONFIRMED: those are the only three top-level .py files, and there is no generate.py equivalent. But the listing is incomplete as stated — `ls -1 Qwen3-0.6B/` also shows builds/, experiments/, results_overview/, __pycache__/, README.md and PLOTS_INDEX.md. The `.generate(` grep is exactly reproducible: 9 files, all under experiments/*/eval_suite.py (6 of them), builds/2026-06-08_reproduce-faithful_qwen3-0.6b/test_model.py, builds/.../train_qwen3.py, and experiments/2026-06-27_qwen3-0.6b_sft-3seed/eval_suite.py. +``` + + +**7.V2 — ✅ CONFIRMED** · [GAP CHECK] No __init__.py in either model folder (flat-module imports required) + +**Checked against** + +``` +ls Qwen3-0.6B/__init__.py 'SmolLM2-134(base)/__init__.py' -> No such file +``` + +**Verifier note** + +``` +Reproduced exactly: `ls: cannot access 'Qwen3-0.6B/__init__.py': No such file or directory` and the same for 'SmolLM2-134(base)/__init__.py'. Neither folder is an importable package, so every published snippet must set cwd to the model folder or sys.path.insert it (repo idiom at Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:55-57). Additional friction worth stating on a card: the SmolLM2 folder name contains parentheses — 'SmolLM2-134(base)' — which must be quoted in any shell cd. +``` + + +### 7.G Gaps — not determinable from disk + +- Qwen3-0.6B has NO standalone end-to-end usage script equivalent to SmolLM2-134(base)/generate.py. `ls /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/` shows only model.py, verify.py, make_phase_plots.py at top level; `grep -rln '\.generate(' --include=*.py .` under Qwen3-0.6B/ returns only experiment eval_suite.py files, builds/.../test_model.py and builds/.../train_qwen3.py. So the Qwen3 usage snippet must be composed from verify.py + an experiment eval_suite.py, which I flagged inline. +- Neither model class exposes a from_pretrained / save_pretrained classmethod, and neither folder has an __init__.py (`ls Qwen3-0.6B/__init__.py 'SmolLM2-134(base)/__init__.py'` -> No such file). There is therefore no importable package API: every published snippet must either set cwd to the model folder or do a sys.path.insert, and I could not find any repo-provided convenience wrapper that hides this. +- I did not EXECUTE either model file, verify.py, or any eval script during this task — no GPU/model run was performed. Every claim here is static source reading. Therefore the parameter counts, the max|Δlogits| tolerance actually achieved, and whether the HF repos are present in the local HF cache are all unverified by me. +- No checkpoint-to-config reconstruction path exists on disk. train scripts save `"config": cfg.__dict__` (train_qwen3.py:353, SmolLM2 train.py:184) but I found no loader anywhere that reads it back — all load sites call Qwen3Config()/SmolLM2Config() with defaults. If a checkpoint were ever trained with non-default config, no repo code would detect the mismatch beyond load_state_dict shape errors. +- I could not determine from disk whether the `torch_dtype=` (SmolLM2 verify.py:53, generate.py:16) vs `dtype=` (Qwen3 verify.py:51, SmolLM2 compare_with_hf.py:45 / eval_after_vs_base.py:35 / tests/test_parity.py:39) split is deliberate or just file-age drift — there is no comment or requirements pin explaining it. SmolLM2-134(base)/requirements.txt exists (62 bytes) but I did not read it as part of this dimension. + +--- + +## 8. Checkpoint inventory on disk + +Audit dimension: checkpoint inventory on disk (what can actually be uploaded as weights) + +### 8.1 How many weights-bearing checkpoint files exist under the repo (>1MB), and what is their total size? + +**Value** + +``` +107 real (non-symlink) weights-bearing files, 290,643,004,043 bytes = 270.68 GiB = 290.64 GB. Extensions found: only .pt (PyTorch) and .pkl (JAX/Flax) plus one .discarded_* suffix file. ZERO .safetensors, .msgpack, .npz, .pth, .ckpt files exist anywhere in the repo. +``` + +**Evidence** — `HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/ , Qwen3-0.6B/ , SmolLM2-134(base)/ (measured via os.walk + os.path.getsize, symlinks excluded)` + +**Source quote** + +``` +weights n=107 bytes=290,643,004,043 GiB=270.68 GB=290.64 +``` + +**Confidence** — measured from code + +**Caveat** — The find command in the task brief double-counts: 12 of the paths it returns are SYMLINKS to other runs' checkpoints (§C13 control-reuse), and 26 more matches named tokcache_*.pt are TOKENIZED DATA CACHES, not weights, plus 10 research/datasets/*.bin are raw uint16 token shards. Naive summing gives ~334 GiB; the true unique weights figure is 270.68 GiB. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Headline is exact and reproduces. Caveat correction: the 10 research/datasets/*.bin shards are uint32, NOT uint16. +``` + +**Verifier note** + +``` +RE-MEASURED independently with os.walk + os.path.getsize (symlinks excluded, >1MB): weights n=107 bytes=290,643,004,043 -> 270.68 GiB / 290.64 GB. EXACT match. Extension split re-measured: .pt n=89 (215.08 GiB), .pkl n=17 (52.51 GiB), .discarded_* n=1 (3.10 GiB) = 107. `find` for *.safetensors/*.msgpack/*.npz/*.pth/*.ckpt (excluding .git) returned nothing. Symlink count 12 confirmed. Naive-sum '~334 GiB' also checks out (315,669,745,533 B + ~12x3.58 GB dereferenced symlinks = ~358.6 GB = ~334 GiB). ERROR IN CAVEAT: 'raw uint16 token shards' is wrong. research/datasets/data-selection-dclm-edu/prepare_dclm_edu.py:94 reads `arr = np.array(train_buf, dtype=np.uint32)` and :125 `np.array(eval_toks, dtype=np.uint32).tofile(...)`; research/datasets/data-selection-dclm-edu/meta.json:5 `"dtype": "uint32"`; research/datasets/math-reasoning-openr1-math-220k/meta.json:2 `"dtype": "uint32"`. uint16 is physically impossible here — vocab is 151,936 > 65,535. +``` + + +### 8.2 What is the total disk footprint of everything the find pattern matches (weights + token caches + dataset shards)? + +**Value** + +``` +315,669,745,533 bytes = 293.99 GiB = 315.67 GB total, split: weights 290.64 GB (n=107), tokcache_*.pt token caches 23.92 GB (n=26), research/datasets/*.bin shards 1.11 GB (n=10). Volume has 2.5T free of 3.7T. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/ (tokcache files), research/datasets/` + +**Source quote** + +``` +tokcache n= 26 bytes=23,921,715,722 GiB=22.28 GB=23.92 +dataset n= 10 bytes=1,105,025,768 GiB=1.03 GB=1.11 +``` + +**Confidence** — measured from code + +**Caveat** — tokcache_*.pt are torch.save'd token-id tensors produced by train_qwen3.py, NOT model weights. The largest single .pt in the repo (9,534,229,373 B = 9.09 GiB, tokcache_1191478400_300000.pt) is a token cache, not a model. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Totals exact. But the largest .pt is 9,534,229,373 B = 8.88 GiB = 9.53 GB, NOT 9.09 GiB. +``` + +**Verifier note** + +``` +Re-measured: total 315,669,745,533 B = 293.99 GiB = 315.67 GB EXACT; tokcache n=26 = 23,921,715,722 B (22.28 GiB / 23.92 GB) EXACT; dataset n=10 = 1,105,025,768 B (1.03 GiB / 1.11 GB) EXACT. `df -h` on /dev/nvme0n1p2: 3.7T size, 2.5T avail, 30% used — CONFIRMED. Largest .pt confirmed as Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/tokcache_1191478400_300000.pt at 9,534,229,373 B, but 9,534,229,373 / 2^30 = 8.879 GiB and / 1e9 = 9.534 GB — the quoted '9.09 GiB' matches neither unit. Producer confirmed: Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/train_qwen3.py:145 `cache = RESULTS / f"tokcache_{n_train}_{n_val}_seed{seed}_{tok_tag}.pt"`. Minor: the shorthand 'research/datasets/*.bin' matches zero files literally — they live at research/datasets//shard_*.bin and .../eval/*.bin. +``` + + +### 8.3 Per-run-directory breakdown of the weights footprint + +**Value** + +``` +18 files 55.60 GiB HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build; 17 files 56.64 GiB Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1; 21 files 23.32 GiB Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results; 9 files 29.99 GiB 2026-06-21_qwen3-0.6b_arch-subdrill-p2; 6 files 19.99 GiB 2026-06-27_qwen3-0.6b_sft-3seed; 6 files 19.99 GiB 2026-06-30_qwen3-0.6b_midtrain-anneal; 5 files 16.66 GiB Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b; 11 files 12.22 GiB builds/2026-06-08_reproduce-modernized.../results; 3 files 9.99 GiB 2026-06-24_data-dclm-vs-fineweb; 3 files 9.99 GiB 2026-06-26_data-mix-composition; 3 files 9.99 GiB 2026-07-02_grpo-phase2; 1 file 3.33 GiB 2026-06-17_vibethinker.../results; 2 files 2.22 GiB builds/2026-06-08_reproduce-exploratory.../results; 2 files 0.75 GiB SmolLM2-134(base) +``` + +**Evidence** — `Qwen3-0.6B/experiments/ , HybridSSM-0.2B/experiments/ , SmolLM2-134(base)/` + +**Source quote** + +``` +18 files 55.60 GiB HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build + 21 files 23.32 GiB Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results + 17 files 56.64 GiB Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1 +``` + +**Confidence** — measured from code + +**Caveat** — The HybridSSM arch-ladder run (2026-07-21_..._arch-ladder) has ZERO checkpoints in its own directory — run_arch_ladder.sh cd's into the 2026-07-19 build dir and writes them there. Likewise the scaling-persistence ladder writes into the normuon-vs-adamw results dir. Directory name does not equal owning run. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +All 14 rows reproduce EXACTLY from an independent os.walk aggregation (file counts and GiB to 2dp). Caveat also verified: HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/run_arch_ladder.sh:19 `BUILD="$ROOT/HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build"` and :392 `( cd "$BUILD" && $PY train_hybrid.py ...` with :395 `--ckpt "checkpoint_${id}.pkl"` — the ladder writes into the 2026-07-19 dir. Likewise Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh:83 `RESULTS=$IMU1/results` points at the 2026-06-16 dir. +``` + + +### 8.4 Is there a config.json anywhere alongside the checkpoints? + +**Value** + +``` +NOT_FOUND. Zero config.json files exist in the entire repo (excluding .git). Architecture config is embedded INSIDE each checkpoint dict under the key 'config' (a plain dict of the dataclass fields), not as a sidecar file. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/checkpoint_persist_420M_normuon_s0.pt (in-file key 'config')` + +**Source quote** + +``` +config: { + "vocab_size": 151936, + "hidden_size": 1024, + "intermediate_size": 3072, + "num_hidden_layers": 28, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 128, + "max_position_embeddings": 40960, + "rope_theta": 1000000.0, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "tie_word_embeddings": true, + "attention_bias": false, + "attention_dropout": 0.0 +} +``` + +**Confidence** — measured from code + +**Caveat** — Searched with: find . -name 'config.json' -not -path './.git/*' — returned nothing. Any HF upload must synthesize config.json from the in-checkpoint dict. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +`find . -path ./.git -prune -o -name 'config.json' -print` returned nothing. I loaded Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/checkpoint_persist_420M_normuon_s0.pt (torch.load map_location='cpu', weights_only=True) and its ck['config'] is byte-for-byte the quoted dict: vocab_size 151936, hidden_size 1024, intermediate_size 3072, num_hidden_layers 28, num_attention_heads 16, num_key_value_heads 8, head_dim 128, max_position_embeddings 40960, rope_theta 1000000.0, rms_norm_eps 1e-06, initializer_range 0.02, tie_word_embeddings true, attention_bias false, attention_dropout 0.0. EXACT match. +``` + + +### 8.5 Are tokenizer files vendored in the repo, or downloaded from HF at runtime? + +**Value** + +``` +NOT vendored — downloaded from HF Hub at runtime. Zero tokenizer.json / tokenizer_config.json / vocab.json / merges.txt / *.model / special_tokens_map.json / generation_config.json exist anywhere in the repo, and no HF cache dir lives inside it. Qwen3 + HybridSSM use AutoTokenizer.from_pretrained('Qwen/Qwen3-0.6B-Base'); SmolLM2 uses 'HuggingFaceTB/SmolLM2-135M'. +``` + +**Evidence** — `Qwen3-0.6B/verify.py:19 ; SmolLM2-134(base)/verify.py:19 ; HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/eval_suite_jax.py:87` + +**Source quote** + +``` +Qwen3-0.6B/verify.py:19: REPO = "Qwen/Qwen3-0.6B-Base" +SmolLM2-134(base)/verify.py:19: REPO = "HuggingFaceTB/SmolLM2-135M" +eval_suite_jax.py:87: TOKENIZER_REPO = "Qwen/Qwen3-0.6B-Base" # model's OWN tokenizer (data cache built with it) +``` + +**Confidence** — measured from code + +**Caveat** — HybridSSM-0.2B is a NOVEL from-scratch architecture that nonetheless uses the Qwen3 tokenizer (vocab_size 151,936 in HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py:22). Any weights upload inherits Qwen's tokenizer licensing, and reproduction requires network access to HF. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Tokenizer-file absence and all three line citations are correct. But 'no HF cache dir lives inside it' is false: research/datasets/data-selection-dclm-edu/.raw/.cache/huggingface/ exists (28 KB). +``` + +**Verifier note** + +``` +All three cited lines verified verbatim at the exact line numbers: Qwen3-0.6B/verify.py:19, SmolLM2-134(base)/verify.py:19, HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/eval_suite_jax.py:87 (comment text matches too). find for tokenizer.json/tokenizer_config.json/vocab.json/merges.txt/*.model/special_tokens_map.json/generation_config.json returned ZERO. HybridSSM vocab_size 151_936 confirmed at HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py:22. HOWEVER an HF cache directory DOES exist inside the repo: research/datasets/data-selection-dclm-edu/.raw/.cache/huggingface/{download/data/000_00000.parquet.metadata, CACHEDIR.TAG, .gitignore}. It holds dataset-download metadata only — no tokenizer or model files — so the substantive conclusion (tokenizer must be fetched from the Hub) survives, but the blanket 'no HF cache dir' wording must be dropped. +``` + + +### 8.6 What is the on-disk format and top-level structure of the Qwen3 checkpoints? + +**Value** + +``` +Raw torch.save'd Python dicts (NOT safetensors, NOT bare state_dicts). Two variants: (a) WEIGHTS-ONLY ~1,192,232,687 B / 1137 MiB — keys ['model','config','step','tok_seen','arm','seed','fineweb_val_ppl','baseline_ppl','recipe']; (b) FULL TRAINING STATE ~3,576,xxx,xxx B / 3411 MiB — adds ['optim','sched','rng_torch','rng_cuda','rng_numpy','rng_python']. model is a 311-key state_dict, all torch.bfloat16, 751,632,384 tensor elements including the tied lm_head.weight duplicate = 596,049,920 unique params. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/checkpoint_persist_420M_normuon_s0.pt ; Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/checkpoint_sft_seed0.pt` + +**Source quote** + +``` +checkpoint_persist_420M_normuon_s0.pt | keys= ['model','config','step','tok_seen','arm','seed','fineweb_val_ppl','baseline_ppl','recipe'] + model: dict len=311 ... model.embed_tokens.weight: Tensor (151936, 1024) torch.bfloat16 + total(incl tied dup): 751632384 minus embed: 596049920 +checkpoint_sft_seed0.pt keys: ['model','config','step','sample_cursor','tok_seen','base_reasoning_ppl','recipe','optim','sched','rng_torch','rng_cuda','rng_numpy','rng_python'] +``` + +**Confidence** — measured from code + +**Caveat** — The 3.4 GB variants FAIL torch.load(weights_only=True) — they pickle numpy RNG state (UnpicklingError: 'Unsupported global: GLOBAL numpy._core.multiarray._reconstruct'). I loaded them under torch.serialization.safe_globals([numpy._core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.UInt32DType]) with weights_only=True still enforced. The 1.1 GB weights-only variants load cleanly with weights_only=True. Any external consumer must be warned they are pickles, not safetensors. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Verified by loading four files CPU-only. checkpoint_persist_420M_normuon_s0.pt: 1,192,233,319 B (=1137.0 MiB), keys exactly the (a) list, 311 keys, all torch.bfloat16, 751,632,384 elems, minus one embed = 596,049,920. checkpoint_sft_seed0.pt (Qwen3-0.6B/experiments/2026-06-27_qwen3-0.6b_sft-3seed/): 3,576,718,301 B (=3411.0 MiB), keys ['model','config','step','sample_cursor','tok_seen','base_reasoning_ppl','recipe','optim','sched','rng_torch','rng_cuda','rng_numpy','rng_python'] — EXACT match to the quote. Raw weights_only=True raised UnpicklingError whose text I captured verbatim: 'WeightsUnpickler error: Unsupported global: GLOBAL numpy._core.multiarray._reconstruct was not an allowed global by default.' Loaded successfully under torch.serialization.safe_globals([...]) with weights_only=True still on. +``` + + +### 8.7 What is the on-disk format and structure of the HybridSSM (JAX/Flax) checkpoints? + +**Value** + +``` +Python pickle wrapping flax msgpack byte-strings, written by train_hybrid.py:132-139. Top-level dict keys: {'params': bytes (flax msgpack), 'opt_state': bytes (flax msgpack, exactly 2x the params size), 'step': int, and (post-fix only) 'rng': numpy uint32[2]}. Params deserialize (flax.serialization.msgpack_restore) to a nested dict keyed block_0..block_23 + embed + norm_f, all float32, weight-tied (no separate lm_head). +``` + +**Evidence** — `HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/train_hybrid.py:132-139` + +**Source quote** + +``` +def save(step_i): + blob = {"params": serialization.to_bytes(params), "opt_state": serialization.to_bytes(opt_state), + "step": step_i, "rng": np.asarray(rng)} + tmp = a.ckpt + ".tmp" + with open(tmp, "wb") as f: + pickle.dump(blob, f) +``` + +**Confidence** — measured from code + +**Caveat** — Two-thirds of every HybridSSM file is Adam optimizer state, not weights. E.g. checkpoint_ssm_base_42M_s0.pkl is 3,669,849,670 B total but params is only 1,223,283,175 B. Stripping opt_state before upload cuts the family from 55.6 GiB to roughly 18-19 GiB. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Structure fully confirmed. Params-only sum is 18,792,709,255 B = 17.50 GiB (18.79 GB), not '18-19 GiB' — a GiB/GB slip against the 55.60 GiB baseline. +``` + +**Verifier note** + +``` +train_hybrid.py save() verified at lines 132-139 (132 `def save(step_i):`, 135-136 blob, 137 tmp, 138 open, 139 pickle.dump) — NOTE the quote silently elides the two comment lines 133-134, so it is not a contiguous verbatim excerpt. Deserialized 4 files with flax.serialization.msgpack_restore: top-level tree has n=26 (block_0..block_23 + embed + norm_f), all float32, single tied '/embed' (151936,768) with no separate lm_head. opt_state_bytes is exactly 2x params_bytes in all four (e.g. ssm_base_42M_s0: 1,223,283,175 -> 2,446,566,418). checkpoint_ssm_base_42M_s0.pkl total 3,669,849,670 B and params 1,223,283,175 B — both EXACT. Summing params_bytes over all 18 pkl-family files gives 18,792,709,255 B = 17.50 GiB. +``` + + +### 8.8 Exact parameter counts of the HybridSSM arms (measured by deserializing the flax tree) + +**Value** + +``` +ssm_base = 305,818,368 params (266 leaves); attn1to3 = 324,867,840 (290 leaves); fullattn = 267,719,424 (218 leaves); swa128 = 277,156,608 (194 leaves). All float32, tied embedding (151,936 x 768 = 116,686,848 of ssm_base's total). Architecture: d_model 768, n_layers 24, n_heads 12, n_kv_heads 4, vocab 151,936. +``` + +**Evidence** — `HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py:21-26` + +**Source quote** + +``` +class HybridConfig: + vocab_size: int = 151_936 + d_model: int = 768 + n_layers: int = 24 + n_heads: int = 12 + n_kv_heads: int = 4 +``` + +**Confidence** — measured from code + +**Caveat** — The folder is named 'HybridSSM-0.2B' but every arm on disk is 267M-325M total params. 0.2B appears to refer to non-embedding params (ssm_base: 305,818,368 - 116,686,848 = 189,131,520). I did NOT find a file stating which convention '-0.2B' uses — do not publish '0.2B' as a total-parameter claim without resolving it. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Independently deserialized checkpoint_{ssm_base,attn1to3,fullattn,swa128}_42M_s0.pkl and counted leaves/elements: 305,818,368/266; 324,867,840/290; 267,719,424/218; 277,156,608/194 — ALL EXACT. All float32; '/embed' is (151936, 768) = 116,686,848 in every arm with no lm_head leaf. HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/model.py:21-26 quoted verbatim and correct. Non-embedding arithmetic 305,818,368 - 116,686,848 = 189,131,520 verified. The '-0.2B' naming caveat is properly flagged as unresolved — I also found no file on disk stating the convention. +``` + + +### 8.9 CRITICAL: do the '42M' / '85M' / '168M' / '420M' filename tokens mean parameters? + +**Value** + +``` +NO — they are TOKEN BUDGETS, not parameter counts. HybridSSM cells.json records rung_base_tokens 42000000 / 85000000 / 150000000. Qwen3 checkpoint_persist_168M_adamw_s0.pt carries tok_seen=168,034,304 with 596,049,920 params; checkpoint_persist_420M_normuon_s0.pt carries tok_seen=420,020,224 with the same 596,049,920 params. +``` + +**Evidence** — `HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/cells.json` + +**Source quote** + +``` +{'id': 'ssm_base_42M_s0', 'arm': 'ssm_base', 'seed': 0, 'rung_base_tokens': 42000000, 'tokens': 42000000, 'steps': 5126, ...} +``` + +**Confidence** — measured from code + +**Caveat** — This is the single easiest thing to get wrong when writing a model card. Every 'persist_*' Qwen3 checkpoint is the SAME 596M-param model at a different token budget. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/cells.json cells[0] is byte-identical to the quote: {'id': 'ssm_base_42M_s0', 'arm': 'ssm_base', 'seed': 0, 'rung_base_tokens': 42000000, 'tokens': 42000000, 'steps': 5126, ...}. The full set of (rung_base_tokens, tokens) pairs across its 15 cells is {(42000000,42000000),(42000000,48000000),(85000000,85000000),(85000000,96000000),(150000000,150000000),(150000000,170000000)} — the three rungs 42M/85M/150M confirmed. Loaded checkpoint_persist_168M_adamw_s0.pt: step=2564, tok_seen=168034304, 751,632,384 elems -> 596,049,920 unique. checkpoint_persist_420M_normuon_s0.pt: step=6409, tok_seen=420020224, identical param count. This is the single most important fact in the set and it is fully backed. +``` + + +### 8.10 Which checkpoint corresponds to the parity-verified (bit-exact) Qwen3-0.6B / SmolLM2 reproduction? + +**Value** + +``` +NOT_FOUND — no such checkpoint exists on disk, by design. Both verify.py scripts download the official HF weights at runtime and load them into the repo's model.py; nothing is saved. The parity EVIDENCE that exists on disk is Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/verify.json (max_abs_error 0.0, argmax_match true, passed true) and SmolLM2-134(base)/results/parity.log (max |Δlogits| = 0.000e+00). +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/verify.json` + +**Source quote** + +``` +{ + "repo": "Qwen/Qwen3-0.6B-Base", + "max_abs_error": 0.0, + "relative_error": 0.0, + "hf_next_token_id": 12095, + "our_next_token_id": 12095, + "argmax_match": true, + "passed": true +} +``` + +**Confidence** — results JSON + +**Caveat** — PUBLISHING IMPLICATION: there is nothing to upload for 'the parity-verified repro' — the weights are Qwen's / HuggingFaceTB's, already on the Hub. What is publishable is the code + the parity artifact, not a weights file. Do not describe any .pt in this repo as 'the bit-exact reproduction weights'. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +verify.json read in full — quoted fields are exact (repo 'Qwen/Qwen3-0.6B-Base', max_abs_error 0.0, relative_error 0.0, hf_next_token_id 12095, our_next_token_id 12095, argmax_match true, passed true; also prompt 'The capital of France is', dtype float32, tolerance 0.001). SmolLM2-134(base)/results/parity.log contains 'max |Δlogits| = 0.000e+00' and 'relative = 0.000e+00' and '✓ Architecture parity verified.' Both verify.py scripts fetch weights at runtime (Qwen3-0.6B/verify.py:51 AutoModelForCausalLM.from_pretrained(REPO); SmolLM2-134(base)/verify.py:53 same) and save nothing. The publishing implication is correctly stated. +``` + + +### 8.11 Which checkpoint is the continued-pretrained SmolLM2? + +**Value** + +``` +SmolLM2-134(base)/checkpoint_tinystories.pt — 269,144,681 B (256.68 MB), mtime 2026-05-14 00:21. bf16, 273-key state_dict, 134,515,008 unique params (162,826,560 incl. tied lm_head duplicate). In-file: step=24414, tok_seen=99,999,744, baseline_ppl=6.894546783281595, trained_ppl=3.7899503859716885. Produced by train_tinystories.py, which initializes FROM the official HF weights (line 39 imports load_official_weights_into_ours from verify.py; line 145 loads the HF model). +``` + +**Evidence** — `SmolLM2-134(base)/train_tinystories.py:39,145` + +**Source quote** + +``` +39:from verify import REPO, load_official_weights_into_ours +145: hf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32) +154: train_ds = load_dataset("roneneldan/TinyStories", split="train") +``` + +**Confidence** — measured from code + +**Caveat** — This is a DERIVATIVE of HuggingFaceTB/SmolLM2-135M (Apache-2.0 upstream) — publishing it is a fine-tune release, and the model card must say so. The in-checkpoint numbers (6.8945 -> 3.7893) match SmolLM2-134(base)/results/tinystories_summary.md exactly, so that prose is backed. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +Every stated value is exact, but the caveat's 'match exactly' is false: in-checkpoint trained_ppl = 3.78995 (rounds to 3.7900); results/tinystories_summary.md:9 reports 3.7893. Δ = 0.00065. +``` + +**Verifier note** + +``` +Loaded the file: size 269,144,681 B, mtime 2026-05-14 00:21:27, keys ['model','config','step','tok_seen','baseline_ppl','trained_ppl'], 273 keys all torch.bfloat16, 162,826,560 elems - one embed (49152x576) = 134,515,008. step=24414, tok_seen=99999744, baseline_ppl=6.894546783281595, trained_ppl=3.7899503859716885 — ALL EXACT. Provenance lines verified verbatim: train_tinystories.py:39 `from verify import REPO, load_official_weights_into_ours`, :145 `hf = AutoModelForCausalLM.from_pretrained(REPO, dtype=torch.float32)`, :154 `train_ds = load_dataset("roneneldan/TinyStories", split="train")`. BUT tinystories_summary.md:9 reads `| TinyStories-val perplexity | **6.8945** | **3.7893** | **−45.0%** |` — baseline matches to 4dp, trained does NOT (3.7893 vs 3.78995). grep for '3.789' across the subproject found only that one line, so there is no second file carrying 3.78995. A model card must not claim the prose is byte-backed. +``` + + +### 8.12 What is SmolLM2-134(base)/checkpoint.pt (the other SmolLM2 file)? + +**Value** + +``` +A FROM-SCRATCH random-init toy run — NOT a reproduction and NOT publishable as SmolLM2 weights. 538,173,921 B (513.24 MB), mtime 2026-05-13 22:20, float32, step=150, 150 loss values. train.py:140 prints 'Initializing model from scratch (random init)' and trains on wikitext-103-raw-v1 (train.py:78). +``` + +**Evidence** — `SmolLM2-134(base)/train.py:78,140` + +**Source quote** + +``` +78: ds = load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train") +140: print("Initializing model from scratch (random init)...") +``` + +**Confidence** — measured from code + +**Caveat** — 150 steps of random init is a demo artifact. It has no 'training_recipe' key even though the current train.py:182 save_ckpt writes one — i.e. the file predates the current script (ckpt mtime 2026-05-13 22:20 vs train.py mtime 2026-05-19 23:17). Same script-drift applies to checkpoint_tinystories.pt (2026-05-14 vs train_tinystories.py 2026-05-19). Provenance is not byte-reproducible from HEAD. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Loaded: 538,173,921 B, mtime 2026-05-13 22:20:18, keys ['model','config','losses','lrs','step'], step=150, len(losses)=150, all torch.float32, 273 keys, 134,515,008 unique params. 'training_recipe' absent — CONFIRMED. Dataset config check (the classic -raw- trap): SmolLM2-134(base)/train.py:78 literally reads `ds = load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split="train")` — the claimed config name is the one the script actually names. train.py:140 `print("Initializing model from scratch (random init)...")` verbatim. train.py:181 `def save_ckpt(step: int):` / :182 `torch.save({` / :186 `"training_recipe": {` — the recipe-writing claim is right (the block starts at 182, the key at 186). Script-drift dates confirmed by ls: train.py 2026-05-19 23:17, train_tinystories.py 2026-05-19 23:16, checkpoints 2026-05-13/14. +``` + + +### 8.13 Which checkpoints back the headline NorMuon-vs-AdamW 'win' (2026-06-16, verdict=win)? + +**Value** + +``` +6 files in Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/: checkpoint_{adamw,normuon}_seed{0,1,2}.pt, each 1,192,229,527-1,192,230,159 B (1137.00 MB), mtime 2026-06-17 01:20-09:46. Verified in-file: step=640, tok_seen=41,943,040, arm='normuon'/'adamw', seed=0..2. checkpoint_normuon_seed0 fineweb_val_ppl=61.3435; checkpoint_adamw_seed0 fineweb_val_ppl=147.4181. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/checkpoint_normuon_seed0.pt (in-file keys)` + +**Source quote** + +``` +checkpoint_normuon_seed0.pt | step 640 tok_seen 41943040 arm normuon seed 0 ppl 61.34354760675183 +checkpoint_adamw_seed0.pt | step 640 tok_seen 41943040 arm adamw seed 0 ppl 147.41809644622612 +``` + +**Confidence** — measured from code + +**Caveat** — The ledger's headline for this run is BPB not the in-file fineweb PPL: wikitext_bpb_normuon_mean 1.6355 vs adamw 2.1098 (research/ledger/ledger.json, run 2026-06-16_qwen3_normuon-vs-adamw). Also present in the same directory: 3 LR-sweep checkpoints (checkpoint_adamw_lr{17,35,48}_seed0.pt) that are sweep artifacts, not arms. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +ls -l on the results dir: adamw_seed{0,1,2} all 1,192,229,527 B at 2026-06-17 01:20/02:54/04:29; normuon_seed{0,1,2} all 1,192,230,159 B at 06:14/08:01/09:46 — size range and mtime span EXACT. Loaded both seed0 files: step=640, tok_seen=41943040, arm='normuon'/'adamw', seed=0, fineweb_val_ppl 61.34354760675183 and 147.41809644622612 — EXACT. Ledger run 2026-06-16_qwen3_normuon-vs-adamw (status done, verdict win) metrics: wikitext_bpb_adamw_mean 2.1098, wikitext_bpb_normuon_mean 1.6355, wikitext_improvement_bpb 0.4743, wikitext_ci95 [0.4435,0.5052], code_improvement_bpb 0.5016 — the caveat's BPB pair is EXACT. checkpoint_adamw_lr{17,35,48}_seed0.pt confirmed present (1,192,231,107 B each, 2026-06-17 13:51/15:28/17:03). +``` + + +### 8.14 Which checkpoints back the scaling-persistence ladder (verdict=null, NorMuon win converges)? + +**Value** + +``` +12 files in Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/: checkpoint_persist_{168M,420M}_{adamw,normuon}_s{0,1,2}.pt, each 1,192,232,687 / 1,192,233,319 B (1137.00 MB), mtimes 2026-07-06 to 2026-07-26. Together with the 6 x 42M cohort files they are the 18 cells the scorer reads. ladder_bpb.json names all 18 by filename. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence/run_ladder_scale_ext.sh:80-83` + +**Source quote** + +``` +IMU1=$ROOT/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw +LDIR=$ROOT/Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence +TRAIN=$IMU1/train_ablation.py +RESULTS=$IMU1/results +``` + +**Confidence** — measured from code + +**Caveat** — Directory/run mismatch: the run_id is 2026-07-05_qwen3-0.6b_scaling-persistence and 2026-07-23_qwen3-0.6b_normuon-at-scale, but the CHECKPOINTS live under the 2026-06-16 experiment. The 2026-07-05 dir contains only .done markers, logs and c5_evidence. The ledger entry for normuon-at-scale explicitly records absolute checkpoint paths pointing back to .../2026-06-16_qwen3_normuon-vs-adamw/results/. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +ls confirms exactly 12 persist_* files: adamw variants 1,192,232,687 B, normuon variants 1,192,233,319 B; mtime span 2026-07-06 00:36 to 2026-07-26 13:18. A regex sweep of Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/ladder_bpb.json returned exactly 18 unique checkpoint_*.pt filenames — the 12 persist plus checkpoint_{adamw,normuon}_seed{0,1,2}.pt. run_ladder_scale_ext.sh lines 80-83 are verbatim at those exact line numbers (ROOT is line 79). Ledger run 2026-07-05_qwen3-0.6b_scaling-persistence is status=done verdict=null with trend_verdict_wikitext=CONVERGES and trend_code_py=CONVERGES. Minor imprecision: the 2026-07-05 directory holds more than 'only .done markers, logs and c5_evidence' — it also contains verdict.json, run_ladder_scale_ext.sh, boot_resume.sh, RESUME_STATE.md, thermal_log.py. The load-bearing claim (zero checkpoints there) is correct. +``` + + +### 8.15 Which checkpoints are flagged DISCARDED / CONFOUNDED and must not be published? + +**Value** + +``` +(1) HARD-QUARANTINED, 1 file: HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/checkpoint_swa128_nope_85M_s0.pkl.discarded_rng_confound_20260723 (3,325,901,386 B, mtime 2026-07-23 15:09) — renamed out of the .pkl namespace for an RNG-restore confound. (2) COMPARABILITY-VOID (iso-FLOP error, +18.88% extra compute), 3 completed files: checkpoint_swa128_42M_s0.pkl, checkpoint_swa128_nope_42M_s0.pkl, checkpoint_swa128_85M_s0.pkl. Their replacements are checkpoint_swa128_42M_isofix_s0.pkl and checkpoint_swa128_nope_42M_isofix_s0.pkl (step 4929, mtimes 2026-07-29/30). +``` + +**Evidence** — `HybridSSM-0.2B/experiments/2026-07-21_hybrid-ssm-0.2b_arch-ladder/c5_evidence_CORRECTION_2026-07-28.md:303-307,326-330` + +**Source quote** + +``` +**Affected — comparability void:** +- `swa128_42M_s0`, `swa128_nope_42M_s0` (both complete, `.done` on disk) +- `swa128_85M_s0` (complete, 11,718 steps — same budget error at the 85M rung ...) +- `swa128_nope_85M_s0` (killed at ~step 7,960 ... checkpoint already discarded for a separate RNG confound) +... +**The words "iso-FLOP" must not be attached to `swa128` or `swa128_nope` in any artifact until those cells are re-run at 4,929 steps / 40,378,368 tokens** +``` + +**Confidence** — measured from code + +**Caveat** — There is a SECOND, undocumented split I found by inspecting file tails: 10 HybridSSM pickles lack the 'rng' key (pre-PRNG-fix) and 8 carry it. Pre-fix (no rng): checkpoint.pkl(smoke), ssm_base_s0, ssm_base_42M_s0, ssm_base_85M_s0, swa128_42M_s0, swa128_85M_s0, swa128_nope_42M_s0, attn1to3_42M_s0, fullattn_42M_s0, and the quarantined 85M. Post-fix (has rng): ssm_base_42M_s1/s2, attn1to3_42M_s1/s2, fullattn_42M_s1/s2, swa128_42M_isofix_s0, swa128_nope_42M_isofix_s0. The quarantine rationale (RNG confound) applies structurally to every pre-fix file that was RESUMED; only the 85M one was actually quarantined. Flag this to the user before publishing any pre-fix seed-0 arm. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +This is the strongest fact in the set. The quarantined file is 3,325,901,386 B at mtime 2026-07-23 15:09 — EXACT. c5_evidence_CORRECTION_2026-07-28.md quote verified verbatim: line 302 '**Affected — comparability void:**', 303-307 the swa128 list, 317-318 '+18.88 % extra compute relative to the base arm', 326-328 'The words "iso-FLOP" must not be attached to `swa128` or `swa128_nope` in any artifact until those cells are re-run at 4,929 steps / 40,378,368 tokens'. (Cited range 303-307 actually starts at 302; 326-330 ends at 328 — trivial off-by-one.) I independently unpickled all 18 files and the rng partition matches the claim EXACTLY, file for file: 10 without 'rng' (checkpoint.pkl step30, ssm_base_s0 step21156, ssm_base_42M_s0 step5126, ssm_base_85M_s0 step10375, swa128_42M_s0 step5859, swa128_85M_s0 step11718, swa128_nope_42M_s0 step5859, attn1to3_42M_s0 step5126, fullattn_42M_s0 step5126, and the .discarded file step7800) and 8 with 'rng' (ssm_base_42M_s1/s2, attn1to3_42M_s1/s2, fullattn_42M_s1/s2, swa128_42M_isofix_s0, swa128_nope_42M_isofix_s0). Both isofix files are step=4929, matching the correction doc's required budget. +``` + + +### 8.16 Which checkpoints are smoke-test artifacts (never a result)? + +**Value** + +``` +7 files, 18.88 GiB total. In Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/: smoke_baseline.pt, smoke_baseline_resumed.pt, smoke_wsd.pt, smoke_zloss.pt, smoke_arch.pt (5 files, 16.66 GiB, all mtime 2026-06-18 13:31-13:40). In Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/: checkpoint_imu1_smoke_step500.pt and checkpoint_imu1_smoke_step1000.pt (1,193,196,283 / 1,193,196,711 B, mtime 2026-06-09). Plus HybridSSM checkpoint.pkl (66,064,696 B, step=30, smoke). +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/smoke_baseline.pt` + +**Source quote** + +``` +-rw-rw-r-- 1 yashb98 yashb98 3576681053 Jun 18 13:31 smoke_baseline.pt +``` + +**Confidence** — measured from code + +**Caveat** — These are §C5.0 smoke-test outputs (1 step on a tiny batch). 18.88 GiB of pure deletable overhead. Never publishable. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +ls -l verified: smoke_baseline.pt 3,576,681,053 B 2026-06-18 13:31 (the quoted line, exact), smoke_wsd.pt 3,576,674,813 13:33, smoke_zloss.pt 3,576,677,309 13:36, smoke_arch.pt 3,579,557,149 13:39, smoke_baseline_resumed.pt 3,576,694,557 13:40 — sum 17,886,284,881 B = 16.657 GiB, mtime window 13:31-13:40, EXACT. checkpoint_imu1_smoke_step500.pt 1,193,196,283 B and _step1000.pt 1,193,196,711 B, both mtime 2026-06-09 — EXACT. 7-file sum = 20,272,677,875 B = 18.88 GiB, EXACT (the HybridSSM pkl is correctly excluded from the 7/18.88 and listed as 'plus'). HybridSSM checkpoint.pkl: 66,064,696 B, unpickled step=30, no rng — EXACT. +``` + + +### 8.17 Are any checkpoints tracked in git? + +**Value** + +``` +ZERO. `git ls-files | grep -cE '\.(pt|pth|bin|safetensors|ckpt|pkl|msgpack|npz)$'` returns 0. Both .gitignore files exclude them. +``` + +**Evidence** — `.gitignore:19-24 ; HybridSSM-0.2B/.gitignore:2-3` + +**Source quote** + +``` +.gitignore:19: # Checkpoints (270MB+ each; not for version control — use HF Hub or git-lfs) +.gitignore:20: *.pt +.gitignore:21: *.pth +.gitignore:22: *.safetensors +.gitignore:23: *.bin +.gitignore:24: *.ckpt +HybridSSM-0.2B/.gitignore:2: *.pkl +``` + +**Confidence** — measured from code + +**Caveat** — MEMORY.md records a prior incident where gitignored-but-force-added files were DELETED from the working tree on branch checkout (guard_branch_switch_wipes_gitignored_evidence). Current branch is harden-research-loop; do not force-add checkpoints to publish them. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Ran the exact command: output 0. .gitignore lines 19-24 are verbatim at those numbers: 19 '# Checkpoints (270MB+ each; not for version control — use HF Hub or git-lfs)', 20 '*.pt', 21 '*.pth', 22 '*.safetensors', 23 '*.bin', 24 '*.ckpt'. HybridSSM-0.2B/.gitignore:2 '*.pkl', :3 '*.pkl.tmp' — both verbatim. The branch-checkout caveat is a live risk worth keeping. +``` + + +### 8.18 Is there tooling on disk to convert a checkpoint to HF/safetensors format? + +**Value** + +``` +Exactly ONE script, and it covers only SmolLM2: SmolLM2-134(base)/scripts/export_to_hf.py. It loads the .pt, round-trips through SmolLM2ForCausalLM, copies into a HF LlamaForCausalLM built from AutoConfig.from_pretrained('HuggingFaceTB/SmolLM2-135M'), then save_pretrained(safe_serialization=True) + tokenizer.save_pretrained. No equivalent exists for Qwen3-0.6B or HybridSSM-0.2B. No hf_export/ directory exists on disk. +``` + +**Evidence** — `SmolLM2-134(base)/scripts/export_to_hf.py:56-67` + +**Source quote** + +``` +from transformers import AutoConfig, AutoTokenizer, LlamaForCausalLM + cfg = AutoConfig.from_pretrained(args.repo) + hf = LlamaForCausalLM(cfg) + missing, unexpected = hf.load_state_dict(ours_sd, strict=False) +... + hf.save_pretrained(out, safe_serialization=True) + tok = AutoTokenizer.from_pretrained(args.repo) + tok.save_pretrained(out) +``` + +**Confidence** — measured from code + +**Caveat** — The script requires network (AutoConfig/AutoTokenizer from the Hub) and has NEVER been run to completion on this box as far as disk shows — hf_export/ does not exist. Uploading Qwen3 or HybridSSM weights requires writing a new exporter; HybridSSM especially, since it is a novel flax architecture with no HF modelling class at all. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +export_to_hf.py lines 56-59 and 65-67 quoted verbatim and correct (56 `from transformers import AutoConfig, AutoTokenizer, LlamaForCausalLM`, 57 cfg=AutoConfig.from_pretrained(args.repo), 58 hf=LlamaForCausalLM(cfg), 59 load_state_dict(..., strict=False), 65 hf.save_pretrained(out, safe_serialization=True), 66-67 tokenizer save). The elision '...' between 59 and 65 skips lines 60-64, which include a real guard: `missing = [k for k in missing if k != "lm_head.weight"]` then `raise SystemExit` on any other mismatch. A repo-wide grep for save_pretrained/save_file over all *.py found these as the ONLY save calls — every other hit is prose or from_pretrained. `find -type d -name hf_export` returned nothing (note: hf_export/ is gitignored at .gitignore:27, so its absence proves only that it is not on disk now). +``` + + +### 8.19 Do the modernized/exploratory Qwen3 build checkpoints load into stock HF Qwen3? + +**Value** + +``` +NO for modernized. checkpoint_imu1_2tpp_step18000.pt has 752,091,220 tensor elements (vs 751,632,384 for faithful) and its embedded config adds non-HF keys including 'use_value_residual': true and 'use_layernorm_sc...'. checkpoint_prope10_2tpp_.pt (exploratory) is 751,632,384 elements with config key 'partial_rotary_factor': 0.1 — that key IS supported by HF Qwen3. Both are weights-only dicts with keys ['model','config','step'] and carry NO tok_seen. +``` + +**Evidence** — `Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/checkpoint_imu1_2tpp_step18000.pt (in-file 'config')` + +**Source quote** + +``` +keys ['model', 'config', 'step'] + step 18000 tok_seen None params 752091220 + config {..., "tie_word_embeddings": true, "attention_bias": false, "attention_dropout": 0.0, "use_value_residual": true, "use_layernorm_sc... +``` + +**Confidence** — measured from code + +**Caveat** — The modernized family is 11 intermediate step-checkpoints (step500,1000,2000..18000) of ONE run, 12.22 GiB — at most one (step18000) is publishable; the rest are training-curve snapshots. Exploratory is 2 files (prope10, prope25) at step 4000. + +**Verdict — ❌ WRONG** + +**Corrected value** + +``` +'partial_rotary_factor IS supported by HF Qwen3' is FALSE on this box. transformers 5.8.0 Qwen3 never reads it, so the exploratory checkpoint would load into stock Qwen3 and silently run FULL RoPE — architecturally wrong, not portable. Also the modernized checkpoint is 423 keys prefixed '_orig_mod.' (torch.compile), not 311 — a second, unmentioned blocker. +``` + +**Verifier note** + +``` +Loaded both files. checkpoint_imu1_2tpp_step18000.pt: 1,193,196,711 B, keys ['model','config','step'], step 18000, 752,091,220 elems, config adds use_value_residual:true, use_layernorm_scaling:true, use_head_gating:true, no tok_seen — the 'NO for modernized' verdict is right and understated: its state_dict has 423 keys named `_orig_mod.model.embed_tokens.weight` etc. checkpoint_prope10_2tpp_.pt: 1,192,229,775 B, step 4000, 751,632,384 elems, config partial_rotary_factor 0.1 — all correct. BUT I disproved the HF-support claim three ways: (1) grep -rn 'partial_rotary_factor' over site-packages/transformers/models/qwen3/ returns NOTHING (it appears only in laguna/moonshine/etc.); (2) inspect.signature(Qwen3Config.__init__) has no such parameter and modeling_qwen3.py source never mentions it; (3) empirically, Qwen3RotaryEmbedding(Qwen3Config(head_dim=128, partial_rotary_factor=0.1)) yields inv_freq of length 64 — identical to the default — i.e. full 128-dim RoPE. The kwarg is merely absorbed into config.rope_parameters and ignored. Installed transformers 5.8.0. +``` + + +### 8.20 Which files are symlinks (§C13 control reuse) rather than real checkpoints? + +**Value** + +``` +12 symlinks, zero extra disk. 3 in 2026-06-21_arch-subdrill-p2 (checkpoint_baseline_seed{0,1,2}.pt), 3 in 2026-06-24_data-dclm-vs-fineweb (checkpoint_control_seed{0,1,2}.pt), 6 in 2026-06-26_data-mix-composition (checkpoint_dclm_seed{0,1,2}.pt, checkpoint_fineweb_seed{0,1,2}.pt). All point at either 2026-06-18_imu1-deconfound-p1/checkpoint_baseline_seed*.pt or 2026-06-24.../checkpoint_treatment_seed*.pt. +``` + +**Evidence** — `Qwen3-0.6B/experiments/2026-06-26_qwen3-0.6b_data-mix-composition/checkpoint_fineweb_seed0.pt (symlink)` + +**Source quote** + +``` +lrwxrwxrwx 1 yashb98 yashb98 133 Jun 26 02:15 checkpoint_fineweb_seed0.pt -> /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/checkpoint_baseline_seed0.pt +``` + +**Confidence** — measured from code + +**Caveat** — The symlinks use ABSOLUTE paths under /home/yashb98/Downloads/BuildFromScratch — they break on any copy/move/archive. Do not tar these into a release bundle without dereferencing. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +`find -type l` over *.pt/*.pkl returned exactly 12, with readlink targets matching the claimed mapping precisely: arch-subdrill-p2/checkpoint_baseline_seed{0,1,2}.pt and data-dclm-vs-fineweb/checkpoint_control_seed{0,1,2}.pt and data-mix-composition/checkpoint_fineweb_seed{0,1,2}.pt all -> imu1-deconfound-p1/checkpoint_baseline_seed{0,1,2}.pt; data-mix-composition/checkpoint_dclm_seed{0,1,2}.pt -> data-dclm-vs-fineweb/checkpoint_treatment_seed{0,1,2}.pt. The quoted ls line is exact: 'lrwxrwxrwx 1 yashb98 yashb98 133 Jun 26 02:15 checkpoint_fineweb_seed0.pt -> /home/yashb98/Downloads/BuildFromScratch/Qwen3-0.6B/experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/checkpoint_baseline_seed0.pt'. All targets are absolute — the tar/archive warning is correct. +``` + + +### 8.21 What is the only HybridSSM checkpoint that has actually been scored by the §C10 eval harness? + +**Value** + +``` +checkpoint_ssm_base_s0.pkl. HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/eval/suite_results.json (dated 2026-07-21T03:21:22Z, suite_version text-lm-v2) names target_ckpt='checkpoint_ssm_base_s0.pkl', baseline_ckpt=null, self_floor=true; wikitext2_val PPL 133.4628, code_py PPL 5142.6426, both on 204,600 tokens. +``` + +**Evidence** — `HybridSSM-0.2B/experiments/2026-07-19_hybrid-ssm-0.2b_build/eval/suite_results.json` + +**Source quote** + +``` +"suite_version": "text-lm-v2", + "target_ckpt": "checkpoint_ssm_base_s0.pkl", + "baseline_ckpt": null, + "self_floor": true, + ... "wikitext2_val": {"target": 133.4628, ...}, "code_py": {"target": 5142.6426, ...} +``` + +**Confidence** — results JSON + +**Caveat** — That file is step=21156 (the 170M-token build run), NOT any of the 42M/85M ladder cells. The ladder cells were scored by a different script (arch_ladder_scores.json) which records NO checkpoint filenames at all — I searched its JSON for /checkpoint_.*\.pkl/ and got an empty list, so ladder scores cannot be traced to specific files from that artifact alone. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +suite_results.json read directly: suite_version 'text-lm-v2', date '2026-07-21T03:21:22Z', target_ckpt 'checkpoint_ssm_base_s0.pkl', baseline_ckpt null, self_floor true; ppl.wikitext2_val.target 133.4628 with n_tokens 204600, ppl.code_py.target 5142.6426 with n_tokens 204600 — ALL EXACT. Dataset-config check passes the -raw- trap: corpus_id is 'Salesforce/wikitext:wikitext-2-raw-v1:validation@b08601e04326c79dfdd32d625aee71d232d685c3' and code is 'codeparrot/codeparrot-clean-valid:train@4db92d2ec0c1b4c41eeb439cfae16854511d9dcd (streaming first-N >500k chars)' — pinned revisions, no baseline. I unpickled checkpoint_ssm_base_s0.pkl: step=21156, EXACT. grep -oE 'checkpoint_[A-Za-z0-9_]+\.pkl' over arch_ladder_scores.json returns count 0 — the traceability gap is real, and the only link is run_arch_ladder.sh:395 `--ckpt "checkpoint_${id}.pkl"` as the gap section says. +``` + + +### 8.22 Which Qwen3 experiment directories contain NO checkpoints at all? + +**Value** + +``` +2026-06-16_qwen3-0.6b_eval-faithful, eval-modernized, eval-prope10, eval-prope25, 2026-06-16_qwen3-faithful_eval-first, 2026-06-27_qwen3-0.6b_midtraining, 2026-07-01_qwen3-0.6b_rlvr-phase1-passk. Also HybridSSM-0.2B/experiments/2026-07-21_..._arch-ladder, 2026-07-28_..._arch-ladder-repair, 2026-07-29_..._throughput, and HybridSSM-0.2B/results/. There is no Qwen3-0.6B/experiments/2026-07-23* directory despite a ledger run of that id. +``` + +**Evidence** — `Qwen3-0.6B/experiments/ (directory listing) ; research/ledger/ledger.json (run 2026-07-23_qwen3-0.6b_normuon-at-scale)` + +**Source quote** + +``` +"artifacts_location": "Qwen3-0.6B/experiments/2026-07-05_qwen3-0.6b_scaling-persistence (shared with the parent ladder; its ch... +``` + +**Confidence** — measured from code + +**Caveat** — Run-id to directory is many-to-one and sometimes non-existent. Any 'weights for run X' claim must be resolved through the launch script, not the run id. + +**Verdict — ⚠️ NEEDS QUALIFIER** + +**Corrected value** + +``` +The Qwen3 list is incomplete — it must also include 2026-07-05_qwen3-0.6b_scaling-persistence. Correct count is 8 of 17 Qwen3 experiment dirs, not 7. +``` + +**Verifier note** + +``` +I enumerated all 17 Qwen3-0.6B/experiments/*/ dirs and tested each for *.pt/*.pkl (excluding tokcache). Zero-checkpoint dirs: the 7 named PLUS 2026-07-05_qwen3-0.6b_scaling-persistence. The omission is odd because the same fact set correctly states elsewhere (checkpoints-back-the-ladder fact) that the scaling-persistence dir holds no checkpoints — so the two facts contradict each other and a reader could conclude the ladder dir does hold weights. HybridSSM side confirmed: only 2026-07-19_..._build holds pickles; 2026-07-21_arch-ladder, 2026-07-28_arch-ladder-repair, 2026-07-29_throughput and HybridSSM-0.2B/results have none. `ls -d Qwen3-0.6B/experiments/2026-07-23*` -> 'No such file or directory', while the ledger carries run 2026-07-23_qwen3-0.6b_normuon-at-scale — confirmed. The quoted artifacts_location string is verbatim, but it lives at runs[24].metrics.artifacts_location, not at the run's top level (research/ledger/ledger.json:2077). +``` + + +### 8.23 Is a GPU job currently running that would make loading checkpoints unsafe? + +**Value** + +``` +No trainer live. pgrep -af train returned only this session's own bash wrapper (a known false-positive documented in MEMORY.md guard_pgrep_self_match). free -g showed 100 GB available of 119 GB at inspection time. The ledger does list one run status=running (2026-07-28_hybrid-ssm-0.2b_arch-ladder-repair, eta_hours 30.66) with no live process. +``` + +**Evidence** — `research/ledger/ledger.json (runs[].status)` + +**Source quote** + +``` +runs (29): crashed=2, done=26, running=1 + in-flight: 2026-07-28_hybrid-ssm-0.2b_arch-ladder-repair status=running eta_hours=30.66 +``` + +**Confidence** — measured from code + +**Caveat** — The ledger 'running' entry is STALE relative to the process table — no matching trainer exists. I loaded at most one checkpoint at a time, CPU-only (map_location='cpu'), and never touched a file >8 GB, per the task's safety constraint. + +**Verdict — ✅ CONFIRMED** + +**Verifier note** + +``` +Independently re-verified before any checkpoint load: `python3 sentinel.py preflight` exited 0 with 'PREFLIGHT OK mem_available=83% disk_free=2705GB load1=0.78 cores=20 trainers=none' — sentinel's own trainer detector says none, which is stronger evidence than pgrep. `pgrep -af train` again matched only my own bash wrapper (the documented self-match). free -g showed 99 GB available of 119 at my check (vs the claimed 100 — time-varying, not a discrepancy). Ledger: 29 runs, Counter({'done': 26, 'crashed': 2, 'running': 1}), the single running entry is 2026-07-28_hybrid-ssm-0.2b_arch-ladder-repair with eta_hours 30.66 — EXACT. The stale-ledger observation is correct and worth surfacing. I loaded at most one checkpoint per process, map_location='cpu', under safe_cuda.guard(0.85), and used msgpack_restore on params only (never opt_state). +``` + + +### 8.V Additional verifier findings (no 1:1 extracted fact) + +**8.V1 — ❌ WRONG** · [GAPS SECTION] No SHA256 or any checksum exists for any checkpoint; every ledger run entry has lineage.artifact_sha256 = null + +**Checked against** + +``` +Every ledger run entry I inspected has lineage.artifact_sha256 = null +``` + +**Corrected value** + +``` +27 of 29 runs have lineage.artifact_sha256 = null; 2 do not. 2026-07-29_hybrid-ssm-0.2b_fineweb-edu-carding carries a real 64-hex digest 'c83b7d608a0ca320ae7b7e41dbee05282f074a004a87a9a90f2f4fd0f5032491', and 2026-06-16_qwen3-faithful_eval-first carries the non-hash string 'checkpoint_qwen3_baseline2tpp.pt@step18150'. +``` + +**Verifier note** + +``` +Verified by walking every runs[] entry in research/ledger/ledger.json and counting lineage.artifact_sha256: Counter({None: 27, 'checkpoint_qwen3_baseline2tpp.pt@step18150': 1, 'c83b7d608a0ca320ae7b7e41dbee05282f074a004a87a9a90f2f4fd0f5032491': 1}). The broader conclusion — that no on-disk model checkpoint has a verifiable checksum — probably still stands (the hex digest belongs to a dataset-carding run and the other value is a filename, not a hash), but the absolute quantifier 'every entry' is false as written and must not be repeated. I did not verify what artifact the hex digest covers. +``` + + +### 8.G Gaps — not determinable from disk + +- Which HybridSSM ladder score row came from which .pkl file. arch_ladder_scores.json contains no checkpoint filenames (regex search for checkpoint_*.pkl returned an empty list), so ladder BPB/CE numbers cannot be traced to specific on-disk weights from that artifact alone. Only the cell id links them, via run_arch_ladder.sh:395 `--ckpt "checkpoint_${id}.pkl"`. +- Whether 'HybridSSM-0.2B' means 0.2B total or 0.2B non-embedding params. Measured totals are 267M-325M; non-embedding for ssm_base is 189,131,520. I found no file on disk that states the convention. +- License/provenance status for redistributing HybridSSM weights. The model is novel but uses the Qwen3-0.6B-Base tokenizer (vocab 151,936) and was trained on FineWeb-Edu; no LICENSE file or data-license record was found alongside the checkpoints. +- Byte-level reproducibility of the two SmolLM2 checkpoints. Both were written 2026-05-13/14 but train.py and train_tinystories.py were modified 2026-05-19; checkpoint.pt lacks the 'training_recipe' key the current train.py:182 writes, so the exact script that produced them is not the version at HEAD. +- Whether the 8 pre-PRNG-fix HybridSSM checkpoints that were resumed mid-run carry the same RNG confound that got the 85M one quarantined. Only checkpoint_swa128_nope_85M_s0.pkl was quarantined; I found no document assessing the others, and 'has rng key' is my own measurement from file tails, not a repo-stated classification. +- No SHA256 or any checksum exists for any checkpoint. Every ledger run entry I inspected has lineage.artifact_sha256 = null, so on-disk file integrity cannot be verified against any record. + +--- From 462e80257581ae964ba40abd400bc722fe816e2e Mon Sep 17 00:00:00 2001 From: yashb98 Date: Wed, 5 Aug 2026 00:36:14 +0100 Subject: [PATCH 31/35] Fix false and stale claims across the READMEs Every replacement number was re-derived from the artifact on disk, then checked by an adversarial pass whose instructions were to refute it. Three rounds were needed: the first two introduced errors of their own, which are also fixed here. Comparability (the load-bearing one): - Qwen3-0.6B/README.md claimed all four Qwen3 perplexities use "the identical 300k-token FineWeb-Edu val slice ... every row is directly comparable". False. 13.40 and the Phase-A sweep sit on tokcache_133072000_300000.pt (hardcoded at eval_original_vs_repro.py:22); 28.65 / 23.52 / 29.54 sit on tokcache_1191478400_300000.pt. Replaced with a per-cache table. The derived 2.14x and 1.76x gaps are relabelled cross-cache wherever they appear (Qwen3 README, root README, both plots READMEs, both build READMEs); 46.31/13.40 = 3.46x is same-cache and kept. NorMuon: - The -0.474 bpb result was advertised as a "significant win" in four places with no mention of the ladder that nulls its persistence. Now scoped to its 42M budget everywhere, with a new "Scaling persistence" section carrying the full n=3 table. The ledger verdict `null` is attributed to the ladder run, not to 2026-06-16_qwen3_normuon-vs-adamw (which is `win`). - The root README's ladder block was stale in five specifics (n=2, +0.073 [-0.038,+0.184], "not significant at the top", code +0.192, slope -0.328). Current: n=3 at every rung, +0.072 [+0.055,+0.088], all six rungs significant, code +0.177, slope -0.342 (r2 0.84). - "Falls within the noise floor" is now stated as what it is: the OLS-fitted edge at the top rung (wikitext 0.0297 vs 0.0368 floor), not the measured gap. On code the fitted edge 0.1255 vs 0.0463 is still resolved, so the corpora are no longer described with one blanket claim. Parity: - "bit-exact / max error 0.0" now says fp32-on-CPU, 5-token prompt, everywhere it appears. For SmolLM2 the GPU numbers are given (final-logits 4.72e-05, per-layer 1.95e-03 at layer 14 - which exceeds the repo's own 1e-3 gate); for Qwen3 it is stated that no GPU parity check exists. Arithmetic and citations: - NorMuon wall-clock "~30% more" -> +43.9% (-30.5% throughput, 5,172 vs 7,444). - Faithful build: 7,480 tok/s was the step-100 reading, final is 7,444. - Modernized build: 1,191,478,400 was the token-cache size; trained budget is 1,189,478,400 (18,150 x 65,536). - SmolLM2 demo loss 6.321 -> 6.288 (min 6.039 @ step 140); "agree to 6 decimal places" -> 5; wikitext double-count ratio 1.97x; tokenization sourced to results.ipynb + POST_DATA.md, not summary.json (which has no such key). - CI lower bound 0.444 -> 0.443 at three sites. - params 596,049,920 no longer attributed to verify.json, which has no such field. Stale status: - Data arm, mid-training and the 3-seed SFT were all done but still listed as running/planned. The n=1 VibeThinker SFT number is marked superseded by 2026-06-27_qwen3-0.6b_sft-3seed and relabelled in-loop, not held-out. - "MC accuracy is near-chance (no-signal)" was wrong: only WinoGrande is at chance; ARC-easy and HellaSwag carry signal:true above their 0.25 floor. - tinystories_summary.md documents an EARLIER run (3.7893 / 137.3 min / 12,150 tok/s) than the committed artifacts (3.7900 / 116.1 min / 14,356 tok/s); it is now labelled as such rather than having one number swapped. - PLOTS_INDEX plot count 74/72 -> 79/79. Known remaining gap: the committed overview figures still print "Published Qwen3-0.6B-Base = 13.40" and annotate 2.21x. Both are wrong, both live in the PNG/PDF and make_overview_plots.py rather than in a README, so they are disclosed in the captions and left for a regeneration pass. Co-Authored-By: Claude Opus 5 (1M context) --- Qwen3-0.6B/PLOTS_INDEX.md | 8 +- Qwen3-0.6B/README.md | 165 +++++++++++++----- .../README.md | 8 +- .../README.md | 24 ++- .../results/plots/README.md | 9 +- Qwen3-0.6B/results_overview/plots/README.md | 20 ++- README.md | 71 +++++--- SmolLM2-134(base)/README.md | 13 +- SmolLM2-134(base)/results/README.md | 17 +- .../results/comparison_with_hf.md | 2 +- .../results/tinystories_summary.md | 19 +- 11 files changed, 254 insertions(+), 102 deletions(-) diff --git a/Qwen3-0.6B/PLOTS_INDEX.md b/Qwen3-0.6B/PLOTS_INDEX.md index 0d880a0..3dadbae 100644 --- a/Qwen3-0.6B/PLOTS_INDEX.md +++ b/Qwen3-0.6B/PLOTS_INDEX.md @@ -6,7 +6,7 @@ relative to this file (`Qwen3-0.6B/`). Captions are one-liners; the exact source file and full data points live in each plot dir's own `README.md`. Generated by globbing the on-disk tree (no GPU, no model loads, no process -interaction). **Plot files on disk: 74** (72 tracked in git; PNG + PDF, some older +interaction). **Plot files on disk: 79** (all 79 tracked in git; PNG + PDF, some older figures are PNG-only — flagged below). The per-file detail sections below enumerate the builds + the NorMuon / VibeThinker experiments (51 files); the newer **Phase-1 deconfound**, **Phase-2 arch-subdrill**, and **data-arm** cohort figures are captured @@ -26,7 +26,7 @@ parent [`README.md`](README.md)). | NorMuon-vs-AdamW ablation | `experiments/2026-06-16_qwen3_normuon-vs-adamw` | **COMPLETE** (n=3 seeds, 42M-tok iso-FLOP) | NorMuon 1.6355 vs AdamW 2.1098 bpb wikitext-2 (`verdict.json`) | | IMU-1 de-confound (Phase 1) | `experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1` | **COMPLETE** (12/12 cells, n=3 seeds, iso-FLOP) — fig `deconfound_bpb_verdict.{png,pdf}` | **arch is the sole driver**: wikitext +0.118 bpb [0.100,0.135], code +0.305 [0.259,0.351]; WSD/z-loss n.s. (`verdict.json`, `attributed`) | | Arch sub-drill (Phase 2) | `experiments/2026-06-21_qwen3-0.6b_arch-subdrill-p2` | **COMPLETE** (9/9 cells, n=3 seeds, iso-FLOP) — fig `plots/phase2_arch_subdrill_bpb.png` | **all 3 arch flags significant**: vr +0.0355 > ln +0.0337 > hg +0.0256 bpb wikitext (`verdict.json`, `attributed`) | -| Data arm (dclm vs FineWeb-Edu) | `experiments/2026-06-24_qwen3-0.6b_data-dclm-vs-fineweb` | **RUNNING** (treatment cells; control reused from Phase-1) | no verdict yet — BPB A/B in flight | +| Data arm (dclm vs FineWeb-Edu) | `experiments/2026-06-24_qwen3-0.6b_data-dclm-vs-fineweb` | **COMPLETE** 2026-06-26 (3 treatment seeds; control reused from Phase-1) — fig `plots/verdict_bpb.png` | wikitext-2 −0.0097 bpb CI [−0.0255,+0.0061] **n.s.**; code_py +0.7034 bpb CI [+0.664,+0.743] **significant**; `overall_verdict: recipe-level` (ledger `directional`) | | VibeThinker SFT | `experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning` | **COMPLETE** — its log reads `DONE in 297.3 min` @ step 238 (reasoning PPL 14.262 -> 11.596). NOTE: its local `plots/README.md` + the two figures still say "IN PROGRESS @ step 230/238" because they were drawn before the final step; the figures are correct for the points they show (steps 10-230) but predate the final eval. | reasoning PPL 11.596 (`vibethinker_sft_driver.log` DONE) | | Cross-build overview | `results_overview` | **COMPLETE** (this run) | combines the three COMPLETE 2-TPP finals | @@ -34,7 +34,7 @@ parent [`README.md`](README.md)). ## results_overview/plots/ (CROSS-BUILD — new this run) -- `results_overview/plots/fig1_matched_compute_final_ppl_bar.png` / `.pdf` — Matched-compute (1.19B-token) final val-PPL bar: Faithful 28.65 / IMU-1 23.52 / pRoPE-25% 29.54 vs published 13.40 dashed line; gap-to-original annotated (2.14x / 1.76x / 2.21x). [COMPLETE] +- `results_overview/plots/fig1_matched_compute_final_ppl_bar.png` / `.pdf` — Matched-compute (1.19B-token) final val-PPL bar: Faithful 28.65 / IMU-1 23.52 / pRoPE-25% 29.54 vs the released Base at 13.40 (dashed). **Caveat: 13.40 was measured on a different val tail (`tokcache_133072000`) than the three bars (`tokcache_1191478400`), so the annotated multiples (2.14x / 1.76x / 2.21x — the last is a misrounding of 29.54/13.40 = 2.204, i.e. 2.20x) are cross-cache, not like-for-like.** 13.40 is our own measurement, not a published figure — **though the rendered image's legend still says "Published" and still annotates 2.21x; it predates this correction and needs regenerating.** [COMPLETE] - `results_overview/plots/fig2_eval_ppl_vs_tokens_faithful_vs_imu1.png` / `.pdf` — Eval val-PPL vs tokens (log y), Faithful vs IMU-1 overlaid; both COMPLETE @ 1.19B tok, IMU-1 below Faithful throughout (28.65 vs 23.52). [COMPLETE] ## builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/plots/ [COMPLETE baseline] @@ -70,7 +70,7 @@ parent [`README.md`](README.md)). ## experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/ [COMPLETE — n=3-seed 42M-tok iso-FLOP ablation] -- `fig1_headline_wikitext2_bpb.png` / `.pdf` — Headline wikitext-2 BPB: AdamW 2.1098 vs NorMuon 1.6355 (mean ± 95% CI, per-seed dots); +0.474 bpb, significant. +- `fig1_headline_wikitext2_bpb.png` / `.pdf` — Headline wikitext-2 BPB: AdamW 2.1098 vs NorMuon 1.6355 (mean ± 95% CI, per-seed dots); +0.474 bpb, significant **at this 42M-token budget**. The budget ladder (`experiments/2026-07-05_qwen3-0.6b_scaling-persistence`) shows the gap SHRINKING rather than persisting: +0.474 (42M) → +0.126 (168M) → +0.072 (420M) — each rung still individually significant, trend `CONVERGES`, run-level ledger verdict `null` (the edge does not persist). - `fig2_code_bpb.png` / `.pdf` — codeparrot-clean-valid BPB: AdamW 3.3847 vs NorMuon 2.8831; +0.502 bpb, significant. - `fig3_fineweb_val_ppl.png` / `.pdf` — In-training fineweb-val PPL per seed (log y): AdamW ~145-156 vs NorMuon ~60-61 (independent corroboration). - `fig4_train_loss_curves.png` / `.pdf` — Training-loss curves, 2 arms × 3 seeds; NorMuon converges to ~4.0-4.2 vs AdamW ~4.7-5.1. diff --git a/Qwen3-0.6B/README.md b/Qwen3-0.6B/README.md index fd6f1f2..5c4646f 100644 --- a/Qwen3-0.6B/README.md +++ b/Qwen3-0.6B/README.md @@ -1,24 +1,31 @@ # Qwen3-0.6B — from-scratch reproduction + research experiment A single-file PyTorch reproduction of [`Qwen/Qwen3-0.6B-Base`][hfbase] (596M-param -decoder-only transformer), **verified bit-exact** against the official HuggingFace -weights (`max |Δlogits| = 0.0`), used as the base for a **three-build experiment**: +decoder-only transformer), **verified bit-exact in fp32 on CPU** against the official +HuggingFace weights (`max |Δlogits| = 0.0` on a single 5-token prompt; there is no GPU +parity check — see [Results](#results-so-far)), used as the base for a **three-build experiment**: reproduce it faithfully, then apply recent (2026) research methods and measure — at matched compute — whether they beat the faithful baseline. -> **Status: IMU-1 win de-confounded → architecture is the driver; Phase 2 now drilling *which* arch tweak.** -> Architecture VERIFIED bit-exact; Phase A LR (`lr24 = 2.4e-3`); Phase B @ 2 TPP: faithful **28.65** · -> **IMU-1 bundle 23.52 — a proven 17.9% win** (gap 2.14× → **1.76×**) · partial-RoPE 0.25 **29.54 (loses)**. +> **Status: IMU-1 win de-confounded → all three arch modules attributed; data arm, mid-training and 3-seed SFT DONE; NorMuon's edge converges away with budget.** +> Architecture VERIFIED bit-exact (fp32/CPU); Phase A LR (`lr24 = 2.4e-3`); Phase B @ 2 TPP: faithful **28.65** · +> **IMU-1 bundle 23.52 — a directional −17.9% delta** (n=1, single seed, same val cache as the faithful +> baseline) · partial-RoPE 0.25 **29.54 (loses)**. > The IMU-1 win was a confounded bundle (NorMuon + WSD + z-loss + 3 arch tweaks). **Phase 1 de-confound is > DONE** (run `2026-06-18_…imu1-deconfound-p1`, 12/12 cells, canonical eval-harness **BPB** verdict > `overall_verdict: attributed`): on the canonical metric **+arch is the SOLE driver** (wikitext **−0.118 bpb**, > 95% CI [0.100, 0.135]; code **−0.305 bpb**, CI [0.259, 0.351] — both significant), while **+WSD is NOT > significant** (CI crosses 0 — the −6.9% *in-loop proxy* gain did **not** survive the canonical metric) and -> **+z-loss is null**. So the −17.9% bundle = **NorMuon (optimizer, proven) + the IMU-1 architecture modules.** +> **+z-loss is null**. So the −17.9% bundle = **NorMuon (optimizer, isolated separately at 42M tokens — +> and it converges away with budget) + the IMU-1 architecture modules (the only significant axis at 131M/cell).** > **Phase 2 DONE** (`2026-06-21_…arch-subdrill-p2`): the arch win splits into **all three flags as significant > drivers** — value-residual (largest), layernorm-scaling (parameter-free), head-gating — `attributed`, and the -> text-lm-v3 downstream battery confirms it. **Now running: the data arm** (`2026-06-24_…data-dclm-vs-fineweb`) — -> a fixed-token **dclm-edu vs FineWeb-Edu** A/B (the bitter-lesson lever toward 13.40), then mid-training auto-follows. +> text-lm-v3 downstream battery confirms it. **Data arm DONE** (`2026-06-24_…data-dclm-vs-fineweb`, 2026-06-26): +> the fixed-token **dclm-edu vs FineWeb-Edu** A/B came back **null on English** (CI crosses 0) but a +> **large significant code win: −0.70 bpb, 95% CI [0.664, 0.743], 3 seeds**; it is logged `directional` +> because it is a single 131M budget, not because the code effect is inside the noise floor. +> **Mid-training DONE** too +> (`2026-06-30_…midtrain-anneal`). > — see [End-to-end lifecycle](#end-to-end-lifecycle--what-weve-done--whats-next). > **This is an index.** Each build has its own detailed README — see @@ -32,12 +39,30 @@ matched compute — whether they beat the faithful baseline. ## Results so far -All perplexities use **identical eval code on the identical 300k-token FineWeb-Edu -val slice** ([`eval_original_vs_repro.py`](builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py)), -so every row is directly comparable. +**The perplexities on this page are NOT all mutually comparable.** They use the same +eval code, but two *different* 300k-token FineWeb-Edu val tails — one per token budget. +Only same-cache rows may be compared: -**Bit-exact reproduction** — `verify.json`: `max_abs_error = 0.0`, argmax `" Paris"`, -params **596,049,920**. Our `model.py` *is* Qwen3-0.6B. +| Numbers | Val cache | How it was built | +|---|---|---| +| **13.40** (released Base) · **46.89 / 46.31 / 49.28** (Phase-A LR sweep) | `tokcache_133072000_300000.pt` | hardcoded at [`eval_original_vs_repro.py:22`](builds/2026-06-08_reproduce-faithful_qwen3-0.6b/eval_original_vs_repro.py) | +| **28.65** (faithful) · **23.52** (IMU-1) · **29.54** (pRoPE-25) | `tokcache_1191478400_300000.pt` | streamed by the faithful Phase-B run; the other arms load it | + +So **46.31 / 13.40 = 3.46× is like-for-like**, but **28.65 / 13.40 and 23.52 / 13.40 are +cross-cache** and must not be read as clean gaps to the released model — no same-cache +score for the released model on the Phase-B tail exists on disk. Both tails were cut by +the pre-decontamination splitter that `train_qwen3.py:130-136` itself calls "leak-suspect". +The fix landed in `86e79f3` (2026-06-16 21:57 UTC) — by which point all four Phase-B arms had +already loaded that pre-fix cache: three had finished, and the fourth (partial-RoPE 0.10) was +still running and never completed (it died at step 5450/18150). + +**Bit-exact reproduction — fp32, on CPU** — `verify.json`: `max_abs_error = 0.0`, +`dtype float32`, argmax `" Paris"` (params **596,049,920** is *not* in `verify.json` — it comes +from `model.py`'s param-count check and the trainer load logs), measured on the single 5-token prompt +`"The capital of France is"` (`input_shape [1, 5]`). That is the entire scope of the claim: +there is **no GPU parity check** for Qwen3, no per-layer or long-context check, and no +determinism flags are set anywhere in the repo. Within that scope our `model.py` *is* +Qwen3-0.6B. **Two tiers of evidence — read them differently.** The repo's *defensible* results are the single-variable, **3-seed, iso-FLOP** ablations scored on the canonical **BPB** metric (bits-per-byte @@ -49,19 +74,22 @@ no seed CI, no downstream evals — a scaling/sanity reading, not a defended cla | Result | Metric (vs faithful baseline) | Verdict | |---|---|---| -| **NorMuon > AdamW** | wikitext −0.474 bpb [0.444, 0.505] · code −0.502 [0.456, 0.547] | **significant win** | +| **NorMuon > AdamW** *at a 42M-token budget only* | wikitext −0.474 bpb [0.443, 0.505] · code −0.502 [0.456, 0.547] | **significant at 42M (this run's own ledger verdict is `win`) — but it [converges away with budget](#scaling-persistence-the-normuon-edge-converges-away): the ladder run `2026-07-05_…scaling-persistence` is ledger verdict `null`** | | **arch modules drive the IMU-1 win** | wikitext −0.118 bpb [0.100, 0.135] · code −0.305 [0.259, 0.351] | **significant — sole driver** | | WSD schedule · z-loss | CI crosses 0 (both corpora) | **not significant** | **Directional (n=1 · FineWeb-Edu val PPL · single seed — NOT a defended claim):** -| Model | Training tokens | val PPL (n=1) | Gap vs original | -|---|---|---|---| -| **Original** `Qwen3-0.6B-Base` | 36T | **13.40** | 1.0× | -| IMU-1 bundle (Build 2) | 1.19B | 23.52 | 1.76× | -| Faithful baseline (Build 1) | 1.19B | 28.65 | 2.14× | -| partial-RoPE 0.25 (Build 3) | 1.19B | 29.54 | 2.20× | -| Our best (Phase A, `lr24`) | 131M | 46.31 | 3.5× | +| Model | Training tokens | val PPL (n=1) | Val cache | Gap vs original | +|---|---|---|---|---| +| **Original** `Qwen3-0.6B-Base` (our eval) | 36T | **13.40** | `…133072000…` | 1.0× | +| Our best (Phase A, `lr24`) | 131M | 46.31 | `…133072000…` | **3.46×** (same-cache) | +| IMU-1 bundle (Build 2) | 1.19B | 23.52 | `…1191478400…` | *cross-cache — not comparable to 13.40* | +| Faithful baseline (Build 1) | 1.19B | 28.65 | `…1191478400…` | *cross-cache — not comparable to 13.40* | +| partial-RoPE 0.25 (Build 3) | 1.19B | 29.54 | `…1191478400…` | *cross-cache — not comparable to 13.40* | + +The three 1.19B rows **are** mutually comparable (same cache): IMU-1 23.52 vs faithful +28.65 is a **−17.9% n=1 delta**, and partial-RoPE 29.54 loses to the baseline. ![Phase B — final val PPL: IMU-1 wins, partial-RoPE loses to the baseline](builds/comparison/phaseB_final_ppl.png) @@ -69,23 +97,30 @@ no seed CI, no downstream evals — a scaling/sanity reading, not a defended cla **What the evidence supports (and what it doesn't):** -1. **Reproduction (directional):** the faithful baseline reaches **2.14× the original's PPL** with - **~30,000× less data** (1.19B vs 36T tokens); the earlier 131M-token probe sat at **3.5× with - ~275,000× less data** — each ~10× data roughly halves the gap (n=1, but the trend is robust). +1. **Reproduction (directional):** the faithful baseline reaches **28.65** at 1.19B tokens against + the released model's **13.40**, with **~30,000× less data** (1.19B vs 36T tokens) — but those two + numbers sit on different val caches, so the implied 2.14× is *cross-cache*. The earlier + 131M-token probe **is** same-cache: **46.31 vs 13.40 = 3.46×** with ~275,000× less data. Taking + the two ratios together suggests each ~10× of data roughly halves the gap, but that trend mixes + caches and is n=1 — indicative, not measured. 2. **The de-confounded win (defensible):** the IMU-1 bundle's improvement over our own faithful baseline is **attributable to NorMuon (optimizer) + the architecture modules** — both proven at 3 seeds, iso-FLOP, on BPB with CIs excluding 0; **WSD and z-loss are NOT significant**. The bundle's *−17.9% PPL* number itself is **n=1 and directional** — the defended claim is the per-component BPB attribution, not the single-seed bundle delta. *Caveat:* "matched compute" = - matched **tokens** (1.19B); IMU-1 also ran NorMuon (~30% more wall-clock, uncounted by the 6ND - FLOP model, though params are iso-FLOP at 1.00043). + matched **tokens** (1.19B); IMU-1 also ran NorMuon at **−30.5% throughput** (5,172 vs 7,444 tok/s + final), which is **+43.9% wall-clock** (63.9 h vs 44.4 h) — uncounted by the 6ND FLOP model, + though params are iso-FLOP at 1.00043. No `train_flops` artifact exists for any Phase-B run, so + the §C18 ≤5% iso-FLOP gate was never actually evaluated for the three-build comparison. > **Rigor status (`text-lm-v3` downstream battery — RUN 2026-06-24):** the §C25 downstream battery > (LAMBADA + per-task **BPB-on-gold** + ARC-e/HellaSwag/WinoGrande) has now been executed on **all 25 > checkpoints** (4 builds + Phase-1 + Phase-2) — full table: > [`research/eval/downstream_v3/RESULTS.md`](../research/eval/downstream_v3/RESULTS.md). As §C25.6 -> predicted, MC accuracy is near-chance (no-signal); **LAMBADA + BPB-on-gold discriminate** — and they -> **independently confirm every attribution**. +> predicted, MC accuracy does not discriminate between arms — but only **WinoGrande** is literally at +> chance (0.52 vs 0.50, `signal: false`); **ARC-easy** (acc_norm 0.45, CI [0.403, 0.490]) and +> **HellaSwag** (0.36, CI [0.321, 0.405]) sit well above their 0.25 chance floor with `signal: true`. +> **LAMBADA + BPB-on-gold are the discriminators** — and they **independently confirm every attribution**. ### Downstream confirmation — the builds (1.19B tok), independent of PPL @@ -172,7 +207,9 @@ whole way; partial-RoPE stays above it. ### Controlled attribution — NorMuon vs AdamW (single-variable, 3 seeds, iso-FLOP, verifier-PASS) The clean optimizer isolation that de-confounds one strand of the IMU-1 bundle: NorMuon beats -AdamW by **+0.474 bpb on wikitext-2 (95% CI [0.444, 0.505])** and +0.502 on code — significant. +AdamW by **+0.474 bpb on wikitext-2 (95% CI [0.443, 0.505])** and +0.502 on code — significant +**at this 42M-token budget**. It does not survive more budget — see +[the scaling ladder below](#scaling-persistence-the-normuon-edge-converges-away). ![NorMuon vs AdamW - wikitext-2 BPB with 95% CI](experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/fig1_headline_wikitext2_bpb.png) ![NorMuon vs AdamW - code BPB](experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/fig2_code_bpb.png) @@ -180,6 +217,36 @@ AdamW by **+0.474 bpb on wikitext-2 (95% CI [0.444, 0.505])** and +0.502 on code ![NorMuon vs AdamW - per-seed training curves](experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/fig4_train_loss_curves.png) ![AdamW LR-sweep robustness control](experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/fig5_adamw_lr_sweep.png) +### Scaling persistence: the NorMuon edge converges away + +**The +0.474 bpb win above is a 42M-token result and does not survive more budget.** A +budget ladder at fixed N=596M (`experiments/2026-07-05_qwen3-0.6b_scaling-persistence/`, +re-scored 2026-07-28, **n=3 seeds/arm at every rung**) sweeps only `--steps`: + +| Budget | wikitext-2 gap (bpb) | 95% CI | code_py gap | 95% CI | +|---|---|---|---|---| +| 42M | **+0.474** | [0.443, 0.505] | **+0.502** | [0.456, 0.547] | +| 168M | **+0.126** | [0.089, 0.163] | **+0.176** | [0.137, 0.215] | +| 420M | **+0.072** | [0.055, 0.088] | **+0.177** | [0.131, 0.223] | + +OLS over log10(tokens): wikitext slope **−0.417** (r² 0.923), code **−0.342** (r² 0.841). +`trend_verdict: CONVERGES` on both corpora; **`ledger_verdict: null`**. + +**Read it as: an early-training speedup that converges away, not an advantage at scale.** +Three honest qualifications: + +1. The gap at 420M is **still nominally significant** (CI excludes 0). "Falls within the + noise floor" refers to the *OLS-fitted* edge at the top rung (0.0297) vs the noise + floor (0.0368) — `edge_resolved: false` on wikitext, `true` on code. +2. **On code the label "converges" is generous**: 0.176 → 0.177 between the last two rungs + is a *plateau*, not convergence, and the negative slope is carried by the 42M point. + `verdict.json` itself hedges: *"still above noise at the largest measured budget but + trending out — the edge is eroding, extend the ladder before claiming it."* +3. **Inherited confound:** both LRs were tuned at 42M and never re-tuned per horizon, so + part of the fade may be a mis-tuned-LR artifact. Nothing on disk separates the two. + +This is a **budget** null at fixed N=596M. Nothing here says anything about larger N. + ### Deconfounding the IMU-1 win — 12-cell single-variable ladder (DONE — arch is the driver) *Which* component of the IMU-1 bundle drives the -17.9%? A single-variable, 3-seed, @@ -196,7 +263,7 @@ vs +arch, all AdamW). **Complete (12/12 cells); canonical eval-harness BPB verdi **The proxy flipped on the canonical metric.** The *in-loop val-PPL* proxy had ranked +arch −22% **and** +WSD −6.9% (significant) — but on the canonical BPB, **only arch survives**; +WSD's CI straddles 0. **+arch is the sole attributed driver** (baseline bpb 1.516/2.639 → arch 1.398/2.334), -so the −17.9% bundle decomposes into **NorMuon (optimizer, proven separately) + the IMU-1 +so the −17.9% bundle decomposes into **NorMuon (isolated separately at a 42M budget; it converges away at larger budgets) + the IMU-1 architecture modules** — not schedule, not z-loss. This is exactly why the loop trusts eval-harness BPB, not in-loop PPL, as the verdict (the in-loop plots below are the proxy; the table above is the verdict). The honest same-step caveat is built in: the deconfound arms are *complete* 2000-step runs @@ -219,9 +286,19 @@ canonical BPB above does not), *not* the verdict: ![Per-component attribution - single-variable, 3 seeds per arm (in-loop proxy PPL)](experiments/2026-06-18_qwen3-0.6b_imu1-deconfound-p1/deconfound_attribution.png) -### Post-training — SFT (VibeThinker reasoning, n=1 preliminary) +### Post-training — SFT (VibeThinker reasoning, n=1 preliminary — SUPERSEDED) + +In-loop reasoning PPL 14.26 -> 11.60 (n=1, response-only scoring — see the correction below); +no catastrophic forgetting (FineWeb-Edu retained). -Held-out reasoning PPL 14.26 -> 11.60; no catastrophic forgetting (FineWeb-Edu retained). +> **This n=1 number is an in-loop metric with a known confound** (response-only vs all-token +> scoring) and has been superseded by the 3-seed run `2026-06-27_qwen3-0.6b_sft-3seed` +> (2026-06-30), which re-scored on a fixed held-out set: masked reasoning PPL +> **14.127 → 11.573 (−18.1%)** vs base, but response-masking does **not** separate from its +> iso-FLOP `--no_mask` control (+0.009 PPL masked *significant* / −0.006 full-sequence *not* +> significant) → `overall_verdict: directional — masked and full-sequence comparisons disagree on +> significance; treat as not-yet-separable`. No catastrophic +> forgetting (held-out FineWeb-Edu 21.495 base → 21.652 SFT / 21.660 control). ![SFT training loss](experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/results/plots/vibethinker_sft_loss.png) ![SFT reasoning PPL vs step](experiments/2026-06-17_qwen3-0.6b_vibethinker-small-reasoning/results/plots/vibethinker_sft_reasoning_ppl.png) @@ -238,11 +315,11 @@ This model is the spine of a full **small-scale LLM lifecycle** run on one GB10 | Stage | Result | Evidence | |---|---|---| -| **Architecture** | bit-exact vs HF (`max\|Δlogits\| = 0.0`), 596,049,920 params | `verify.json` | -| **Pretrain — 3 builds @ 2 TPP** | faithful 28.65 · **IMU-1 23.52 (win, −17.9%)** · partial-RoPE 0.25 29.54 (loss); 0.10 died incomplete @ step 5450/18150 (~30%) | build logs (above) | -| **Optimizer ablation (clean, single-variable)** | NorMuon **beats** AdamW: wikitext **−0.474 bpb** (95% CI [0.444, 0.505]), code −0.502 bpb ([0.456, 0.547]) — **significant win** | ledger `2026-06-16_qwen3_normuon-vs-adamw` | +| **Architecture** | bit-exact vs HF **in fp32 on CPU, 5-token prompt** (`max\|Δlogits\| = 0.0`), 596,049,920 params. No GPU parity check exists. | `verify.json` | +| **Pretrain — 3 builds @ 2 TPP** | faithful 28.65 · **IMU-1 23.52 (−17.9%)** · partial-RoPE 0.25 29.54 (loss); 0.10 died incomplete @ step 5450/18150 (~30%). All n=1, single-seed, in-distribution val PPL on a leak-suspect tail — **directional, not a defended claim** | build logs (above) | +| **Optimizer ablation (clean, single-variable)** | NorMuon beats AdamW at **42M tokens**: wikitext **−0.474 bpb** (95% CI [0.443, 0.505]), code −0.502 bpb ([0.456, 0.547]) — significant *at that budget*. **The scaling ladder nulls its persistence: wikitext gap 0.474 → 0.126 → 0.072 (`CONVERGES`); code plateaus 0.176 → 0.177 and stays significant. Ladder ledger verdict `null`.** | ledger `2026-06-16_qwen3_normuon-vs-adamw`, `2026-07-05_…scaling-persistence` | | **De-confound attribution (Phase 1, single-variable, 3-seed, iso-FLOP)** | the IMU-1 win is **architecture**: **+arch −0.118/−0.305 bpb** (95% CI excludes 0, both corpora) is the **sole driver**; +WSD not significant on canonical BPB, +z-loss null → bundle = **NorMuon + arch modules** | ledger `2026-06-18_…imu1-deconfound-p1`, `verdict.json` | -| **Post-train — SFT** | reasoning OpenR1-Math PPL **14.26 → 11.60 (−18.7%)**; catastrophic forgetting **retained** (wikitext +0.2%, code −3.0%, fineweb-edu +0.74% — none significant). **n=1 → verdict inconclusive** | ledger `…vibethinker-small-reasoning` | +| **Post-train — SFT (3 seeds + iso-FLOP `--no_mask` control, 2026-06-30)** | held-out masked reasoning PPL **14.127 → 11.573 (−18.1%)** vs base; response-masking does **not** separate from the control (+0.009 masked *sig* / −0.006 full-seq *n.s.*) → **`directional` — not a win**. No catastrophic forgetting (FineWeb-Edu 21.495 → 21.652). Supersedes the n=1 VibeThinker probe. | `2026-06-27_qwen3-0.6b_sft-3seed/reasoning_verdict.json` | | **Paper** | consolidated single-model study **`qwen3-0.6b-study`** — status **drafting** (arXiv/HF source tree, PDF built via Tectonic, 14 API-verified refs); the earlier per-result *"Reproduce, Then Modernize…"* paper is **abandoned/superseded** by it | ledger `papers[]` (2 entries) | | **Harness-search side-quest** | Meta-Harness replication: on the bin-packing target, gated search **beat** the hand-designed baseline by **+5.3 pts** (95% CI [+4.5, +6.2], held-out) — but selecting by raw search-score crowned a brittle overfit (0.0 on an unseen seed). The transferable contribution is the **promotion gate** (held-out + brittle-exclusion + significance), which recovers the real win and refuses the brittle one; oracle-integrity fixes (codeharness reward-hack + seqpack module-shadowing) committed (`bdc5ec6`, tests 309→317). | `research/harness_search/` | @@ -416,9 +493,9 @@ proxy→canonical flip is exactly why we don't pre-commit the technique list. | 1 | **Phase 2 arch sub-drill** (Ch.3) | which of value-residual / LN-scaling / head-gating carries the win | **✅ DONE** — all 3 significant drivers (vr largest, ln parameter-free), `attributed`; downstream confirms | | 2 | **Data arm** (Ch.2) | fixed-token data-selection A/B — **dclm-edu vs FineWeb-Edu** (control reused), OOD-BPB, strict decontam (the *bitter-lesson* lever: the gap to 13.40 is data-not-skill) | **✅ DONE** — null on English, **large significant code win** (−0.70 bpb, PPL 1890→247); data composition beats method. `directional` (single budget). | | 2b | **Data-composition curve** (Ch.2) | add a **50/50 mix** arm (reuse both A/B arms, §C13) → does the code gain survive mixing without an English tradeoff? | **✅ DONE** — best-of-both: mix keeps English (on par) AND captures ~84% of the code win. The mix is the data for mid-training. | -| 3 | **Mid-training** (Ch.7) | anneal a base checkpoint on the **50/50 mix** @ low LR + RoPE context-extension | **🔄 NEXT** (auto) | +| 3 | **Mid-training** (Ch.7) | anneal a base checkpoint on the **50/50 mix** @ low LR + RoPE context-extension | **✅ DONE** (`2026-06-30_…midtrain-anneal`, `final_verdict: win`) | | 4 | **Serving export** (Ch.14) | vLLM registration shim — *pulled forward*, it unblocks GRPO rollouts | planned | -| 5 | **Post-training** (Ch.9–11) | SFT **≥3-seed** + paired control → DPO → GRPO/RLVR (turn the n=1 SFT into a real verdict) | planned | +| 5 | **Post-training** (Ch.9–11) | SFT **≥3-seed** + paired control → DPO → GRPO/RLVR | **SFT ✅ DONE** (`2026-06-27_…sft-3seed`, 2026-06-30 — `directional`, masking does not beat its control); DPO / GRPO still planned | | 6 | **Serving bench** (Ch.14) | `/serving-bench` continuous-batching + paged-KV + `--quant fp8`; `/observability-slo` SLOs | planned | | 7 | **Safety** (Ch.12) | `/safeguards-eval` + red-team passes (methodology demo; 0.6B isn't ASL-relevant) | planned | | 8 | **Interpretability** (Ch.13) | SAE / probing demo, control-floor-first | **on-box core BUILT** (`research/interp.py`: BatchTopK SAE + PCA/random floors + the CI-disjoint anti-laundering gate; `roc_auc`/`bootstrap_ci`/`mcnemar` verified vs sklearn/scipy; 342 tests). Honest verdict baked in: a **CI-backed null at the floor is the *passing* result** at 596M/1.19B (arXiv:2602.14111). Skill wrapper + GPU run pending | @@ -475,7 +552,7 @@ The three things that differ from Llama/SmolLM2: **per-head QK-Norm** (RMSNorm o ``` Qwen3-0.6B/ -├── model.py # the architecture, one file — verified bit-exact vs HF +├── model.py # the architecture, one file — bit-exact vs HF (fp32/CPU) ├── verify.py # parity gate ├── README.md # this index └── builds/ @@ -506,12 +583,14 @@ the machine); the scripts import [`safe_cuda`](../safe_cuda.py) to cap the proce ## Honest accounting (short) -- ✅ **Architecture** — verified bit-exact vs HF (`max|Δlogits| = 0.0`). -- ✅ **Reproduction gap is data, not skill** — 2.14× PPL gap against ~275,000× less - data, with a clean scaling curve. +- ✅ **Architecture** — verified bit-exact vs HF in fp32 on CPU (`max|Δlogits| = 0.0`, 5-token prompt; no GPU parity check). +- 🔶 **Reproduction gap looks like data, not skill** — 3.46× at 131M tokens (same-cache) + and ~2.1× at 1.19B (cross-cache, indicative). The trend mixes two val tails and every + point is n=1, so it is suggestive, not a clean scaling curve. - ✅ **Phase A LR (2.4e-3)** — an original verified finding (Qwen3 never published the 0.6B LR). -- 🔶 **Phase B** — baseline (28.65), IMU-1 (**23.52, a proven −18% win**), and +- 🔶 **Phase B** — baseline (28.65), IMU-1 (**23.52, a directional −17.9% delta — n=1, + single seed, no CI**), and partial-RoPE 0.25 (**29.54 — loses to baseline**) done; 0.10 **died incomplete** at step 5450/18150 (~30%; last eval 50.71). The partial-RoPE *vs* baseline comparison is **decided (it loses)**. - ✅ **IMU-1 attribution** — de-confounded across two phases (3-seed iso-FLOP, canonical BPB): diff --git a/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md b/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md index be76ce5..c82cc67 100644 --- a/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md +++ b/Qwen3-0.6B/builds/2026-06-08_reproduce-faithful_qwen3-0.6b/README.md @@ -52,7 +52,7 @@ This is a **NVIDIA GB10 unified-memory box**: CPU and GPU share one ~119 GB pool | `verify_run.py` | The Phase-5 bit-exact gate. Loads official Qwen3-0.6B-Base safetensors into our model (via `load_official_weights_into_ours` from `../../verify.py`), runs one prompt through both, computes `max\|Δlogits\|`, writes `results/verify.json`, asserts `< 1e-3` + argmax match. | `python verify_run.py` | | `throughput_probe.py` | Measures steady-state tok/s + peak mem at the training shape, compile OFF then ON, backing off `micro_batch` on OOM. Drives the cost gate. Writes `results/throughput_probe.json`. | `python throughput_probe.py` | | `run_lr_sweep.sh` | Phase-A matched-compute LR sweep: three 2000-step (~131M-token) runs, identical except peak LR — `lr17`=1.7e-3, `lr24`=2.4e-3, `lr30`=3.0e-3. | `./run_lr_sweep.sh` | -| `eval_original_vs_repro.py` | Evaluates the **published** Qwen3-0.6B-Base and our sweep checkpoints (`lr17/lr24/lr30`) with **identical eval code on the identical val slice** (from the shared tokcache) → the true reproduction gap. Writes `results/original_vs_repro.txt`. | `python eval_original_vs_repro.py` | +| `eval_original_vs_repro.py` | Evaluates the **released** Qwen3-0.6B-Base (our own eval of it) and our sweep checkpoints (`lr17/lr24/lr30`) with **identical eval code on the identical val slice** (from the shared tokcache) → the true reproduction gap. Writes `results/original_vs_repro.txt`. | `python eval_original_vs_repro.py` | | `wait_then_eval_original.sh` | Polls until the sweep's `train_qwen3.py` processes exit, then launches the original-vs-repro eval — keeps the GB10 at one GPU job at a time. | `./wait_then_eval_original.sh` | | `make_plots.py` | Reads a run's CSV+log → PNGs (loss/LR, LR schedule, grad-norm, peak-mem-vs-cap, val-PPL, combined dashboard) under `results/plots[_]/`. | `python make_plots.py [--run_name ]` | | `_build_results_notebook.py` | Regenerates `results.ipynb` (live-computed from `results/`). Re-run after training to refresh. | `python _build_results_notebook.py` | @@ -162,9 +162,9 @@ The faithful baseline re-trained at the Phase-A-winning LR (`2.4e-3`) on a **1,1 | Token budget | 1,189,478,400 (~1.19B) | | Baseline (random-init) PPL | 185,810.49 | | **Final val PPL** | **28.65** | -| Gap vs original (13.40) | **2.14×** | +| Gap vs original (13.40) — **CROSS-CACHE** | 2.14× (indicative only) | -This is the headline Build-1 number. Scaling 131M → 1.19B tokens at the same recipe **narrowed the gap to the original from 3.5× to 2.14×**. Mid-training eval (from `qwen3_baseline2tpp_train.log`) shows the monotone descent: PPL `60.10 (@2k) → 45.06 (@4k) → 39.44 (@6k) → 35.71 (@8k) → 33.04 (@10k) → 30.93 (@12k) → 29.59 (@14k) → 28.93 (@16k) → 28.66 (@18k) → 28.65 (final)`. Training ran ~2663 min (~44 hr) at ~7,480 tok/s; peak mem held flat at **52.4 GB** (well under the 109 GB cap). At 1.19B tokens the model reliably completes `"The capital of France is" → "Paris…"` (vs the lr24 @131M checkpoint, which did not). +This is the headline Build-1 number. Scaling 131M → 1.19B tokens at the same recipe **cut val PPL from 46.31 to 28.65**. The implied gap-to-original moves 3.5× → 2.14×, but those two ratios use **different val tails** (13.40 and 46.31 sit on `tokcache_133072000`; 28.65 on `tokcache_1191478400`), so the narrowing is indicative, not a strictly comparable trend. Mid-training eval (from `qwen3_baseline2tpp_train.log`) shows the monotone descent: PPL `60.10 (@2k) → 45.06 (@4k) → 39.44 (@6k) → 35.71 (@8k) → 33.04 (@10k) → 30.93 (@12k) → 29.59 (@14k) → 28.93 (@16k) → 28.66 (@18k) → 28.65 (final)`. Training ran 2,663.1 min (44.4 hr) at a final cumulative **7,444 tok/s** (7,480 was the step-100 reading); peak mem held flat at **52.4 GB** (well under the 109 GB cap). At 1.19B tokens the model reliably completes `"The capital of France is" → "Paris…"` (vs the lr24 @131M checkpoint, which did not). ![Faithful baseline — training dashboard (loss, val PPL, LR, grad-norm, peak mem)](results/plots/dashboard.png) @@ -186,5 +186,5 @@ This is the headline Build-1 number. Scaling 131M → 1.19B tokens at the same r - **Never raise `--micro_batch` above 4** at seq_len=4096 — it OOMs (probe-verified). Use `--grad_accum` for effective batch. - **`safe_cuda` must import before torch** — it sets the memory cap and `expandable_segments` before CUDA is initialized. The entrypoints already do this; preserve the import order if you edit them. - **Token cache files are huge** (`tokcache_1191478400_300000.pt` ≈ 9.5 GB; checkpoints ≈ 3.6 GB each) — they live in `results/` / this dir but are training scratch, not deliverables. -- **"Faithful" ≠ matching released Qwen3 quality.** It means faithful to the *architecture* (bit-exact) and to the paper's *LR shape*. The token budget is ~30,000× smaller than the real run; a 2.14× PPL gap at 1.19B tokens is the expected, honest outcome. +- **"Faithful" ≠ matching released Qwen3 quality.** It means faithful to the *architecture* (bit-exact) and to the paper's *LR shape*. The token budget is ~30,000× smaller than the real run; a ~2.1× PPL gap at 1.19B tokens is the expected, honest outcome (cross-cache, so treat the multiple as indicative). - The smoke `after.txt` says `smoke=True` whenever `--steps == 1000`; longer runs (including `baseline2tpp`) correctly report `smoke=False`. diff --git a/Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/README.md b/Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/README.md index 8c364db..a58b435 100644 --- a/Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/README.md +++ b/Qwen3-0.6B/builds/2026-06-08_reproduce-modernized_qwen3-0.6b/README.md @@ -165,15 +165,19 @@ Phase B run 2, completed 2026-06-14 (reached step 18,150, LR→0). Source: [`results/qwen3_imu1_2tpp_train.log`](results/qwen3_imu1_2tpp_train.log) (this trainer writes **no** `after.txt`; the final number is the `@18000` eval + the step-18,150 line). -- Config: `steps=18150`, **1,191,478,400 train tokens** (≈2 TPP), 65,536 tok/step, 224/198 split, NorMuon ~5,170 tok/s. +- Config: `steps=18150`, **1,189,478,400 train tokens** (≈2 TPP; 18,150 × 65,536, drawn from the 1,191,478,400-token cache), 65,536 tok/step, 224/198 split, NorMuon ~5,170 tok/s. - Eval descent: `@2000 44.38 → 35.62 → 32.54 → 30.41 → 29.14 → 28.43 → 27.40 → @16000 24.86 → @18000 **23.52**`. -- **Result: 23.52 vs the faithful baseline's 28.65 at matched 2 TPP → −17.9%.** Gap to the - original (13.40) is **1.76×** (vs the faithful 2.14×). The first *proven* matched-compute - win in this repo. +- **Result: 23.52 vs the faithful baseline's 28.65 at matched 2 TPP → −17.9%.** Those two + numbers share a val cache, so the −17.9% is a like-for-like delta — but it is **n=1, + single-seed, in-distribution val PPL with no CI: directional, not a proven win.** The gap to + our own eval of the released model (13.40) is 1.76× vs the faithful 2.14×, and those ratios + are **cross-cache** (13.40 sits on `tokcache_133072000`, these on `tokcache_1191478400`) — + indicative only. The defended attribution is the per-component 3-seed BPB result, not this + bundle delta. ![IMU-1 (NorMuon bundle) training dashboard — val PPL vs baseline, CE loss, the WSD-to-zero LR schedule, grad-norm, z-loss, throughput](results/plots/qwen3_imu1_2tpp_dashboard.png) -![Phase B final val PPL — IMU-1 wins at matched compute](../comparison/phaseB_final_ppl.png) +![Phase B final val PPL — IMU-1 lower than baseline at matched compute (n=1)](../comparison/phaseB_final_ppl.png) - ⚠️ **Confound:** this is the full bundle (NorMuon + value-residuals + LN-scaling + head-gating + **WSD-to-zero**) vs the baseline's cosine-to-3.2e-4 — a recipe-level win, **not** attributable to any single component (a NorMuon-only ablation would be needed). @@ -186,14 +190,16 @@ meaningless. Same-budget comparison only: | Token budget | Faithful (Build 1) | IMU-1 bundle (this build) | |---|---|---| | 65.5M (smoke) | 95.87 (`../2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_after.txt`) | **39.83** | -| **1.19B (2 TPP)** | **28.65** (faithful Phase B) | **🥇 23.52** (this build, done) | +| **1.19B (2 TPP)** | **28.65** (faithful Phase B) | **23.52** (this build, done — n=1) | -- **Matched-compute verdict (the 1.19B row): IMU-1 wins, 23.52 < 28.65 (−17.9%).** Both - used the same 2 TPP / 1.19B tokens and the same eval; this is the apples-to-apples result. +- **Matched-compute comparison (the 1.19B row): IMU-1 is lower, 23.52 vs 28.65 (−17.9%).** Both + used the same 2 TPP / 1.19B tokens, the same eval and the **same val cache**, so it is + apples-to-apples — but it is **n=1 per arm with no CI: directional, not a verdict.** The + defended attribution is the separate 3-seed per-component BPB result. - Compare **only within a row** — the smoke `39.83` (65.5M) must NOT be read against the `28.65`/`23.52` numbers (1.19B = 18× more data); that's why `39.83` looks "worse" than the 2-TPP numbers even though IMU-1 is the stronger recipe at equal data. -- Caveat (again): the win is the **full bundle** (incl. WSD-to-zero), not any single +- Caveat (again): the delta is the **full bundle** (incl. WSD-to-zero), not any single component — see the confound note above. --- diff --git a/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/README.md b/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/README.md index 7f23b6c..d0642ef 100644 --- a/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/README.md +++ b/Qwen3-0.6B/experiments/2026-06-16_qwen3_normuon-vs-adamw/results/plots/README.md @@ -7,7 +7,14 @@ and the fineweb PPLs were re-verified against the logs. Each figure is saved as Scope reminder (from RESULT.md): this is an **early-training convergence-speed signal at one architecture and one 42M-token budget**, single-variable optimizer swap on the 196 2D weights, n=3 seeds on a FIXED data split (seed 0). It is NOT a steady-state quality claim and there is -NO scaling curve — do not extrapolate. The off-baseline LR-sweep points are single-seed (labeled). +no scaling curve *at the time it was written*. One exists now: the budget ladder +`experiments/2026-07-05_qwen3-0.6b_scaling-persistence/` extended it to 168M and 420M at n=3 per +rung. On **wikitext-2** the gap converges — +0.474 → +0.126 → +0.072 bpb, with the OLS-fitted +edge at the top rung (0.0297) inside the 0.0368 noise floor. On **code_py** it erodes but +plateaus — +0.502 → +0.176 → +0.177 — and remains significant *and* above its noise floor at +420M (`edge_resolved: true`). `trend_verdict: CONVERGES` on both; run-level ledger verdict +`null`. **Do not read these bars as a demonstrated advantage at scale.** The off-baseline +LR-sweep points are single-seed (labeled). | # | File | Caption | Source file(s) | |---|------|---------|----------------| diff --git a/Qwen3-0.6B/results_overview/plots/README.md b/Qwen3-0.6B/results_overview/plots/README.md index 932a926..eb1bbac 100644 --- a/Qwen3-0.6B/results_overview/plots/README.md +++ b/Qwen3-0.6B/results_overview/plots/README.md @@ -15,8 +15,8 @@ each run's own training log header. | File | Caption | Source file(s) | |------|---------|----------------| -| `fig1_matched_compute_final_ppl_bar.{png,pdf}` | Matched-compute (1.19B-token) final val-PPL bar: Faithful 28.65, IMU-1 (Modernized) 23.52, partial-RoPE-25% (Exploratory) 29.54, with the published Qwen3-0.6B-Base 13.40 as a dashed reference line; each bar annotated with its gap-to-original (2.14x / 1.76x / 2.21x). All COMPLETE runs. | Faithful: `builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_after.txt`; IMU-1: `builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log` (eval@18000); pRoPE-25: `builds/2026-06-08_reproduce-exploratory_qwen3-0.6b/results/qwen3_prope25_2tpp_train.log` (DONE); original: `builds/.../faithful.../results/original_vs_repro.txt` | -| `fig2_eval_ppl_vs_tokens_faithful_vs_imu1.{png,pdf}` | Eval val-PPL vs tokens-seen (log y), Faithful vs IMU-1 overlaid; both COMPLETE and reach 1.19B tokens. IMU-1 sits below Faithful at every logged eval point and ends 23.52 vs 28.65. Published 13.40 shown as a dashed floor. step->tokens via tok/step=65,536 (verified in both logs); init PPL ~185k omitted (off-scale). | Faithful: `builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log` (eval lines) + `.../qwen3_baseline2tpp_after.txt` (final); IMU-1: `builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log` (eval lines) | +| `fig1_matched_compute_final_ppl_bar.{png,pdf}` | Matched-compute (1.19B-token) final val-PPL bar: Faithful 28.65, IMU-1 (Modernized) 23.52, partial-RoPE-25% (Exploratory) 29.54, with **our own eval** of the released Qwen3-0.6B-Base (13.40) as a dashed reference line; each bar annotated with its gap-to-original (2.14x / 1.76x / 2.21x — the last is a misrounding of 29.54/13.40 = 2.204, i.e. 2.20x). **Those multiples are CROSS-CACHE** — 13.40 was scored on `tokcache_133072000`, the bars on `tokcache_1191478400` — see Notes. All COMPLETE runs. | Faithful: `builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_after.txt`; IMU-1: `builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log` (eval@18000); pRoPE-25: `builds/2026-06-08_reproduce-exploratory_qwen3-0.6b/results/qwen3_prope25_2tpp_train.log` (DONE); original: `builds/.../faithful.../results/original_vs_repro.txt` | +| `fig2_eval_ppl_vs_tokens_faithful_vs_imu1.{png,pdf}` | Eval val-PPL vs tokens-seen (log y), Faithful vs IMU-1 overlaid; both COMPLETE and reach 1.19B tokens. IMU-1 sits below Faithful at every logged eval point and ends 23.52 vs 28.65. Our own eval of the released Base (13.40) shown as a dashed floor — *cross-cache vs these curves, see Notes*. step->tokens via tok/step=65,536 (verified in both logs); init PPL ~185k omitted (off-scale). | Faithful: `builds/2026-06-08_reproduce-faithful_qwen3-0.6b/results/qwen3_baseline2tpp_train.log` (eval lines) + `.../qwen3_baseline2tpp_after.txt` (final); IMU-1: `builds/2026-06-08_reproduce-modernized_qwen3-0.6b/results/qwen3_imu1_2tpp_train.log` (eval lines) | ## Exact data points plotted (cross-check) @@ -46,7 +46,17 @@ gap-x = final / 13.40: 28.65/13.40=2.138, 23.52/13.40=1.755, 29.54/13.40=2.204. - All three 2-TPP runs are COMPLETE (each logs `DONE` / an AFTER eval / a final checkpoint). No extrapolation, no fabricated points. -- The published Qwen3-0.6B-Base 13.40 is an EXTERNAL reference (36T-token model - evaluated on the same 300k-token val set, per `original_vs_repro.txt`); it is - drawn only as a reference line, never as a same-budget competitor. +- The released Qwen3-0.6B-Base 13.40 is **our own measurement** of that 36T-token + checkpoint (`original_vs_repro.txt`, 2026-06-09) — not a figure copied from the + Qwen3 tech report. Only the "36T tokens" label is theirs. It is drawn only as a + reference line, never as a same-budget competitor. +- **13.40 was scored on a DIFFERENT val tail than the 1.19B bars.** It uses + `tokcache_133072000_300000.pt` (hardcoded at `eval_original_vs_repro.py:22`); the + faithful / IMU-1 / pRoPE bars use `tokcache_1191478400_300000.pt`. The annotated + gap-to-original multiples are therefore **cross-cache**, not like-for-like. +- **⚠️ The committed figures carry two stale labels.** Their legend still reads + *"Published Qwen3-0.6B-Base = 13.40"* (`make_overview_plots.py:101,140`) although 13.40 is + **our own eval**, and fig1 annotates the pRoPE bar **2.21x** where 29.54/13.40 = 2.204 + (`make_overview_plots.py:56`). Neither the cross-cache caveat nor these fixes are in the + rendered images yet — fix those three literals in the generator, then regenerate. - Regenerate: `CUDA_VISIBLE_DEVICES="" MPLBACKEND=Agg python3 ../make_overview_plots.py` diff --git a/README.md b/README.md index 435187d..e7e12e5 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # BuildFromScratch From-scratch language-model reproductions — each built single-file from a blank -editor, verified **bit-exact** against the official HuggingFace weights, then +editor, verified **bit-exact in fp32 on CPU** against the official HuggingFace weights, then carried forward through a multi-stage research lifecycle (pretraining-era architecture/optimizer/data studies → post-training) where every cross-run claim is held to multi-seed CIs, iso-FLOP matching, and a held-out noise floor. | Path | What it is | |---|---| -| [`SmolLM2-134(base)/`](SmolLM2-134(base)/) | Single-file PyTorch reproduction of [SmolLM2-135M](https://huggingface.co/HuggingFaceTB/SmolLM2-135M) (134,515,008 params), verified **bit-exact** vs the official weights (`max \|Δlogits\| = 0.0`). Includes from-scratch training, continued pretraining on TinyStories, multi-axis parity diagnostics, in-domain vs OOD eval, and an `lm-evaluation-harness` wrapper. | -| [`Qwen3-0.6B/`](Qwen3-0.6B/) | Single-file PyTorch reproduction of [Qwen3-0.6B-Base](https://huggingface.co/Qwen/Qwen3-0.6B-Base), verified **bit-exact** (`max \|Δlogits\| = 0.0`), then a **full research lifecycle** on top (architecture / optimizer / data / post-training, below). See [its README](Qwen3-0.6B/README.md). | +| [`SmolLM2-134(base)/`](SmolLM2-134(base)/) | Single-file PyTorch reproduction of [SmolLM2-135M](https://huggingface.co/HuggingFaceTB/SmolLM2-135M) (134,515,008 params), verified **bit-exact in fp32 on CPU** vs the official weights (`max \|Δlogits\| = 0.0`; on GPU it is **not** bit-exact — final-logits max 4.72e-05, per-layer hidden-state max 1.95e-03 at layer 14 — see `SmolLM2-134(base)/results/comparison_with_hf.md`). Includes from-scratch training, continued pretraining on TinyStories, multi-axis parity diagnostics, in-domain vs OOD eval, and an `lm-evaluation-harness` wrapper. | +| [`Qwen3-0.6B/`](Qwen3-0.6B/) | Single-file PyTorch reproduction of [Qwen3-0.6B-Base](https://huggingface.co/Qwen/Qwen3-0.6B-Base), verified **bit-exact in fp32 on CPU** (`max \|Δlogits\| = 0.0`, single 5-token prompt; no GPU parity check exists for this model), then a **full research lifecycle** on top (architecture / optimizer / data / post-training, below). See [its README](Qwen3-0.6B/README.md). | > The repo is driven by a set of **local-only** Claude Code skills (an ML-research > loop whose scanning, briefing, data-prep and scoring stages run autonomously, @@ -25,10 +25,13 @@ lifecycle. Every headline carries BPB on ≥2 corpora, across-seed CIs, iso-FLOP matching, and a held-out noise floor — and is rewritten to exactly what the evidence supports. -**1 · Reproduction (done).** Faithful Qwen3-0.6B trained from scratch lands -within **2.14×** of the original's perplexity using **~30,000× less data** -(1.19B vs 36T tokens); an earlier 131M-token probe sat at 3.5× with ~275,000× -less data — each ~10× more data roughly halves the gap. +**1 · Reproduction (done).** Faithful Qwen3-0.6B trained from scratch reaches val PPL +**28.65** at 1.19B tokens, against **13.40** for the released Base (our own eval of it) +using **~30,000× less data** (1.19B vs 36T tokens). The implied **2.14×** ratio is +**cross-cache** — 28.65 was scored on the 1.19B run's val tail, 13.40 on the 131M run's +tail, and no same-cache score for the released model exists on disk — so read it as +indicative, not as a measured gap. The earlier 131M-token probe *is* same-cache: 46.31 +vs 13.40 = **3.46×** with ~275,000× less data. **2 · Architecture + optimizer — the "IMU-1" study (done).** A three-build experiment (faithful baseline / modernized *IMU-1* bundle / exploratory @@ -39,6 +42,12 @@ incomplete). A two-phase, single-variable, **3-seed, iso-FLOP de-confound** then **attributed** the win to **NorMuon** + the IMU-1 architecture modules (value-residual / layernorm-scaling / head-gating — each individually significant on canonical BPB), with learning-rate schedule and z-loss **not** significant. +The **NorMuon axis** of that attribution was isolated at a **42M-token** budget (the +schedule / z-loss / architecture axes ran at **131M tokens per cell**), and that NorMuon +strand **fades with budget** — the ladder in §7 returns `null`: the wikitext gap falls +0.474 → 0.072, and while every rung stays individually significant, the OLS-fitted edge at +the top rung (0.0297) lands inside the 0.0368 noise floor. On code the gap plateaus at ++0.177 and its fitted edge (0.1255 vs a 0.0463 floor) is still resolved. **3 · Data composition (done).** A pretraining data-mix curve found a **50/50 mix** to be best-of-both — it keeps English while capturing ~84% of the code win. @@ -102,26 +111,40 @@ with the Chen-2021 estimator, on decontaminated GSM8K + MATH-500) and tested it. The reasoning capability lives in SFT/distillation, not RL at this scale — the gate saved a multi-seed cohort before it was spent. -**7 · Scaling persistence of the NorMuon win (done, 2026-07-12).** Study #2 attributed -the IMU-1 win largely to **NorMuon**; this ladder asks whether its **+0.474 -wikitext BPB** edge over AdamW **persists or converges with budget**. At fixed -N=596M it sweeps the token budget — 42M (reused) + **168M ×{NorMuon,AdamW}×3 seeds** -+ **420M ×2 seeds**, ten cells — varying only `--steps`. All ten completed. +**7 · Scaling persistence of the NorMuon win (first 10 cells done 2026-07-12; the two 420M seed-2 cells 2026-07-25/26; re-scored at n=3 2026-07-28).** Study #2 isolated +**NorMuon** (at 42M tokens) as one strand of the IMU-1 win — note its de-confound arms were +all AdamW, so no iso-budget NorMuon-vs-architecture comparison exists. This ladder asks +whether NorMuon's **+0.474 wikitext BPB** edge over AdamW **persists or converges with budget**. At fixed +N=596M it sweeps the token budget — 42M (reused) + **168M** + **420M**, each +×{NorMuon,AdamW}×**3 seeds** — varying only `--steps`. All **12 newly-trained** cells +completed (the six 42M cells are reused from `2026-06-16_qwen3_normuon-vs-adamw`, so 18 are +scored in total); re-scored at n=3 on 2026-07-28 after the third 420M seed (one cell per +arm) landed. - **The gap shrinks with budget, and both corpora agree.** wikitext-2 (AdamW − NorMuon, BPB): **+0.474** [+0.443, +0.505] at 42M → **+0.126** - [+0.089, +0.163] at 168M → **+0.073** [−0.038, +0.184] at 420M — significant at - the two smaller budgets, **not significant** at the top. code_py: +0.502 → +0.176 - → +0.192. OLS over log10(tokens) gives slope **−0.416** (r² 0.92) on wikitext and - **−0.328** (r² 0.81) on code → **CONVERGES** on both. -- **Verdict: directional, not a headline** — the 420M rung is n=2 (< 3 seeds, §C17). -- **What is resolved, and what isn't.** The *slope* is resolved; the *edge at the top - rung* is not. At n=2 the 420M CI is wide enough to hold both "converged" and "still - ahead", and the code_py gap does not even shrink monotonically (+0.176 → +0.192 — - only the fitted slope is negative). A disclosed **inherited confound** cuts the same - way: both learning rates were tuned at the 42M horizon and never re-tuned per budget, - so part of the fade may be a mis-tuned-LR artifact rather than true convergence. - Earning more needs a 3rd 420M seed, an 840M rung, and a per-horizon LR check. + [+0.089, +0.163] at 168M → **+0.072** [+0.055, +0.088] at 420M. code_py: + **+0.502** [+0.456, +0.547] → **+0.176** [+0.137, +0.215] → **+0.177** + [+0.131, +0.223]. OLS over log10(tokens) gives slope **−0.417** (r² 0.92) on + wikitext and **−0.342** (r² 0.84) on code → **CONVERGES** on both. +- **Verdict: `null`** (ledger `2026-07-05_qwen3-0.6b_scaling-persistence`) — an + early-training speedup that converges away. Note the verdict was *also* capped by + construction: the §C25 `scaling` HARD battery is incomplete (no `log_rmse_r2`, + `holdout_extrapolation_pctdev`, `bootstrap_forecast_ci`), so `win` was unreachable + regardless — though the CONVERGES trend independently maps to `null` anyway. +- **What is resolved, and what isn't.** The *slope* is resolved on both corpora; the + *edge at the top rung* is resolved on **code** but not on **wikitext**. All six + rungs remain **nominally significant** (every CI excludes 0) — "falls within the + noise floor" refers to the *OLS-fitted* edge at the top rung, which on wikitext is + 0.0297 against a 0.0368 floor (`edge_resolved: false`), while on code it is 0.1255 + against 0.0463 (`edge_resolved: true`). The code_py gap also does not shrink + **monotonically**: it falls 0.502 → 0.176, then ticks up to 0.177 — flat between the + top two rungs, a **plateau**, even though the 3-point OLS still scores `CONVERGES` + (a negative slope carried largely by the 42M point). A disclosed **inherited confound** + cuts the same way: both learning rates were tuned at the 42M horizon and never + re-tuned per budget, so part of the fade may be a mis-tuned-LR artifact rather than + true convergence. Earning more needs an 840M rung and a per-horizon LR check. This + is a **budget** null at fixed N=596M — it says nothing about larger N. - Read it as: NorMuon looks like an **early-training speedup that converges away** — exactly what IMU-1's own Limitation #3 warned it might be. Numbers: `experiments/2026-07-05_qwen3-0.6b_scaling-persistence/verdict.json`. diff --git a/SmolLM2-134(base)/README.md b/SmolLM2-134(base)/README.md index 9829def..edb4a46 100644 --- a/SmolLM2-134(base)/README.md +++ b/SmolLM2-134(base)/README.md @@ -2,8 +2,13 @@ A from-scratch PyTorch reproduction of [SmolLM2-135M][hfmodel], faithful to the shipped `config.json` and the [SmolLM2 paper][paper]. The reproduction is -verified **bit-exact** against HuggingFace's reference `LlamaForCausalLM` in -this session (`max |Δlogits| = 0.0`). +verified **bit-exact in fp32 on CPU** against HuggingFace's reference +`LlamaForCausalLM` (`max |Δlogits| = 0.0`). On **GPU** it is close but *not* +bit-exact — final-logits max 4.72e-05, per-layer hidden-state max 1.95e-03 at +layer 14 (which exceeds the repo's own 1e-3 gate); see +[`results/comparison_with_hf.md`](results/comparison_with_hf.md) for why +(SDPA backend dispatch) and note that **no determinism flags are set anywhere +in this repo**. This README is the long-form script: read it top to bottom and you should be able to narrate every decision on camera without referring back to the paper. @@ -57,7 +62,7 @@ plots below). | wikitext-2 val perplexity — ours | **15.370989** | `results/perplexity.json` | | wikitext-2 val perplexity — HF | **15.370990** (Δ ≈ 9 × 10⁻⁷) | `results/perplexity.json` | | Argmax for `"The capital of France is"` | `' the'` (logit 14.023) — *Paris is only rank #2* | `results/topk_predictions.json` | -| Tokenization of that prompt | `[504, 3575, 282, 4649, 314]` | `results/summary.json` | +| Tokenization of that prompt | `[504, 3575, 282, 4649, 314]` | `results.ipynb` (executed cell output) + `results/POST_DATA.md:37` — *not* in `summary.json` | | TinyStories-val PPL, before → after | **6.8945 → 3.7900** (**−45.0%**) | `results/tinystories_{before,after}.txt` | | TinyStories run wall-clock (NVIDIA GB10, bf16) | **116.1 min**, 100M tokens, 24,414 steps | `results/tinystories_train.log` | | Best single-batch training loss | **0.9088** @ step 22,353 (deep in WSD decay) | `results/tinystories_train.csv` | @@ -137,7 +142,7 @@ character-driven dialogue) at the expected cost of out-of-domain quality. A 150-step from-scratch mini-run on a wikitext-2 slice with the nanotron-canonical recipe (AdamW(0.9, 0.95), peak LR 3e-3, WSD warmup 20 / decay 20%). Loss drops -11.254 → 6.321, well below the uniform baseline ln(49152) = 10.803 — proof the +11.254 → 6.288 (min 6.039 @ step 140), well below the uniform baseline ln(49152) = 10.803 — proof the training loop learns. ### 0.7 Architecture diagnostics diff --git a/SmolLM2-134(base)/results/README.md b/SmolLM2-134(base)/results/README.md index 4afb9e7..1c12c98 100644 --- a/SmolLM2-134(base)/results/README.md +++ b/SmolLM2-134(base)/results/README.md @@ -1,21 +1,24 @@ # SmolLM2-135M reproduction — results catalog Every file here is produced live by `../results.ipynb` (or by re-running -`../verify.py` / `../model.py`). No values are typed in by hand. +`../verify.py` / `../model_full.py`) — with two documented exceptions: the GPU-side +numbers in `comparison_with_hf.md` have no machine-written backing JSON on disk, and +the min-loss / tokenization values below are read out of `loss_curve.csv` and the +executed `results.ipynb` rather than `summary.json`. -## Headline numbers (also in `summary.json`) +## Headline numbers (most also in `summary.json`) | Metric | Value | |---|---| | Unique parameter count | **134,515,008** (target match ✓) | | `lm_head` tied to `embed_tokens` | True | -| `max │Δlogits│` vs HuggingFace `LlamaForCausalLM` | **0.000e+00** (fp32, same input) | +| `max │Δlogits│` vs HuggingFace `LlamaForCausalLM` (fp32, **CPU**) | **0.000e+00** — on GPU 4.72e-05 final-logits / 1.95e-03 per-layer, see `comparison_with_hf.md` | | Deterministic argmax for `"The capital of France is"` | `' the'` (logit 14.023) | | Runner-up token | `' Paris'` (logit 12.997, rank #2) | | Tokenization of `"The capital of France is"` | `[504, 3575, 282, 4649, 314]` | | Perplexity on wikitext-2 val (ours) | **15.371** | | Perplexity on wikitext-2 val (HF) | **15.371** (Δ ≈ 1e-6) | -| Demo from-scratch loss (start → end, 150 steps) | 11.254 → **6.321** | +| Demo from-scratch loss (start → end, 150 steps) | 11.254 → **6.288** (min 6.039 @ step 140) | | Baseline ln(vocab) = ln(49152) | 10.803 | ## Files @@ -36,8 +39,10 @@ Every file here is produced live by `../results.ipynb` (or by re-running - `generations.txt` — 4 prompts × 3 temperatures (greedy / 0.4 / 0.9) using the official weights loaded into our class. - `perplexity.json` — sliding-window CE perplexity on wikitext-2-raw-v1 - validation, 62,403 target tokens, seq=1024 stride=512. Ours vs HF agree to - 6 decimal places. + validation, 62,403 scored targets (= 31,743 distinct positions, ~1.97x + double-counted by the overlapping window), seq=1024 stride=512. Ours vs HF + agree to 5 decimal places (Δ = 8.7 × 10⁻⁷; the 6th differs: 15.370989 vs + 15.370990). ### Visualizations - `plots/rope_tables.png` — RoPE cos/sin tables, 256 positions × 64-dim head. diff --git a/SmolLM2-134(base)/results/comparison_with_hf.md b/SmolLM2-134(base)/results/comparison_with_hf.md index 401631f..3c58efa 100644 --- a/SmolLM2-134(base)/results/comparison_with_hf.md +++ b/SmolLM2-134(base)/results/comparison_with_hf.md @@ -88,5 +88,5 @@ Two real corrections in this session (both pre-comparison): match by construction. - **The model's own training-time loss curve**. HF doesn't publish the per-step loss log, so we have no point of comparison for our 150-step demo loss - trajectory (11.254 → 6.321) — only the qualitative shape against the WSD + trajectory (11.254 → 6.288) — only the qualitative shape against the WSD schedule. diff --git a/SmolLM2-134(base)/results/tinystories_summary.md b/SmolLM2-134(base)/results/tinystories_summary.md index d64701c..0c640cc 100644 --- a/SmolLM2-134(base)/results/tinystories_summary.md +++ b/SmolLM2-134(base)/results/tinystories_summary.md @@ -1,5 +1,19 @@ # TinyStories continued pretraining — final wrap-up +> **⚠️ This document describes an EARLIER TinyStories run, not the one whose artifacts are +> committed.** `POST_DATA.md:165` labels it `(prior run)`. Its numbers differ from the +> committed run and **must not be quoted for it**: +> +> | | This doc (prior run) | Committed run (`tinystories_train.log`, `checkpoint_tinystories.pt`) | +> |---|---|---| +> | After PPL | 3.7893 | **3.7900** (3.78995) | +> | Wall clock | 137.3 min | **116.1 min** | +> | Mean throughput | 12,150 tok/s | **14,356 tok/s** (cumulative) | +> +> The recipe table and throughput table below belong to the prior run. For the committed +> run use `tinystories_train.log`, `tinystories_train.csv`, and +> `tinystories_{before,after}.txt`. + ## Headline **SmolLM2-135M, 100M tokens of continued pretraining on roneneldan/TinyStories.** @@ -91,7 +105,10 @@ The decay phase delivered an additional **−0.03 nats** on top of the stable pl The drift is not thermal (max temp 72 °C, well below limit). Most likely shared-machine effect (5 users on the box); reproducible benchmarks under load would need exclusive access. -## Files this run produced +## Files the prior run produced + +> Note: the committed files of these names belong to the **116.1-min run**, not to the run +> this document describes. ``` checkpoint_tinystories.pt 269 MB — bf16 state_dict + metadata From 4a40aba55349c08a43898bea33e2a76ccac29244 Mon Sep 17 00:00:00 2001 From: Yash Bishnoi <87704585+yashb98@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:32:58 +0100 Subject: [PATCH 32/35] Correct last-bucket loss to 1.3138 and drop the unbacked model.py line count The 1.316 figure was the (23000,24000] bucket; the true last bucket (24000,24414] over 414 rows is 1.3138, matching tinystories_summary.md. The 198-line row cited wc -l model.py for a file that does not exist in this tree (only model_full.py does), so the row is removed rather than guessed. --- SmolLM2-134(base)/results/POST_DATA.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SmolLM2-134(base)/results/POST_DATA.md b/SmolLM2-134(base)/results/POST_DATA.md index 1f47156..295cf98 100644 --- a/SmolLM2-134(base)/results/POST_DATA.md +++ b/SmolLM2-134(base)/results/POST_DATA.md @@ -17,7 +17,6 @@ illustrative values. Source-of-truth pointer next to each one. | **116.1 min** | Wall-clock for the 100M-token TinyStories run on NVIDIA GB10 | `results/tinystories_train.log` | | **24,414 steps** | Total steps at seq_len 1024, batch 4 | `results/tinystories_train.csv` | | **0.9088** | Best single-step training loss (step 22,353, deep in WSD decay) | `results/tinystories_train.csv` | -| **198 lines** | `model.py` line count for the from-scratch architecture | `wc -l model.py` | --- @@ -54,7 +53,7 @@ SwiGLU · tied embeddings · no biases anywhere. target tokens. - Wall-clock: **116.1 minutes** on NVIDIA GB10. Sustained throughput ~14,300 tok/s through the second hour. -- Bucket-mean training loss (1000-step buckets): 1.586 (first) → **1.316** (last). +- Bucket-mean training loss (1000-step buckets): 1.586 (first) → **1.3138** (last bucket = (24000, 24414], 414 rows). - Best single-batch loss: **0.9088** at step 22,353 (deep in the decay phase). --- From a9a90e314c5b9b32af34e1b3b656484ce467959a Mon Sep 17 00:00:00 2001 From: Yash Bishnoi <87704585+yashb98@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:37:23 +0100 Subject: [PATCH 33/35] Correct the loss-curve caption to the true last bucket mean, 1.3138 1.316 was the (23000,24000] bucket. The final bucket (24000,24414] is 1.3138, which is what tinystories_summary.md already reports. --- SmolLM2-134(base)/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SmolLM2-134(base)/README.md b/SmolLM2-134(base)/README.md index edb4a46..fd9d105 100644 --- a/SmolLM2-134(base)/README.md +++ b/SmolLM2-134(base)/README.md @@ -132,7 +132,7 @@ character-driven dialogue) at the expected cost of out-of-domain quality. ![TinyStories continued-pretraining loss curve](results/plots/tinystories_loss_curve.png) 24,414 steps. Per-step loss (light band) with the 1000-step bucket mean -(1.586 → 1.316) and the LR schedule overlaid; the dashed line marks where the +(1.586 → 1.3138) and the LR schedule overlaid; the dashed line marks where the 20% linear decay begins (step 19,531). The decay phase delivered an extra ≈ −0.03 nats on top of the stable plateau. From 454326faadae93cd46c8d100c2e91d3553929be4 Mon Sep 17 00:00:00 2001 From: Yash Bishnoi <87704585+yashb98@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:38:33 +0100 Subject: [PATCH 34/35] Stop pointing param_count.log at a model.py that does not exist The architecture is in model_full.py. Keeping the recorded param count, dropping the unrunnable command. --- SmolLM2-134(base)/results/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SmolLM2-134(base)/results/README.md b/SmolLM2-134(base)/results/README.md index 1c12c98..a051ca6 100644 --- a/SmolLM2-134(base)/results/README.md +++ b/SmolLM2-134(base)/results/README.md @@ -25,7 +25,7 @@ executed `results.ipynb` rather than `summary.json`. ### Top-level summaries - `summary.json` — single-shot digest of every claim the notebook proves. -- `param_count.log` — output of `python3 model.py` (param count + random-init forward). +- `param_count.log` — recorded parameter count + random-init forward. The command it names, `python3 model.py`, is stale: there is no `model.py` in this tree. The architecture lives in `model_full.py`. - `parity.log` — output of `python3 verify.py` (architecture parity gate). - `training_recipe_resolved.json` — verified training hyperparameters from `huggingface/smollm/text/pretraining/smollm2/config_smollm2_135M.yaml` From 6911743f7e04b8b6086b851ab67dab0c6ac106a5 Mon Sep 17 00:00:00 2001 From: Yash Bishnoi <87704585+yashb98@users.noreply.github.com> Date: Wed, 5 Aug 2026 01:40:11 +0100 Subject: [PATCH 35/35] Call 1.00043 a parameter-count ratio, not a measured FLOP ratio No train_flops artifact exists for any Phase-B run, so the C18 5% iso-FLOP gate was never evaluated. 1.00043 is the +0.077% parameter-count ratio used as a proxy. The three sites that presented it as a measured FLOP ratio now say so; references to the gate as a protocol are left alone. --- Qwen3-0.6B/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Qwen3-0.6B/README.md b/Qwen3-0.6B/README.md index 5c4646f..6b18233 100644 --- a/Qwen3-0.6B/README.md +++ b/Qwen3-0.6B/README.md @@ -110,7 +110,7 @@ The three 1.19B rows **are** mutually comparable (same cache): IMU-1 23.52 vs fa per-component BPB attribution, not the single-seed bundle delta. *Caveat:* "matched compute" = matched **tokens** (1.19B); IMU-1 also ran NorMuon at **−30.5% throughput** (5,172 vs 7,444 tok/s final), which is **+43.9% wall-clock** (63.9 h vs 44.4 h) — uncounted by the 6ND FLOP model, - though params are iso-FLOP at 1.00043. No `train_flops` artifact exists for any Phase-B run, so + though the often-quoted 1.00043 is a **parameter-count** ratio, not a measured FLOP ratio. No `train_flops` artifact exists for any Phase-B run, so the §C18 ≤5% iso-FLOP gate was never actually evaluated for the three-build comparison. > **Rigor status (`text-lm-v3` downstream battery — RUN 2026-06-24):** the §C25 downstream battery @@ -337,8 +337,8 @@ bit-identical to the faithful model, so the baseline is genuinely faithful): | +z-loss | cosine | **1e-4** | off | | | +arch | cosine | 0 | **on** | model_imu1's 3 tweaks | -3 seeds/arm (paired), **iso-FLOP** (token-matched; +arch adds 0.077% params → FLOP -ratio 1.00043, within the 5% gate), 2000-step proxy (131M tok/cell, ~5h/cell, +3 seeds/arm (paired), **iso-token** (token-matched; +arch adds 0.077% params → **parameter-count** +ratio 1.00043, used as the proxy for the 5% gate — no measured FLOP count exists), 2000-step proxy (131M tok/cell, ~5h/cell, **~2.5 days** total). Verdict by the across-seed 95% CI (`eval_stats.seed_delta_significant`). NorMuon is already isolated (the win above), so it is excluded here. _The proxy-budget caveat played out exactly as designed: small @@ -360,8 +360,8 @@ run `2026-06-18_qwen3-0.6b_imu1-deconfound-p1`. **Pre-launch validation (the "tests" for this build).** Before any GPU budget: CPU dry-run of all **4 arms** (single-variable confirmed — baseline/wsd/zloss share an identical forward, only +arch changes it); GPU **smoke 4/4 + a resume round-trip** -(checkpoint → reload → continue); **iso-FLOP** check via `flop_accounting.py` (arch-on -adds 0.077% params → FLOP ratio **1.00043**, inside the 5% gate). Results→verdict is +(checkpoint → reload → continue); **iso-token** check via `flop_accounting.py` (arch-on +adds 0.077% params → **parameter-count** ratio **1.00043**, used as the proxy for the 5% gate; no measured FLOP artifact was written). Results→verdict is pre-wired: `score_cohort.py` (scores all 12 checkpoints — uses `model_imu1` with per-arm arch flags so the +arch checkpoints load) → `verdict.py` (`seed_delta_significant`, 15 tests green) → auto-fired by the conditional `post_cohort.sh` watcher on `cohort.done`.

Hbu*s9U|Je&S; z5Z#9>ml9OhRey5IxZ!@C+67e61VOe2m9#IPP#Bh(OFF;QvQ#8G=1<&<96sUDxIAq; zLD;$NuC)=o@7CJ5>uL^N&Nq6{bnG``!Fkw7uaBir&5cG2Vx`6~=IwQsD^+5BdTa$9>s?b!|>n=i$N?fjJ7q3BK^S-2lqLLM$ zuF}WX7Nxqzmm8D=`_K$8_vcojGU#jD1HEQ+?Re zrwfT12&AG!^pyA+!>DM8JV&n{p8a`_d+NJwr#qy?`KgXyyLJ#EpINW=5iB1OczSnB zDbyy>pFvJNF@)SNYTG#8X001>-w$@@q~nHw-+yg2`x!i+7$ip65kv!h-fu5MsN)Ad zZ%-x7uEO8hA-|kbm-e{nvlk4?2JfJE;mHO*5O!hD+<{7a9@vV^C4*9(b=Q2&sEU&A z^z8XKCAJcnW1j~o3KZ?*4fNsELSOl7gD4Tdc<0>I>|+i1nc2v$z>mhv>?51Qbk4kD zdVhaWx)2QR5P8*n{$3FT8M7D_oOr*dPp^67NT|s^o-@zXfgmmfYt8Zo&Gs5~loxJ_ z^*VT75DuAZ2Cl9Vf59g11AEbw>_NTYHtT#4VH5jm{2)2hHcG3Lh*Ewp(^$L@M(l_0 z|9Po*$#y9-)LL|S#FtDMiG))#(Ns}uS%24IsZ^0rIlYRg5huOLO`GjVP#+4$G$!0+ zZyidFhw1Q@70+LEW1}DlNKYd0T@{f+1~#l-LkXL76g)Ols&%25EJ}!lMLs?t7BTW! zkm?g9j2yRWNLQUevHwCjkiFJjYksX!9zC3dJ>2-REP7wd5|aO8_~Tj` zGqRA4)f!*TpR34V8^Egx?JX1livVVd9w!H)W|}e6h56}z=S5$$lh?=PnREX^O&m5) zzvEb+LZ=bv9r>K$)%H>{-f6fbntv{r#%$fE)JNoXS_MZ7=JomL<9(f@J39(D?j%Y- z6l3|<6i?tme6As?y#WxKm~h(2Q2egRGSF1>RHCp5KJ9JN;4Po0APV@PfT+gZ@>(?A}T{$faOyPlln{-EWZ7)b7+ zD;%1T`{Q8oi6}_)=)LHO-dO^L!u;yq zS){$VZ6XPgqPO+kP@NAgdQ9URb0$pv8}I;k5E=$n)nDC>J$|2~bANvPD>}!qPb30a zfKPZc@76uqAT)F<{kqM7=4c%)aTH63-R@+&{sHr_;G9w5mgHk9g zTmEex)UvxgIu~48(|=3z#g4YbpxWG2+z{QnmVE=6_d4Mv``RE^=GZqcT+$O9&QB^L zodBBQIjIuOQq)i^j8(!N|zlnK(< zwtGgP?HlBv1ApW>3Wz6w5Aq7@B=*Xwuji+-*>6CQ2Nnsyk4xnfn_zFqI<$w&{lvE| zU+zvTc^s05DSGivt5~BzZM#@AkZmsE(cv9z0>c7~LP&-Jj9e3#fQ5MwF36cp(9Jet ziB0fai*$jrbVxJkllXK>r&bvG1U#Zu(n3|d)n8(e9e9EziwG@q2)p; z0#zAW;{2^a=vZ*sLGx=^=uSaQPE79X=!C^GsbxB{55~!cOl9FX!=(6*jDQ$1Y+}A@uJRvsJ9q% zT|RyH3x9p-0N!kC*M6f^Kb6-0V|gmr=>PjDsKUr)@PHqP0x%6%N+z4?jZ@#>JT!X= zFhZudfe8FNGF;Rjf^70a0>eE6eaOzXw}d(`$Mh|aIN|v@5vhSJWN2)_L`jZJP*gbV z?zw(61PBZe6zk9>etddP4U~$cHZ>}$Dh(>u>Ne>`PnVEuanmi2EtWoBdgCju7yKPO=S=`jQ2zY(yw{|x~P{`Umz-Go>)%)$20HNn_9$W%%LVOA`zy#rx))d*Q1x3`qw_N zg@5;P-|tSSkHgLPRtkQIFHiEckS|z?Ymt_wT&i9rPk266T*q1V6yJwdcz+i)!h-XN zAJ}6Nc#+oO8i9q=L5S;T7N(D^W4_#L-#KEV+e7>AGsuZeP;#W<8{yi?PhLzR)PE>p z(*KR#Gcf=6ooA38*`t@e*_&gM|BSpS4sGyGBEpUdd~j7tACKK&oJrKP85{8QOq z;?wkh#=sdES^nH+7KT5s$IA96u($O;iSGY6dU*XRGs_OW6xbq+H1KU4>JO6wB*TL9P$H3-~DxIti|929= z|1<>tQ~vjlSD%sT-}ql9x*vaxU0N3A|KWfCpSb-G|NEExFC#NEH48l>8{;3d%ReSx z^t4RW40KGaj7&_dtbcz&1cU#dAc7m!q#Q7t;j$RKfujwRfcM({HU?OYlg7gcB9z!x zxJM9$R<*;yPeqryr!6KfelTndR+(lH$a8e24haBmezqN7jGx2s=xF)4*inCf$!9LM zmxJmZ(7bfUsek!z$IOjDd~E%SYS-8`P&D*5(jJO72|%luVRK!H$*&jTYWLR&qb^nRvX!_J#*s*9d}e4J&F zH>G;;`Oa{r_D6I?0`G|#>)Jjk4*ok@g2Q9~sftsf4 zOa2u=*vdM#f%BOKf?8Nhg%`b6QL={Y+KMF)9o=Kd+OVdR_PWPZM>( zT&jr6E#Z$IXN)q5G8Yrwmj()hSNMK1#7{B@BMTup%YXGWNhe68IgWtJ$$|>u8&x`t zHj$RqMBuX}HGutqztBT04d^>57@6rqtgndZ#WmhZ6gcEXV|DioNz^z0Yh#*flD z?$}f$IDey$`yp%~eGlF`6}ooF{wChC1{gw&!L4+_k4IYA=19JZGc*b&ZL@3P>H4i} zE-Rr$kAWPIk4?zs;+jGB$o(tZ&X5!L7ut@#PnhE3 zJrTP7*ckg9K4f|`kKPURqv*RY^)HdG1e0m4QGYY?e)4UJkQ=h7J8X;jJ0@=?>retz zer#)f=`|w@fYEE}1Nvsrip(3Oj+6p!=}F1$#K`T=J`Sb| z9Dh0y?-`>`FD*8wQ4x}wHLbiy@mc&GobScSI$M+ZT(vGzD4H^E8S`=ZP7Hg436yh~ zkhScn&j=WoIayu7oyv4%6olEd)IFG*`X&R=32ElA{m1c}`cA~aA&sY+y{7E#2lP1i z8o56IXNu9Myr}F$7ni-%pOj zbBz-!(T3D>jq92VSfj>t;qVE}a6EOJV&eoFrj>G&3i{ZiIxxG?spAl_gDDC^Jb#EA zTiON2Vh^u=k_B9+-iRM0w(qKV?!DVB7{54@htf7?c>jKY;6)VS1-iw{F{R-LaaAgM zh0?vET}58I-^I|B;Pv5{q6HDuF*$$62rm9yzOkx>knw@Q=m79l>6gj09h%Ovn^)ls(%YooBZ}W;TN#S|{R~k57n$fI^AyZb&H(Ee9FIIvw=Lk6N6TC*JEj)I@*L z%PQsONg*|ciz-5*krXygz^uo?HRVyarS9zw0`89*DO6*M97x-)HPi7#-hV$2BC5Km zPg>`^H?UQ*keTddKRzm8JRMEf#Ea+IkSDxU(HS?TY_6p2z&zhvl;gJ~RR&?s>@eJ= zW7oW=c`|3zstU>%PA)0q8u)n|*4Yd0S9mkWdzhB6Cv}sd;X2iSb52&QqX4iqis+b{ z%e|xSE>F{no#X>!3L7;TpnpJGMGSKjQ<$NgUe+)>BWj|NoGde)?dLG}(__l+CUG5; zJBfeS5Inr*oWnCw!AFi{H{;{^Tz`Y?1J!S_%?^91 zT{t<%fkPa62{B>>Euc{pQjl@{v?;0MDce|RcgW5sJE7VdAia`hI%=2P@a6+-nb$U# z+N&(!(bX|FNlO-(r*kB{_vpT>%Ph3{FQcuP3~Ok0frrKhjw@T-5q(|kc{V?y^&dV< z(IxvlJV?_!2V9q2;(xbB)8;M<1{!(fG*H4-(DH&Sd((353*)=TGB-U%Zy79I$&QnH zKOmetjJxl;l)YEJGiyiSeim68TL)z=QLrZ7?I`UFbJo6Iz_JCA znb%76JaEMDVP2tE#nv8O%m_`a z+|>jZSc=IV+zK`FM_xjqs3Zn76IVZPyK{4|pylQOjTUlh*~>4m%Y0_kW4seAN zeR3@=nciypDN1gDgjo<1q5)J+g|`0Rb3XKmxbnuP4}S-LNyCkaD}Jz@ubj(#VJx!{Fu4WLpSsK-w>0)z-^*mHZEJjV_ZFKJ zr&e1TfD^NP1Ma`(-J#bA%gVHWrX?cQ5wN>?>J$&-Ch1z_o6pSQ2 zo8poqw;Fna>gO^I_3fG1T_Su1?eG+julgRNcm?LA-G9M*5p0Wzb6U;M#wP+P%rD#) z6r8$ViMjdOIx>GERBTpP@>FZIT{SX$^`^(Q2K6S{5(oC0R^--Zm~eD<|Nb4nL3w#M6-F)o~i+|($*o26=j$#d;7Ml?SKRAK~zpmlA z*R^+fQ$s2^`2BMs;-2KcV8H1a=;;4T2Au9slP@}4#=nWI{`I5-9WBFuKM~th_Hb2P zY}9%Gc*-6gB}@||#={#mgh!2q*9O~x5<;WxAOp%}^qrNYPYV>mCjKcD8Hiq~+sZ9V z4u6MJsC{Myr?9r}9~PdcLMm85^wQP2;(kfs+PZ`P6*@wy*HKYMJtwNRr++=vj+&xWX#=M1W$PbtJIt9pgl=)BjBYUn zSISngt$|=>O-SM*UqK=>xdv6cy)EAvUx6s9SyCA-xcNx4#e2}v<3HL+DeAuFh7f1L zxxd@!M8#5Ug9IFzLO-RtE{l{K@$hTEF*4x{pM{0}h+~MhE6J{gTV6$LCv9sa^?x3r ztv0u{@Y~FsfWFdmn3-9LWea`v^XL9P=6$>2;vcirNnIfMrU`W7sKU~YTQ!Je`MEVw z2`(dBsMg141g#^>bEZY9#{={SgbId>)xmpTnb<1bt=|$-v^1BhOR!DTJ(h!=IoP<} zSkK9#3I*|N1X50q3)-uQYC@Mvaer^dMU9J6k}Qmsg-_{cAV(JIWy`W2Rt4Es=H(P> zb;R;GxlfI8!M}yrzMtAeC8!#gt7{3NnBv&YZPZn!lc{Ro(2E+E-h7&dr!!(<&^I$F zO7R;L^mKKH&Jh{pTO7ftd+?wHpqF;+DWWDL@2-mHAwzWDg+l73KkguyQGYR4pQgen zpfxfk-GDCHB`~II7}B@4w|!h^Ja&A>yENsN)SZ)KLc{Rk@~z*XD5~-{5k;!BzELGA z-Ktf^y(+2=$0zTgYZH&zUxCmnwh28diun-bh&D$wMY-|XPSM5f#!Z5>!S%?SJbxrjgGLC9R93iw1|pS|zL#Ne+a`>**07D)bKdu=KN0 zg|UP~Na51*22OHOSdUXviP;BI7U`TK-Ah#x>khTsMrj8r_6X>t6T?0)oU*4PG=&o) z#r6blf0}&2W(lUzmj+zHkwvzZhTMDSaOW9Q8$@2obq<*>N`#z3c7IOU=PmZRlRMA~ zKicz%$jq5nO`z$OXoqa4kr)gCpCL43ctk?zr_#0*hiXb-=0h;kX^H>FH10(}`va7%h@jar#r!`JKUN9!L}6M9q`6<(B(P(hNM0hOS=;HZZq@e&)Lr( z%Ic^5at6Jq)v{^&tk8~F9n#PkSZ0Z9M;k{s^{tt37;=ui{oiRMZNe$mv(CX=a$qP4ioXsuVD^sDG}g?xdZHEh0RMH*&EZomzY@ z*iX2HX(Cdh45m=DxUN`E8(6J6PA{A~53yCG5Ak8#`mSQ@xz`;6teJ$v42DQI-&Pa! zkL`9Bj}>h^8{p`x8m9`axo?|n4>b?==Svtd#d8Bjr`e#PH;=eecX3JN0kREUgp(cd zxpu^4?tj_m8RCvRS5#cft`&1^L~CgV-_(N6-cvnpUurV$I$!tiB<>4IQGyakq%i5! zU*u`%@Nwrp>=SPefsN9){SuRwb-L@GJCmKL&bJtjHr!LZMRL9@>*)kE$$|%JIvsaa zkk3=NEvhQQ1s^~N99|;Uw^$3^+hrCm8)O6xUVo=xqO0kxMYG>P$DCG0mPXg(x4Ia8 z<{a&G>|@&3y2`n`Ge*>k9YQ^fI0s!`;+k(+c;+SO-D)Z3G3X$KTtMcSHQRZB)geH{ggkrBFl`3qG zW(5}!bJ@vtPc_R--fN#O8!Zrts*H0UCG#!KzG(8KyD*)|MAWBimFs1= zG&I}PnOMNmj&7J>B`%QHK<_|yr5y%f!2pXB7ble#j4G7 zsZw3M?^vyp^_0IHp}->}TkQYh2H-t}%oDLsSnWd}xQu;MvK+iSQ10!d52Sr5CRhD7 zo8Z4+!JQpo;QjGnn(|gUG)r)JW{MKsG)l1hrNVzE$`&1XNP+j!J~S@Z@kiaA1$L|$ zH0B+@?M`0F&SWvCd6EjVbASAgv$4vuDF*5{xgRfN9bJgZt#R72Z%e8SIrn?{pq*v+ zeTQV`ZuSvBGKOLpy-R9+etE}+O*wK-o^xUib7wyd-XgB^Bn5SHURn)4Gj*{5e3#u8 z47o5pnSMlskuY}EbBK9~7`VVYha5Jy%5F#R+c~>&`VPCgQ$~xp-G9dz@54oD!MK!a zp~JW&mWQ<8*k)X`kQA_w&B7&fFk$Z%@8d<;L=8eaZ@~R}-sf}nq74P*T$#$dytm_# zdA8UXc50!)dUm&y`So2|rtCmlN zkRiEIG+Sm@J~YH?9e-vJFeG*UT#~`KbluyQOxZ{solMC%Ji8E%!Gw+ht|;*HY`-MJ zzzWlXQQ4wmuB7s=WQ~8pbO2lqi%F?bctMbnvYcE>>5#85OjIa6{DL>=rZk*!LwC zIEqo(94j)_L#a@j>Pf!}fGX{FHFAO87iUiT^>9b4n0YpgLjo;H^lJTwxih-)D;-00 zyhXgC7=4vkU*sts6G8N8$I%09-+a81s=?Z#n7KNAm2{ty0>YxwX=-{XjR>7k{z{C` zTnjx?0tu8-f`33N<7_oC)L?;teO_7#@f%&O6f%;#VjPZ;p4v%o;H}33P}FV5*9#Zw zaqo)aABVxbp#iyAWO-l(TJ~@nAv*E>aK=c?Jp}?lR5(FW-`)O}ANe7mv=bqo;)<_! z2SfxklmXrc>A!$t6S7L}`A4Lsl~&V6xO+^7yyz+51b-Ng%X2G74t}TI$K}!ugg8bg zV~e1V&+*@g#GMI*2glUP^kWFw$Ey$(_;1VcYBB6u7Q4xpwx+S zOWMA$!Ymi-NuByL1VfVKBC0xaUWpZ(a1o4{b3Y}!)ap~At{`f=3 z!t{SEI*IN-L?^NQeRLB2zn*1a{+s{)Z+}Z!nEx(vho1E>PCC%hGySby|MID2rDdf3 z2QL4(yLl=NZa&)>Ph~nhbG5R(Yt^I1>*9xnC2At^`|<(^isJeO0{ybU*AO)Y2*=Qe zKtxXhB~llcgS(dv@2ynLUjlt?0AE^OQr6fnezY?8dY@4yV|)IbY-RDoT^i=e_J5#1 zo|s6xY@T{%VdMSL6U_o}IZXxWTJ-B1F0Ui+5Nm%VV8t8;A9d z;m1Gklm@U6%SwDYLrWm+=pDh&)RlZ+u`X2I8s#Kb7Yz$$?KK@wV^fkI( z8)A}hfUwaDIW#XAURrAN?k~RozR~iv^Ur=?TcWah-@-l{rk^57utyPf*cN(*;#P#11qQ(?VPgUmT1@*#x(BcE+HzCyjtn(9PQS^_>AH=F%;fp`NU zRRPLyy`B}jcCC&flILl$H?|wI{dUUv@E1Of#dDO(B6q~;>G;0SB5`Uk`_qA%b?H3yzgo8Zj2Uiw9G|C-+#2a(_c>#be;B7Sf zOr2SguQ}3iXOw+d&y1o+gIX8(CDL?Cxq4e{x_G;W;-9=3XtvfATWs2LT5gEZ{u1*^ zQ$tQs61pz?ZA=z=&+$%C6uK@7`)iM`Hfn9q*I4Ju{a1$)8g$*Bn}2gkHSL9VR83}T zZ9o>vOjTBao8=f9oqN&jF3agdP*gP@m*jHuZ5?({7uwB#thZ5Pd1>F#9D3W$Ps0>l zsa@M)7ildHyC?+^w3a|1L@g>jIW4%Um-@^r>M4~)Gs3!lKeM(nI2k&FZ0TejNbzLfheb~8~Vr-LBghfN;TAL z_;+##pCi=-CIr1tZgh8=2dW3BBeNq4aU=vHLsf6--G3$5sG&&}Ymu{rOF3a= z6vQ!hwSTbL63>_)YO4ORHbK%b@#qFRce$NK!+MW;D62}ZSU8nl%?h#_Jx ziOiDLk|z*xIKIpPq7IBuYeTl`Fp( zT%#>RTz2qS1;%%AYO<92sXLzf-5FP1LVqfzLxl`Ty8fhuvoK-%z`A^cZ0bWU_>SG6 zX&+&;S_~PbF#N^c%ami_bm4qq_ai#Cz0lWDkUT_d_J6SsWJRbs;Zr`q*^2MeG6YZ_ z&`sXCw_LwcRUwE%3xr13Jf!}B*q&wGk76BPxo6w2e$0`Krj5KXLWhmd9-_|metcvmYNnJWW4tsF$~kFRp=FK&~K$TD=B?U-W8=qUC=J^H1| z%H9f6J*Fj@v`m7Br0`9!OF`A;{Il&9@l`+fBzWd=Sra+PI<(Y5-P){fMp?}Z(k^Vv zlr2md$AK&R7Qj`6er zf>*NN!X{?2)!X~<`{ZNegxLa@`43-^*B|gd_ z3*B*Oj?{Ks;^Eld>RzY5Z{@o=B{<;Qcp{jFBqb*?S70Q#32cWasbEF$O+aHt__@XL zR)5E!pwba$!9xTxh+im!L*M!?D9dl7k=$O7k=z20ed#E(qWx=4xFXWqrZjD}%$YrC zDSN7!i28VD02#PRwe)7#86;_EwfUXu=k-)eCK*e`CA@k+O}^Q-MO-eFW6FxfP+vRY z)2!nGM(luCnlS@9;YA_vMI`+h9EOU;JGFv9)Dh89lUuEza`bH=TDI)1&#bbAH+b`gZTJj2!`C& zi&fwm$J}!E4Df+R0F?JJy1cL}@1UM9Jp;P$3)H_3Y~?JWF7T`T1ngMN1X<+R8pFs^ zew_gwiq$b2ENt9?42PnVki#m(r8B_&zJ8;o;%A@63X7i9Sv=uDfmR3}SAQ|=LlhD& zJY;3j#Ysaj95lhajlEfu?(6EoAm&%qgKO7uF zZWFgAc}{pg;kxS_n?C@1MSn90k8OxyjkCpHk+3}I8B|%^F7r`J(H4yV4*KXwopeIs zLD{mDJ|OUhd;Z0dmo%zZ?Cl4LOgQN<0j-{(&}na<$!6M4n?LQZtY4+gnmBLkCG>kAAWd$l-KsB0hNJRdu?i$1Fl z5i!)$$fHD#i zRCK5c^Mvrt3rgo|zJGLDl?}Lz)7%^ki?PO%{A9#L8i|HJMVB8luetNZa-J^{@s4^u zPp3TGLQ2u$v-p|X7k+&ba|k6c9#Oa%@L}^wz-lf1C2e&x7TxLXHv*P-ih(`Ru6 z;?!U-MuuYOVnS2C(>d94iQ?dQ5OExCgeYl1T_3KennQET=YKwTDeCH7X5FYH_x43CCndMtYA=*@w?v5U>6LzHt-ZjrKXS!Q2I?`ku_SgPz4-I|S zmY==_1q33rcL!AIA?i3!VwvM5(}{MDu|##oT)#yzow7;k57Wyd&8lTLAp~9tm=C66 z)MxF2z?Go%K!3|*~ z{dT8kYG}Ip>w~70@;iXCZ}^$TpjkMuoHLTzDKmX%1)+>RK|UBy`c0Oc7#TDI!A;{$ zT~2ga@nqvC>3dD=-qoejkm^i=JtE5SOBJ%M@K?Qa)_?Vj#6x+>8QfGyutlNN9=`0F zAckSi#6D+U_X|s3c;WCn<*GSoU3cSWbk>WZqX1Rdb*1)Zhf}Ubl5VYY0b3Ec5D>_@ zAh;k9l--qy9o^XxoZX!XQSbL{qF~y~_scgXZ=tTBZmq5^Z`U`350qA31q7;VmlrIq z9v|O40DpNPpN!A#2=7hUI@#JC*6hpd#T^V6Fp){?eqR;HwBV%*WOPzhFLysd^lNQjBBgsA5n(eg@_L{S1uBSL5oR z%S)>jiqCrnI-OJFChb8A4!DZlPwys>PVvgZ-y#kGh1PeNc`~_@%y%x9$U%3l5UPn! zCx0v_WMBYtbp(8Y<6_fs`7T7%Y^}rNvmG^P)DM_2-Gs$8-_`UHbZFsz@AXMoorAsU zU}}B1v0c&n9k-g~;4QVZQ28=%5mVm1!C2nbD%PW;I4U%fe_AqVMoW-snWL9sRAt)> zNiR_*dg?g&v=*wR%hbGW(N62jIMfj?%YXFPSG*7B&I$x{CoFAYmxa<<8qA(IwMP#p zF8LEvE>~h+pxLv#>_~UFLkQ-o+O>>Z_EGQ>G{?!R^sAq%{z(EsIvR(k)b;E8(%n|a z`bsS4IzR`LAU8_ML69)~{E)20k!0+_wgsOm$z7qP|4K|$GJaz;H6O@@)swvykberE z!5D%`t#wn6rq+L5L(g8NX0=FKBa2}0v~IO>$3F0aI^tqkkR0M1oFd{<*`qXT5eEDn ze#wz4cSz6-gJKFUH`|ie?8%r`9tT~b9pky-2U^@^Ae>o!`ssJ$;Rr>>S`kmPtKuI&OU>eCe7XRhlwd#peT2s?J8E}U+Ms2Uyyt* z@xslw^*b~HpaBs9g?!pP1+B;)mKbP=cgX7o8bvdUH!aGhM3oIml}n9h8h=ogLrc0A zB}|{*$DdUpSupJt?V0VG?R(CH*DhBsqxv*F+9d)hx!;ucDQTT)g=z9!XhhlXUE(q` zY&FlQ>y1Xk>p_-%g#>jE3x1Q`We|*68uz2!ZS2OK>$h{)QjR&3HrF3l^Pnz;G4+?8 zB{#tdX=yl_Ucm>J>Do-L?0+u6=10;4y2p%_r!X>%w)u-C$IAJUVe~*XnHrTt@YV#x zTIG^$v>@^AP{aX)lxSJZozQW3`g#li0ZB{UZh0tWzDp3grEZtK9}*=BuhBGQi=;Oks~9RsDAD~`s(-Fx<-x3gQ@nD1 zy09E6bITpHTS9m73(eZ>kMmsv+!k*#(_>X36e^6DZHdEMYfR^j+6CR1b~l}(J^&vk zu0_8=pQUQ?%rN58G^jub7BgA~&F$Dr#-ov#4Z{it1}=CXkgB9DveHs3PkvbvqX|&9 zCmFU>;z`iTk;Vw!2Y*Ct7FtijfEU(Uc-!IwQ0y{vt-%caHgv=h=bvQX@+}i(4C1Wi zC}j)_zwBo#BbRLHf?JGbw&rQnvQcey-S*gm-o05H_wIDthIx3HvBmNttmdq8pPhz* zxlYpR2Su(t-9*R?c=F5q?KT1i7t$D1-R_LJ1mq;&9N8ZDD}Rt4kC2Qn3s({E5m0X{ zc%8elo7T0BRrqM9usU=iI^+?+melGE#48EgDAX@i-j}iA`S`));k!u3*~|V&rxP|| zX1IkK@Pk`3+F~}W(Kbt3PYqzy#7tCsy7za#OA8hL6`e0%#fGfEZFa47$^5u`7T-x3 z>k&k5RtyN8oPQo<8w~{ij_c;ii6l#%c^08ESJ=F(y6Ewoxpt++E`{UziMBZ{U6vw# z3aOp~RQc|G(rX)%XJd!_sV!&AI9L&z_*hlHwcC~{|x8%}|u<$0z+kLNHsNqw2iFwNt#CE%n)ISehGm|nVh#^=q zq~ro-b(&IvvQraLuZ!%kvlCG_{o~;i+~no;!-mcE<-vzu38i)7t}Z3dq-hfZ2MlBS z&3DC&g?|C*^(_@}iAFQwaM9?e#l<1bd#K@(&IL8(0n0vKYuMr!BbKi!mMX?5TfI$B zKEg}@pngtvF`mQ2ZeQ6B_ceqCpb$eJ=9g@$WUQ6+Mio$m)4ikwT(F@{neh$`!7d2d zbC0}|;j{Yfs>Agw9i3D+_0qip5VciC<+XNPet){0I8H)bxp7!Yd^=ApZjjh_pTbPI z8h9NOG@Wf*uYQI~Gt7z9lMK?BdhgfnYCX)#n)US+;!`kTYSekIw@Gbtf5cYX4=`T| zof7ucTDZ}ShrYj=UoUz=T{~?qyI1#GcvREWDz0A*=hv@g4Gg)~l&>NiETS$yQfw!G zJ%7ImFnr6rt4=?jA@q9`26qi5rSFGdUid>e3#uvfMYn*9md&{IS>_Du+pDH-qc8w& z3_pTOX-Gv#6%vm?J`2h^}N{}``hw3HS9iz#;uJ+r8tYyK!nOa!=s_eaT^l=x3N%`se#t#v=HEhxd97`G4uP zY0{xfN1oGdz36p2$M@v3@zlnW?QO1T>2#XP#)s3vi)Zw-c=#E+q^j)0_7V_gDvjyb z#ZbuL{(2fyc^dD}Il39={AZy0;Lw3cI(U)=+sZ*;Of*rOh2X;OB2bR?nIY*_7 zAi-db7&E!92~x7OS95%bV2+OhL4R9yK^bvoiEWm4$UF8`hMe`V``UhrP4e<*NK!5pUlN50+ zXtzmVNYIgLm_yg!`FZW7kzFp_UXW@WBmWaf0s2{BMz zTCft^vA|L}H!htMSyQ*f)Hm&BDzE9=;1M{lURaC!4H-{nIp7CcmwyD;dytNd!d6l5 zCdprnIk?#F#DFCGa++1H#)bsr9VH z8#>9fQ%-kWwl;JL#Eja%`z3~uJo-qOuP|+#Pcwx`JlV9*YaQf6wT}=!n)vJ61%E~_ z{Gh*rM5ENeEwqFpn}4;mOQcA!{Lu;o*k?^mppUN!U^L*6Dv}j^NUrkJe}u*0hZm(` z8)Q6U6_&P5k;}YQBAMdwjN^eT_MVU^A^RaH#qS45w(Ym<5*$}F~}6Ox_{ZS%DOxQ&6oyNs#Q{K zho(`;Jv};%NDzglSyNIvj7MhLJL;xI{fnkFCOv?8wbn`URbkOC;>P&j$^z!8D8;!p zL!3cYwBP<9ESSp&y`dZnTHIrfdJ&#BMpPV}~U`Ro#$18^}!SpI~f z^^}8e`U16w!hh$CpNs;nts8UDmjzkYOE%1w^w#uFs_Yfd=gwy?s8{q`Mr(Rq({q4a z1tCwUPiW6+?i0&+IITg33HS{K@_mfT+G|g$%h?S{3@Y2|AFmYCa&PwI?d4A@E>-d2 z)L6z4xafdmE_m^fy#{((Ag&0J(&s8x1j`$IjGH*DJb%7EC&HKR^_@!64l@N+vBguf zRt-l&a;aAMTX)0vwzB$)1IZ_1t)mfg#mlMjnFH_&Zs;ry8S{OW(&hRXbyAq<=yRB7 zrp}TzEbuo-$Vgj^cl?Op2kr9j>1Ub*yW0|F{>@zl+!{4TuMxxg*IhH>sPM{-f&zV} zsxKc__kV&>uJ}#7TedmDbqKTZ6YzxdlJNBD6LwDpHhuUWi9**fny8qr-%UCZ`Mzr7J;m{Fzl`kH9t6#%7&ic_V@rlFGW-VSF-N`Kg3@hB|1i65gR_|ji0G#>BrRmkKj zNq!kVcutv}TOF*>05+5vj{zOB8e7VHTSqgRN8jtcQ3#*Tt#^hhc^ym)} z23GLzCKv!}A%g>0$XvugVv=)qA^mrS0)BCVqNz4cG{Otw4Gw4JGN7uVnt#l-g*MJc zX|mut2={-myAJQjde1@g)qj70lT8P6zF4goW4L%(8%vKQ#FhjsIDWs_^a1)Lg4$@8 zLfDON>6UAN9Wt?*F0)!VYn5=&E#g3F%VGr!LlxU%PLJ-lvbBnBB#a<%g4vxWNTzsMSO zjdr8Il4=BvXYMEHV}8V<=XzG+;QsAD!x2KVx{glre$gHuTg=+}Qa5-s*}FAKm2G08nbIw))-vp5b~n)&^j zco)GZ@%LQOwO>LX@Q9;Gv?1+1gkC^Qb2z!^_w$8DHNSEq3h-0~w*i0vWu+r>HQhPi z6sLGwOA@$|FY!@U9VbizGJtj49{?_rv_mrwtM$4sOy%)Qm^g+j7+}zMhHb+th`fIe zTpYo$bQxBEnnfY*-oN4;I^At2Iv*lw6<$eyT7J6qv-V#{GDPBJAg6g-Z|UYHzgKL$ z)m4h^8-Qd4asLLE93jI999$Nk;GY~}5_@+x^Lw!toFlj&ve@Pb1cf<`JTwXf-VV^nMOuIR4YPTLHm;kt`# zS^sm%Kpj*f7jCJCwIU0|qSlN0$U1n@)lF~eMD@IJPTrGxvFoQsxcwAx|F`Z^->h1> zU6B;)viL z%2dRshTW6!<>uo4#GWE9F)#oS4!|ii@kD|KG_V|An>!eF@{cWXH z7sV&vG3T<2FRaoJ@1#pnqvykV4<%<6&h@G~4khQ1v8GbX^079hB_@J__Vm?erKZ6t zc8=`hEb{Xe&1sHo6Rh=u$xFPh)@&X4jQFBjhg59IE=7!pRD> zcBbbV;v_0F##5O2n@v(>*%*6^j}`2clTfv6#vcp8ULU5{sKCZB}>yqGbN>%nVFdxQq0WE%*@Qp%*@Q3 zVwO_O%*;$Tt7odG=H%(_>O0eK=Dv0QwIbwUVPS6OX5lWE{=Gjd<5>kI?)k|vMdta* zQ3Yl}!+1rfQ!0PWH>Je&nE@*$)hB$4&S7Fr0|W@`Z3IQ9*(p`91F?fTJ*Krmt1~@8 z70LuVmu)Hwzfmh0RbH^M?%~07m8?-a1!k#%?{vY&+EZ2awUkD?s$Y{*N9kv%IP+5r z)nQrCTu8IONn&H4>3!Ae1th=KANl0G)pgf~zb!&*T^xVyc#oakewRy&Ej&>*QxU7J zHL%E9>x*+!(c=ptVXg;?_9qEQ8R}WP#=GcB9SBy*7}X%R*D`>12XHPKkIG*`NRc#D zN>sH05{F^f4LG0MePCUL!tO_1k(ojMZCU*gguOZy0HPT;MRL2&Aj0k}1 zWtfppyZnEE9{JdO@AqL-@C@}C(fe3r9k;7G9!OUTKm&NcHUBWtU)6M@~IF7Fgl_@ zzy55?EW^U3Lz)4V%>ns%k6V&4jomu#H0F@nc~a<}Ys<#_sLAKth19H095C%r4gI;( zxF{Rvcd|cD`L_Qvh16-}&@j!m#F0hF5bU~`p?I14>Z)Q88HeCY$Yv$q>L~@MNA@IP zruBc|+m4T0kf(xTPL2J(7RK!YB&e;(#{B5>d&il^3fGP~9jNIRSR*R-rsz#BSY0rLZcIJlFgA4n zAw=A_F+anyOxyDab7K*^p&KFpbR2*Q2D^Vt#uh(x*p68-d4Mn^G~TWtIpR$9E2cHF zR?D1kn^^L^Ah2OR=t%46==2aP?OR^TZYql4c@X3+=YHWk zmU!h~3{1?djK4NnX#XX9(cfHEEX;qunqp%5TLqTC)n#F1{k8G8m%q+|>i<0j65YRK zN5cKLvm^a2BNF4kBw+eaWJtpOpED%=t?X|-_?OH`;B@qKf8YLxEJ;j%=CxvArTZUV zD>eqEf8yLd@8+(dsQB=-Vw^eT+1-+S_>J27`kQY_$^OKghhc(=o@!1=8X14785t6{ zP{dixE znAfwRW%u)YEpr@b+AlWKlE9mt{=txev9y3bj>WQdjK2sVZjz391kbIcKvZB*D>vIHFjLC|;sJJef zTt75$B%&vQY2QL$I!X*q#+N$}c%XIixX&`wWYhrAHFbzlby8pY{ql^d8D|;nnS89b z8_2!|28CWKeQ{|1dAl)d^5dnfDe%R=G@I4(V6E88jm?PdzF7A24Pk#E9)jEIWmfyE z8>q*@B&AE4z((0*(TDEIs}a(8M%+zbX5`$b-xZstoCdq|Tuy!A7gP7+=jAvyd9fFB z?_4AH^J)pe9*WxfCs7AHgBcZ?BXP^?aUd?;FCzZIQKR2h5x&_Q^Rj|U@v39|h)hZu zF_K7SfPl;^L`lUBS^0kwcUP4{rYV54ga>K$wgTKo<2}NXg&TH%pZUPC+PYwzy(7FbWafW$_e$p3{jN8qj`zmV-PB{bux0fo zDfQFU2E>y`YfSo@rIfqsG3pIN#=k`wWLFvw#lLNb>8R>2H31~6h)u zOM|vN;bOLEJonX)P5S%&9NEBJ7Q68%WJ#71ZTNviI(?{+5#l(x&j|^5nCFfZ_m(mP z#DSFmi7G?C$n~6GstlL$24xysv&n)&n>51JCkRjO?QMVPu{?y;H4;6L=QsZiw@O$? zs~vkz2R!Y7{!5pm{6+3GJXxCX3+@{Q_(;08yj9Z=fG%H5RGqKq_O*!(0ZmgR+Rnr+ zjV8_?j(5uq4-ZS;yjT`M&L5xLFvLKIVJ|}M2odUoBn~H2t;=}7O$+V zP{;5*S2szoN;P@@e9&5zz1e?V@mg(qRKm1={kE%xt~zAUcvj^53gDOyb|j`DiAv&}|6o%u600Bs#Wv7IjBg~aYIe?Z zHaWb8VQWpYjsXM9*n$Fm_wu z7{C;ma?l6hS|KVuh!7{+M6jcocsuUZubZRKD};XUSvesOYZP1}h)BEyi6n|3({tHW zedN6goRLBCE1+zV3kX2xXnE*4sP$X6W{EvkOMqN8z)En0nOcp}H!NzcvLA6KS_4XA z-v05dDGik(T@oG0A;%rY^dfpT5^!FgH7H4&9Zf_9k8I!=?4q zSM$w{`w_|c2_5VM>cOHx>}+J)W1q*{$y{yG<7k)V=2-MCSSK-WJ&epooJ@kzL8czV zI`64V4N{_c-P_NOT_dv|zwSS!j~+Yp#k#S3y$EBUV-vy4t9HE;M-@;6%ejBO+F$T1 znpK&;Cxhb=r|+H(x5cboL+O}ttg%=dWNKRbcUx}^L>L)6Jz#B_m`WL!?b){X!p9X> zXjO5`sSd;yjsY;8E>tONYTvd~Ywo;vzhq&(?qHgm6}_98cI$a|t8{B3yqZFeQ?vDd zY1s5BXW0%i2x{ZB5*0r>A3c8|nui~DWb!o>PcEx6;@(DYYKu6?A|)VqYAleDCE zLS%LA1MU9C(j$jti2QY%{44vrEUZ?x1PpP22Ag906K_{Ij**v|^6;eS{*2sCfIW^y zfYZC&uj#y^+F^@JWmvD>1E6rMhnx zcr7}Ij?{M1-VJZCGZSU^6M-?4cS-F4%6i16vb z`#PoA8`zKpvG-giS{Kl5s*^?#{Q3y#;#tOh%^LCz4Khib4K#mEF8T;g*IHo^h)EkN zjva2d`=peeIGB|kEf|Au{EOaxoOD1Nt1xdLOrcUl%MY4D><5m%m5hQ$z8M5yiURLu zLfp=~#um!jcZ%oW>xu5gl@M)M)=-VI!4VU~!KrrUPH9q>E8FpPlo_rM&@&iBtIsRo z9(oiQBG%H?Z*_mMkYjV)&-KpkcI#f?!3bZKb2y#3#8qCf*uP(qm;zbX0hn<)bB-~7T$;n`yU!?kQpKrPLA2?E;8@Z@mI+SG%38r@L~`~V_2#-o&8m=K*KlxNyM2yyJSTH6on%AtchQgLOn_!daC_-n3cW7rtjSeMj z1`TsZi3aPRQW=KSaTMd!sJBLv#w|I2+(e&jZ2Ny{h@=Sj<$nON$=NHh<`Dp(sF;p^HLZP^FHN+L*$t2o%^eX`_TxQK1bDX8Be~pb;d0g~@5v zB1p-Ho~Quj(b{s_GFolt*(3mr8x zc_=0%*|gyLfQCh@i@fKX)u`N(=rLhdf;1_KLY}*H1Y@kE!FAaX7J7f8t6N=w3m6Be z=oOeVVlo{0Jon>C`IM~gpe#-F%T_|?XBK~&HhrKr+m!mVVNze7P-nzDjD^?vR?i|c zP7LxK3Gr?B4GvD)u=m@C`A6nO4sk6NT`#{Sfn}Nyy{v8>kA|%nX|6o6cWw-)46jg? z$~-0&4%6{9Ze=`94&Mnr5aS0HyTkK6|F7zH569_;&XLI}VXv z5)(1Bs!G;qC!39@S8=@G9`d-lp7hJP-w)EIm9g}{*(Xz`iU%?p(a9=^A_ktK$L9