diff --git a/core/continuum-core/src/capacity/cold_twin.rs b/core/continuum-core/src/capacity/cold_twin.rs new file mode 100644 index 0000000000..6a52759e8a --- /dev/null +++ b/core/continuum-core/src/capacity/cold_twin.rs @@ -0,0 +1,184 @@ +//! Verified cold-twin detection — the "safe to drop from NVMe" primitive for the +//! storage serving-tier governor (`docs/architecture/STORAGE-SERVING-TIER-GOVERNOR.md`). +//! +//! When the NVMe hot-serving tier is over budget, the governor's `evict_at_least` +//! migrates the coldest FROZEN/DUPLICATE artifact to the Cold drive. A frozen GGUF +//! whose identical twin ALREADY exists on cold storage is a pure duplicate — dropping +//! its NVMe copy loses no data. But "identical" must be VERIFIED, never assumed from a +//! path: dropping a 662 GB model on a bad guess is the failure this guards +//! ([[no-masking-fallbacks-my-style-tell]]). The verdict here is the gate an eviction +//! pool consults before it drops anything. +//! +//! Verification tiers (cheap → strong), the caller picks the floor: +//! * **structural** — same shard count, same names, same per-shard sizes. Catches a +//! partial/truncated copy or a different quant. The default floor (what a human +//! `dir` comparison does, but mechanical). +//! * **content** (caller's job, not here) — a header-magic check or a hash. This +//! module returns the candidate; the caller escalates if it wants stronger proof. +//! +//! Pure over its inputs (the shard lists); the fs scan that produces them is a thin +//! separate step so the match logic is unit-testable without touching disk. + +use std::path::{Path, PathBuf}; + +/// One shard's identity for twin comparison: file name + byte length. Deliberately +/// NOT the full path — a twin lives under a different root by definition. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardStat { + pub name: String, + pub size: u64, +} + +/// The shard set of one artifact (a multi-file GGUF), sorted by name for order-stable +/// comparison. Produced by [`scan_shards`]; compared by [`is_structural_twin`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArtifactShards { + pub dir: PathBuf, + pub shards: Vec, +} + +impl ArtifactShards { + /// Total bytes across all shards — the reclaim that dropping this copy frees. + pub fn total_bytes(&self) -> u64 { + self.shards.iter().map(|s| s.size).sum() + } +} + +/// STRUCTURAL twin test (the default safe-to-drop floor): the two artifacts have the +/// same shard count, and each shard matches its counterpart by NAME and SIZE. Order is +/// normalized (both sorted by name). An empty artifact never twins anything (dropping +/// on "both empty" is a bug, not a duplicate). Pure — no I/O. +pub fn is_structural_twin(nvme: &ArtifactShards, cold: &ArtifactShards) -> bool { + if nvme.shards.is_empty() || nvme.shards.len() != cold.shards.len() { + return false; + } + nvme.shards + .iter() + .zip(cold.shards.iter()) + .all(|(a, b)| a.name == b.name && a.size == b.size && a.size > 0) +} + +/// Scan `dir` for shards with `ext` (e.g. `"gguf"`), returning name+size sorted by +/// name. Missing dir / no matches → empty shard list (never twins). Thin I/O layer +/// over the pure [`is_structural_twin`]. +pub fn scan_shards(dir: &Path, ext: &str) -> ArtifactShards { + let mut shards: Vec = std::fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| { + let path = e.path(); + if path.extension().and_then(|s| s.to_str()) != Some(ext) { + return None; + } + let name = path.file_name()?.to_str()?.to_string(); + let size = e.metadata().ok()?.len(); + Some(ShardStat { name, size }) + }) + .collect(); + shards.sort_by(|a, b| a.name.cmp(&b.name)); + ArtifactShards { + dir: dir.to_path_buf(), + shards, + } +} + +/// Find a VERIFIED structural twin of `nvme_artifact` under any of `cold_roots` (the +/// Cold-drive artifact roots). Returns the cold twin's dir when found — the signal that +/// the NVMe copy is a safe-to-drop duplicate. `None` = no verified twin → the governor +/// must NOT drop the NVMe copy (migrate the bytes instead, or keep it). +/// +/// A cold root is scanned one level deep for a subdir whose shards structurally twin the +/// NVMe artifact (the model's own dir name may differ across drives, so match on shard +/// identity, not the dir name). +pub fn find_cold_twin( + nvme_artifact: &ArtifactShards, + cold_roots: &[PathBuf], + ext: &str, +) -> Option { + if nvme_artifact.shards.is_empty() { + return None; + } + for root in cold_roots { + // the root itself might hold the shards… + let here = scan_shards(root, ext); + if is_structural_twin(nvme_artifact, &here) { + return Some(root.clone()); + } + // …or one of its immediate subdirs (Steam-library-style per-model dirs). + if let Ok(entries) = std::fs::read_dir(root) { + for sub in entries.flatten() { + let p = sub.path(); + if p.is_dir() { + let cand = scan_shards(&p, ext); + if is_structural_twin(nvme_artifact, &cand) { + return Some(p); + } + } + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn art(shards: &[(&str, u64)]) -> ArtifactShards { + ArtifactShards { + dir: PathBuf::from("x"), + shards: shards + .iter() + .map(|(n, s)| ShardStat { + name: n.to_string(), + size: *s, + }) + .collect(), + } + } + + // what this catches: identical shard sets (name+size) verify as twins — the K3 + // case (16 shards, matching sizes) → safe to reclaim the NVMe copy. + #[test] + fn identical_shards_are_a_twin() { + let a = art(&[("k3-00001.gguf", 41 * 1024), ("k3-00002.gguf", 45 * 1024)]); + let b = art(&[("k3-00001.gguf", 41 * 1024), ("k3-00002.gguf", 45 * 1024)]); + assert!(is_structural_twin(&a, &b)); + } + + // what this catches: a TRUNCATED/partial cold copy (one shard short) is NOT a twin + // — never drop the NVMe copy against an incomplete backup. + #[test] + fn missing_shard_is_not_a_twin() { + let a = art(&[("k3-00001.gguf", 41 * 1024), ("k3-00002.gguf", 45 * 1024)]); + let b = art(&[("k3-00001.gguf", 41 * 1024)]); + assert!(!is_structural_twin(&a, &b)); + } + + // what this catches: same names but a DIFFERENT size (different quant, or a + // corrupt/partial shard) is NOT a twin — size is load-bearing. + #[test] + fn size_mismatch_is_not_a_twin() { + let a = art(&[("k3-00001.gguf", 41 * 1024)]); + let b = art(&[("k3-00001.gguf", 20 * 1024)]); + assert!(!is_structural_twin(&a, &b)); + } + + // what this catches: an empty artifact never twins anything — dropping on + // "both empty" would be a bug, not a verified duplicate. + #[test] + fn empty_never_twins() { + assert!(!is_structural_twin(&art(&[]), &art(&[]))); + assert!(!is_structural_twin(&art(&[("a.gguf", 10)]), &art(&[]))); + } + + // what this catches: a zero-byte shard (the interrupted-write case, like the + // container's L11=0) never counts as a matching shard. + #[test] + fn zero_byte_shard_is_not_a_match() { + let a = art(&[("a.gguf", 0)]); + let b = art(&[("a.gguf", 0)]); + assert!(!is_structural_twin(&a, &b)); + } +} diff --git a/core/continuum-core/src/capacity/device_fit.rs b/core/continuum-core/src/capacity/device_fit.rs new file mode 100644 index 0000000000..d9f62d1b57 --- /dev/null +++ b/core/continuum-core/src/capacity/device_fit.rs @@ -0,0 +1,400 @@ +//! Device-fit VRAM planning — the governor's answer to "how does THIS model's +//! resident (non-expert) tier meet THIS device's VRAM budget, and how much VRAM +//! is left to hold hot experts on-device?" +//! +//! A streaming MoE serve splits into two tiers with different homes: +//! * **experts** (`*_exps`) — paged; the [`ServingExpertPager`] plans which +//! layers sit HOT in VRAM and streams the cold complement from NVMe. +//! * **resident** (attention / embeddings / output / dense / norms / +//! shared-experts) — must live in VRAM for fast compute, offloaded WHOLE. +//! A fused attention op (K3's Gated Delta Net) CANNOT span CPU/GPU, so the +//! resident tier is all-or-nothing on the device: a partial `-ngl` corrupts +//! the graph (`buffer->buft` assertion). This file decides how the whole +//! resident tier fits, and — crucially — RECONCILES it with the expert tier. +//! +//! The reconciliation is the point. Resident weights, the KV cache, the compute +//! graph, AND the hot experts all sit on the SAME device budget. Before this, +//! the expert pager was handed the full VRAM ceiling while resident silently ate +//! most of it — the two tiers double-counted the card. Here the budget is +//! partitioned ONCE: compute reserve, then resident, then a SUFFICIENT context's +//! KV, and **everything left over is the hot-expert VRAM budget**. Maximizing +//! that leftover is "as much GPU as possible": the more experts resident, the +//! fewer NVMe fetches per token — the structural win over WASTE (which keeps the +//! trunk in CPU RAM and streams every expert). [[k3-beats-waste-decisively]], +//! [[pager-control-law-is-fractal-to-grid]], task #34. +//! +//! Three resident outcomes, no guessing ([[no-masking-fallbacks-my-style-tell]]): +//! * [`ResidentFit::Native`] — resident fits as-shipped → offload all, no override. +//! * [`ResidentFit::Override`] — resident overflows, but a device-fit artifact +//! supplies a precision-shrunk resident that fits → offload all, load resident +//! from the override (`LLAMA_RESIDENT_OVERRIDE`), experts stream from the +//! primary. The misfit-design move: fit the model to the owned device. +//! * [`ResidentFit::Unfittable`] — resident overflows and no fitting override +//! resolves → this device cannot GPU-serve this model. Route to CPU/grid, +//! LOUD; never silently OOM a doomed launch. +//! +//! Context is DERIVED, never hand-picked (#31): a desired window clamped to what +//! the leftover-after-resident affords and to the model's own max. Pure over its +//! inputs; the artifact resolver is INJECTED (this module never knows a path, a +//! url, or a cache layout). + +use std::path::PathBuf; + +const GB: u64 = 1024 * 1024 * 1024; + +/// A resolved device-fit resident-override artifact: where its first shard lives +/// and how many bytes its (precision-shrunk) resident tier occupies on-device. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResidentOverride { + /// First shard of the device-fit GGUF whose resident tensors the loader + /// sources (`LLAMA_RESIDENT_OVERRIDE`); its experts are ignored (the primary + /// streams those). May sit on cold storage — only its resident bytes are + /// mapped, once, at load (the loader hook's lazy, prefetch-0 mmap). + pub path: PathBuf, + /// The resident (non-expert) byte total this override loads onto the device. + pub resident_bytes: u64, +} + +/// How the model's resident tier meets the VRAM budget. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResidentFit { + /// Resident fits as-shipped — offload all resident to GPU, no override file. + Native, + /// Resident overflows; this override supplies a shrunk resident that fits. + Override(ResidentOverride), + /// Resident overflows and no fitting override resolves. Route to CPU/grid. + Unfittable { + /// The as-shipped resident tier that did not fit. + resident_bytes: u64, + /// The budget it had to fit under (VRAM − compute reserve). + usable_bytes: u64, + }, +} + +impl ResidentFit { + /// The resident bytes that actually load onto the device under this fit + /// (as-shipped for `Native`, shrunk for `Override`). `None` when unfittable. + pub fn device_resident_bytes(&self, native_bytes: u64) -> Option { + match self { + Self::Native => Some(native_bytes), + Self::Override(o) => Some(o.resident_bytes), + Self::Unfittable { .. } => None, + } + } + + /// The override GGUF the launcher exports as `LLAMA_RESIDENT_OVERRIDE`, if any. + pub fn override_path(&self) -> Option<&PathBuf> { + match self { + Self::Override(o) => Some(&o.path), + _ => None, + } + } +} + +/// The device + serve facts a device-fit decision needs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeviceFitInputs { + /// As-shipped resident (non-expert) weight bytes (`weights_bytes − expert_bytes_total`). + pub resident_bytes: u64, + /// Governed VRAM budget for this serve (`SystemProfile::serving_budget_bytes`). + pub vram_budget_bytes: u64, + /// KV cache bytes per ONE token per lane (from the model's attention geometry). + pub kv_bytes_per_token: u64, + /// Fixed compute-graph + backend-context VRAM beyond weights and KV. + pub compute_reserve_bytes: u64, + /// The context window we'd LIKE to serve (the planner's target). Clamped down + /// to what leftover-after-resident affords and to `model_max_context`. + pub desired_context: u32, + /// The model's own maximum context (`{arch}.context_length`) — the ceiling. + pub model_max_context: u32, + /// Concurrent decode lanes (`--parallel`); KV is per-lane, experts are shared. + pub lanes: u32, +} + +/// The complete governed VRAM serving plan: resident source, derived context, and +/// the hot-expert VRAM budget the leftover affords. The launcher turns this into +/// `LLAMA_RESIDENT_OVERRIDE` + `-c`; the expert pager consumes `expert_vram_budget_bytes`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeviceFitPlan { + /// How the resident tier fits (native / override / unfittable). + pub resident: ResidentFit, + /// The per-lane context window the plan serves (derived, clamped). Zero iff + /// the resident tier is `Unfittable`. + pub context_window: u32, + /// VRAM left for HOT experts after resident + KV + compute reserve — the + /// budget handed to the [`ServingExpertPager`]. This is the number to MAXIMIZE + /// (more on-GPU experts = fewer NVMe streams = beats WASTE). Zero iff unfittable. + pub expert_vram_budget_bytes: u64, +} + +impl DeviceFitPlan { + /// True when this model can be GPU-served on this device (native or via override). + pub fn is_gpu_servable(&self) -> bool { + !matches!(self.resident, ResidentFit::Unfittable { .. }) + } +} + +/// Runtime reserve when the caller has no measured compute figure: mirrors +/// fit-device.cpp's default (`max(2 GiB, 10% of budget)`) so the Rust plan and the +/// C++ artifact planner hold back the same headroom. +pub fn default_compute_reserve_bytes(vram_budget_bytes: u64) -> u64 { + (vram_budget_bytes / 10).max(2 * GB) +} + +/// Largest per-lane context (in tokens) whose KV fits `kv_budget_bytes`, clamped +/// to the model max. Zero when KV or lanes are zero. +fn context_fitting( + kv_budget_bytes: u64, + kv_bytes_per_token: u64, + lanes: u32, + model_max_context: u32, +) -> u32 { + if kv_bytes_per_token == 0 || lanes == 0 { + return 0; + } + let per_lane = kv_budget_bytes / lanes as u64; + let tokens = per_lane / kv_bytes_per_token; + u32::try_from(tokens).unwrap_or(u32::MAX).min(model_max_context) +} + +/// Plan how a model's resident tier fits the device, the context the leftover +/// affords, and the hot-expert VRAM budget. `resolve_override` is the adapter: +/// given the usable byte budget (VRAM − compute reserve) it returns an override +/// whose resident is PROMISED to fit — the plan re-verifies and rejects an +/// oversized one. +/// +/// Budget partition (in order): compute reserve → resident → KV for a sufficient +/// context → **everything left = hot-expert VRAM budget** (maximized). +pub fn plan_device_fit( + inputs: &DeviceFitInputs, + resolve_override: impl FnOnce(u64) -> Option, +) -> DeviceFitPlan { + let usable = inputs + .vram_budget_bytes + .saturating_sub(inputs.compute_reserve_bytes); + + let resident = if inputs.resident_bytes <= usable { + ResidentFit::Native + } else { + match resolve_override(usable) { + // Trust-but-verify: a shrunk resident that STILL overflows is a broken + // artifact — refuse it, don't OOM the launch on a false promise. + Some(ov) if ov.resident_bytes <= usable => ResidentFit::Override(ov), + _ => ResidentFit::Unfittable { + resident_bytes: inputs.resident_bytes, + usable_bytes: usable, + }, + } + }; + + let Some(on_device_resident) = resident.device_resident_bytes(inputs.resident_bytes) else { + return DeviceFitPlan { + resident, + context_window: 0, + expert_vram_budget_bytes: 0, + }; + }; + + // VRAM after the resident weights: shared between the KV cache and hot experts. + let after_resident = usable.saturating_sub(on_device_resident); + + // Reserve KV for the DESIRED context, but never more than after_resident can + // hold — a tight resident (K3 ~30 GB on 32 GB) squeezes the window rather than + // overflowing. The context is the floor we give up first so experts keep VRAM. + let desired = inputs.desired_context.min(inputs.model_max_context); + let desired_kv = inputs + .kv_bytes_per_token + .saturating_mul(desired as u64) + .saturating_mul(inputs.lanes.max(1) as u64); + + let (context_window, kv_bytes) = if desired_kv <= after_resident { + (desired, desired_kv) + } else { + // Squeeze: the largest window whose KV fits what's left after resident. + let ctx = context_fitting( + after_resident, + inputs.kv_bytes_per_token, + inputs.lanes.max(1), + desired, + ); + let kv = inputs + .kv_bytes_per_token + .saturating_mul(ctx as u64) + .saturating_mul(inputs.lanes.max(1) as u64); + (ctx, kv) + }; + + // Everything the resident + KV didn't claim is the hot-expert budget. This is + // the lever we maximize: more on-GPU experts, fewer streams, beat WASTE. + let expert_vram_budget_bytes = after_resident.saturating_sub(kv_bytes); + + DeviceFitPlan { + resident, + context_window, + expert_vram_budget_bytes, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn override_of(bytes: u64) -> impl FnOnce(u64) -> Option { + move |_usable| { + Some(ResidentOverride { + path: PathBuf::from("D:/k3-fit/Kimi-K3-fit-00001-of-00016.gguf"), + resident_bytes: bytes, + }) + } + } + fn no_override() -> impl FnOnce(u64) -> Option { + |_usable| None + } + + // what this catches: a resident tier that fits as-shipped serves NATIVE (no + // override artifact) and hot experts get the leftover VRAM — the small-MoE path. + #[test] + fn resident_under_budget_is_native_and_experts_get_leftover() { + let plan = plan_device_fit( + &DeviceFitInputs { + resident_bytes: 20 * GB, + vram_budget_bytes: 32 * GB, + kv_bytes_per_token: 128 * 1024, + compute_reserve_bytes: 2 * GB, // usable 30 + desired_context: 8192, + model_max_context: 262_144, + lanes: 1, + }, + no_override(), + ); + assert_eq!(plan.resident, ResidentFit::Native); + assert!(plan.is_gpu_servable()); + assert_eq!(plan.context_window, 8192); + // usable 30 − resident 20 − KV(8192×128KiB=1GiB) = ~9 GiB for hot experts. + assert!(plan.expert_vram_budget_bytes > 8 * GB); + } + + // what this catches: THE K3 case — 58 GB resident on 32 GB overflows, the 29 GB + // device-fit override is selected (not a partial -ngl, not an OOM), and the + // leftover funds hot experts. + #[test] + fn overflowing_resident_takes_override_and_frees_expert_vram() { + let plan = plan_device_fit( + &DeviceFitInputs { + resident_bytes: 58 * GB, + vram_budget_bytes: 32 * GB, + kv_bytes_per_token: 64 * 1024, + compute_reserve_bytes: 2 * GB, // usable 30 + desired_context: 4096, + model_max_context: 262_144, + lanes: 1, + }, + override_of(24 * GB), // aggressive shrink → room for experts + ); + match &plan.resident { + ResidentFit::Override(o) => assert_eq!(o.resident_bytes, 24 * GB), + other => panic!("expected Override, got {other:?}"), + } + assert!(plan.resident.override_path().is_some()); + // usable 30 − resident 24 − small KV → ~6 GiB hot experts (the WASTE-beating margin). + assert!(plan.expert_vram_budget_bytes > 5 * GB); + assert_eq!(plan.context_window, 4096); + } + + // what this catches: a shrunk override that STILL overflows is rejected as a + // broken artifact rather than OOMing the launch on a false promise. + #[test] + fn oversized_override_is_rejected_as_unfittable() { + let plan = plan_device_fit( + &DeviceFitInputs { + resident_bytes: 58 * GB, + vram_budget_bytes: 32 * GB, + kv_bytes_per_token: 64 * 1024, + compute_reserve_bytes: 2 * GB, + desired_context: 4096, + model_max_context: 262_144, + lanes: 1, + }, + override_of(40 * GB), + ); + assert!(matches!(plan.resident, ResidentFit::Unfittable { .. })); + assert!(!plan.is_gpu_servable()); + assert_eq!(plan.context_window, 0); + assert_eq!(plan.expert_vram_budget_bytes, 0); + } + + // what this catches: overflow with no override → Unfittable, so the caller + // routes to CPU/grid instead of launching a doomed GPU serve. + #[test] + fn overflow_without_override_is_unfittable() { + let plan = plan_device_fit( + &DeviceFitInputs { + resident_bytes: 58 * GB, + vram_budget_bytes: 32 * GB, + kv_bytes_per_token: 64 * 1024, + compute_reserve_bytes: 2 * GB, + desired_context: 4096, + model_max_context: 262_144, + lanes: 1, + }, + no_override(), + ); + assert!(matches!( + plan.resident, + ResidentFit::Unfittable { resident_bytes, usable_bytes } + if resident_bytes == 58 * GB && usable_bytes == 30 * GB + )); + } + + // what this catches: when resident nearly fills the card, the context is + // SQUEEZED to fit the sliver left (never overflow), and experts get ~nothing — + // the honest tight-fit signal that says "shrink resident more to beat WASTE." + #[test] + fn tight_resident_squeezes_context_not_overflow() { + let plan = plan_device_fit( + &DeviceFitInputs { + resident_bytes: 58 * GB, + vram_budget_bytes: 32 * GB, + kv_bytes_per_token: 64 * 1024, + compute_reserve_bytes: 2 * GB, // usable 30 + desired_context: 32_768, // wants a big window… + model_max_context: 262_144, + lanes: 1, + }, + override_of(29 * GB), // …but resident eats 29 of 30, ~1 GiB left + ); + assert!(matches!(plan.resident, ResidentFit::Override(_))); + // 1 GiB / 64 KiB ≈ 16k tokens max, below the 32k desired → squeezed, not overflowed. + assert!(plan.context_window > 0 && plan.context_window < 32_768); + // KV ate the sliver → almost nothing for experts (the tight-fit truth). + assert!(plan.expert_vram_budget_bytes < 1 * GB); + } + + // what this catches: more lanes → smaller per-lane window from the same VRAM. + #[test] + fn more_lanes_shrink_the_per_lane_window() { + let mk = |lanes| { + plan_device_fit( + &DeviceFitInputs { + resident_bytes: 20 * GB, + vram_budget_bytes: 32 * GB, + kv_bytes_per_token: 256 * 1024, + compute_reserve_bytes: 2 * GB, + desired_context: 262_144, // want max so VRAM is the binding limit + model_max_context: 262_144, + lanes, + }, + no_override(), + ) + .context_window + }; + assert!(mk(4) < mk(1), "more lanes must shrink the per-lane window"); + } + + // what this catches: the default reserve tracks fit-device.cpp (max(2 GiB, 10%)). + #[test] + fn default_reserve_matches_fit_device_cpp() { + assert_eq!(default_compute_reserve_bytes(32 * GB), 3 * GB + GB / 5); // 10% of 32 = 3.2 + assert_eq!(default_compute_reserve_bytes(8 * GB), 2 * GB); // 10% of 8 = 0.8 → floored to 2 + } +} diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index b4120f7a72..1f3ef47540 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -23,7 +23,9 @@ //! IS a real regression test. pub mod bandit_plan_controller; +pub mod cold_twin; pub mod consumer; +pub mod device_fit; pub mod expert_container; pub mod expert_decay_policy; pub mod expert_ecache; diff --git a/core/continuum-core/src/capacity/trace_tail.rs b/core/continuum-core/src/capacity/trace_tail.rs index 69af58ad9c..535424c0b4 100644 --- a/core/continuum-core/src/capacity/trace_tail.rs +++ b/core/continuum-core/src/capacity/trace_tail.rs @@ -483,4 +483,34 @@ mod tests { assert_eq!(pin_ceiling(4000 * mb, 0, 10, 100), 0); assert_eq!(pin_ceiling(4000 * mb, 10_000 * mb, 0, 0), 0); } + + // MEASUREMENT (go/no-go for the LiveUploadPager predictive pipeline, #23/#34): the REAL K3 + // routed-access trace's schedulable coverage decides whether predictive prefetch can hide the + // per-op H2D — H2D/token = (1 - coverage) × ~11GB. Prints, never asserts a value (the fixture is + // a real routing sample, not a synthetic invariant); run with --nocapture to read it. + #[test] + fn k3_fixture_measured_schedulable_coverage() { + let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../docs/architecture/prototypes/expert-pager/fixtures/k3-routed-access.trace"); + if !fixture.is_file() { + eprintln!("[K3-COVERAGE] fixture absent ({}), skipping", fixture.display()); + return; + } + let mut tail = MoeTraceTail::new(92); // K3: 92 MoE layers + let mut tokens = 0u64; + for _ in 0..100_000 { + let n = tail.drain(&fixture); + tokens += n; + if n == 0 { + break; + } + } + eprintln!( + "[K3-COVERAGE] tokens={} repeat_recall={:?} predicted_delta={:?} schedulable_coverage={:?} (x100)", + tokens, + tail.repeat_recall_x100(), + tail.predicted_delta_recall_x100(), + tail.schedulable_coverage_x100() + ); + } } diff --git a/core/continuum-core/src/cognition/eval.rs b/core/continuum-core/src/cognition/eval.rs index 05192525f4..8ec7856821 100644 --- a/core/continuum-core/src/cognition/eval.rs +++ b/core/continuum-core/src/cognition/eval.rs @@ -668,6 +668,7 @@ async fn spawn_gene_eval_lane( }], placement: placement_evidence.placement, expert_placement: None, // eval lanes run the whole model; no K3 expert paging + resident_override: None, // eval lanes serve resident as-shipped; no device-fit override }; emit_eval_phase("loading_lane", &format!("cold-loading gene eval lane ({})", gene.name)); let lane = EphemeralServingLane::spawn(&target, EVAL_LANE_BASE_PORT) @@ -788,6 +789,7 @@ async fn build_base_eval_lane_inner(base_id: &str) -> Result…", not a frozen bar. diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index ff9b37ce86..2bd09a015c 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -212,6 +212,15 @@ pub struct ServingTarget { /// `relaunch_needed`), not by the probe-based reconcile here, since there is no API to /// read a running server's `-ot`. Per-expert paging (no relaunch) is slice-2. pub expert_placement: Option, + /// Device-fit resident-override: when the model's RESIDENT (non-expert) tier + /// overflows the governed VRAM budget as-shipped, the governor's + /// [`device_fit`](crate::capacity::device_fit) plan resolves a precision-shrunk + /// resident-override GGUF that fits. The launcher exports it as + /// `LLAMA_RESIDENT_OVERRIDE` so llama.cpp sources the resident tensors from it + /// (all offloaded to GPU) while the primary GGUF streams experts. `None` = + /// resident fits as-shipped (Native), served with no override — the default and + /// the only shape for a dense or small-MoE model. [[device-fit-repeatable-primitive]] / #29. + pub resident_override: Option, } /// Where a serving lane's model weights are resident — see [`ServingTarget::placement`]. @@ -1686,6 +1695,16 @@ impl LlamaServerControl for LlamaServerProcess { mmproj_local_path) or drop the Vision capability so the row stops claiming sight." ); } + // Device-fit resident-override (#29): source the RESIDENT (non-expert) + // tensors from the precision-shrunk fit GGUF so the whole resident tier fits + // VRAM offloaded to GPU, while this primary GGUF streams the experts. The + // loader hook (`LLAMA_RESIDENT_OVERRIDE`) lazy-maps only the override's + // resident bytes (its experts are ignored). Set by the governor's device_fit + // plan when as-shipped resident overflows the VRAM budget; absent = resident + // fits as-shipped (no override, no env). [[device-fit-repeatable-primitive]]. + if let Some(ov) = &target.resident_override { + cmd.env("LLAMA_RESIDENT_OVERRIDE", ov); + } // `--embeddings` is deliberately NOT set on this GENERATION lane. On the // current llama.cpp build it puts the server in embedding (non-causal) // mode, which makes generation fail with `500 "Compute error."` on EVERY @@ -2157,6 +2176,7 @@ mod tests { adapters: Vec::new(), placement: LanePlacement::Gpu, expert_placement: None, + resident_override: None, } } diff --git a/core/continuum-core/src/inference/vision_sidecar.rs b/core/continuum-core/src/inference/vision_sidecar.rs index d98272984b..7448794ef3 100644 --- a/core/continuum-core/src/inference/vision_sidecar.rs +++ b/core/continuum-core/src/inference/vision_sidecar.rs @@ -194,6 +194,7 @@ pub async fn ensure_sidecar( adapters: Vec::new(), placement: LanePlacement::Cpu, expert_placement: None, + resident_override: None, // vision sidecar serves as-shipped; no device-fit override }; let lane = EphemeralServingLane::spawn(&target, VISION_SIDECAR_BASE_PORT) .await diff --git a/core/continuum-core/src/model_registry/artifacts.rs b/core/continuum-core/src/model_registry/artifacts.rs index ef718a738e..673d5bcb7d 100644 --- a/core/continuum-core/src/model_registry/artifacts.rs +++ b/core/continuum-core/src/model_registry/artifacts.rs @@ -180,6 +180,53 @@ fn resolve_from_local_model_roots(model_id: &str) -> Option { None } +/// Resolve a CACHED device-fit resident-override for `(model_id, usable_bytes)`. +/// The device-fit foundry (`tools/moe-fit`) writes a precision-shrunk RESIDENT +/// (non-expert) GGUF into a per-model cache dir plus a `resident-bytes` sidecar; +/// this looks it up and returns it ONLY when its resident tier fits the caller's +/// usable VRAM. Generation / HF discovery is #35 — absent a cached artifact this +/// returns `None` and the caller falls to `Unfittable` (loud), never a hardcoded +/// path. [[device-fit-repeatable-primitive]] +pub fn resolve_device_fit_override( + model_id: &str, + usable_bytes: u64, +) -> Option { + let dir = device_fit_cache_dir(model_id); + if !dir.is_dir() { + return None; + } + let first_shard = first_gguf_in_dir(&dir)?; + // The foundry records the resident byte total the shrink produced, so the plan + // verifies fit without loading the GGUF. Missing sidecar = not our foundry's + // artifact → refuse rather than guess its size ([[no-masking-fallbacks-my-style-tell]]). + let resident_bytes: u64 = fs::read_to_string(dir.join("resident-bytes")) + .ok()? + .trim() + .parse() + .ok()?; + (resident_bytes <= usable_bytes).then_some(crate::capacity::device_fit::ResidentOverride { + path: first_shard, + resident_bytes, + }) +} + +/// Per-user cache dir a device-fit override for `model_id` lives in: +/// `/device-fit//`. A convention (mirrors +/// [`local_model_roots`]), never a hardcoded operator path. +fn device_fit_cache_dir(model_id: &str) -> PathBuf { + let slug: String = model_id + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '-' + } + }) + .collect(); + storage_root().join("device-fit").join(slug) +} + fn local_model_roots() -> Vec { let mut roots = Vec::new(); if let Some(home) = home_dir_string() { diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index dcfbe39e72..6b58b06855 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -720,6 +720,76 @@ impl ServingDaemonModule { ctx.committed_placement.clone() } + /// Device-fit resident-override (#29): does this model's RESIDENT (non-expert) + /// tier fit the governed VRAM budget as-shipped, or must the launcher source a + /// precision-shrunk resident from a device-fit override GGUF? The resident-FIT + /// decision turns ONLY on `resident_bytes` vs the budget minus a fixed compute + /// reserve — per-layer KV (M5's #2107 `ModelCapabilities`) drives the CONTEXT + + /// expert-VRAM split, NOT this, so KV is deliberately not consulted here. `None` + /// = resident fits as-shipped (dense / small MoE, served normally) OR no fitting + /// override is cached yet (a >VRAM-resident MoE like K3 until the resolver #35 + /// discovers/generates one — then the launch OOMs LOUD rather than silently + /// mis-serving, [[no-masking-fallbacks-my-style-tell]]). + fn compute_resident_override(&self, model: &Model) -> Option { + let budget = governed_vram_ceiling(&self.resource_daemon)?; + if budget == 0 { + return None; + } + // Expert-tensor total — present only for a MoE serve (a dense model's + // resident always places on GPU normally, no device-fit split). Read under + // the same never-across-await sync Mutex as the placement path. + let expert_bytes_total = { + let guard = self.moe_serving.lock().ok()?; + match guard.as_ref() { + Some((id, ctx)) if id.as_str() == model.id.as_str() => ctx.expert_bytes_total, + _ => return None, + } + }; + let fp = footprint_for(model)?; + let resident_bytes = fp.weights_bytes.saturating_sub(expert_bytes_total); + let model_id = model.id.clone(); + let inputs = crate::capacity::device_fit::DeviceFitInputs { + resident_bytes, + vram_budget_bytes: budget, + kv_bytes_per_token: 0, // not consulted for the resident-fit decision — see doc + compute_reserve_bytes: + crate::capacity::device_fit::default_compute_reserve_bytes(budget), + desired_context: model.context_window, + model_max_context: model.context_window, + lanes: 1, + }; + let plan = crate::capacity::device_fit::plan_device_fit(&inputs, |usable| { + crate::model_registry::artifacts::resolve_device_fit_override(&model_id, usable) + }); + crate::probe!( + class = "serving.device_fit", + model = model.id.as_str(), + resident_bytes, + budget_bytes = budget, + gpu_servable = plan.is_gpu_servable(), + has_override = plan.resident.override_path().is_some(), + "device-fit resident tier: {}", + match &plan.resident { + crate::capacity::device_fit::ResidentFit::Native => + "fits as-shipped (native, all resident on GPU)".to_string(), + crate::capacity::device_fit::ResidentFit::Override(o) => format!( + "device-fit override {} GiB resident from {}", + o.resident_bytes / (1024 * 1024 * 1024), + o.path.display() + ), + crate::capacity::device_fit::ResidentFit::Unfittable { + resident_bytes, + usable_bytes, + } => format!( + "UNFITTABLE: resident {} GiB > {} GiB usable — route to grid or generate an override (#35)", + resident_bytes / (1024 * 1024 * 1024), + usable_bytes / (1024 * 1024 * 1024) + ), + }, + ); + plan.resident.override_path().cloned() + } + /// #287 slice 2 — the governed host-cache lease, derived LIVE on the tick and /// published to the per-port plan file her `ResidencyCache` mtime-polls. This is /// the loop-closer that retires the `GGML_MOE_HOST_CACHE_GB=40` scratchpad @@ -1098,6 +1168,12 @@ impl ServingDaemonModule { // for a dense model (or before the pager has committed a placement) — served exactly // as before, no override. let expert_placement = self.compute_expert_placement(&model); + // Device-fit resident-override (#29): computed from the model's resident + // (non-expert) footprint vs the governed VRAM budget. `Some(path)` when + // resident overflows as-shipped and a cached device-fit override fits (the + // launcher sources resident from it via `LLAMA_RESIDENT_OVERRIDE`); `None` + // when resident fits natively OR no override is cached yet (resolver #35). + let resident_override = self.compute_resident_override(&model); // #302 invariant 1: mark the model's artifact ACTIVE before any spawn // touches it — the NvmeServingTierPool must never migrate the GGUF the // engine is loading or serving. Model change swaps the registration. @@ -1111,6 +1187,7 @@ impl ServingDaemonModule { // offloadable layer). [[LanePlacement]]. placement: crate::inference::llama_server::LanePlacement::Gpu, expert_placement, + resident_override, }; // One reconcile at a time. If the swap finds `true`, another is already diff --git a/core/expert-pager-policy/src/bin/cooccur-ceiling.rs b/core/expert-pager-policy/src/bin/cooccur-ceiling.rs new file mode 100644 index 0000000000..46b693b707 --- /dev/null +++ b/core/expert-pager-policy/src/bin/cooccur-ceiling.rs @@ -0,0 +1,242 @@ +//! cooccur-ceiling — offline VDD measurement of the CROSS-LAYER PREFETCH +//! predictor's ceiling (the `CrossLayerExpertPredictor` lever) on a real +//! `GGML_MOE_TRACE_FILE`. +//! +//! The bandit EMA residency curve (moe-pager-driver) answers "which +//! experts to KEEP resident across tokens" — a recency-family signal. This +//! bin answers a DIFFERENT question: within ONE forward pass, given the +//! experts that fired in layer L, how predictable are layer L+1's experts +//! from learned co-occurrence? If high, prefetching L+1 RAM→VRAM on a copy +//! stream while L computes can HIDE the per-expert H2D latency the residency +//! curve can't avoid — the two levers compose. +//! +//! Method (honest held-out): segment the trace into per-token, per-layer +//! expert sets. Train an adjacent-layer noisy-OR co-occurrence model on the +//! first `--train-frac` of DECODE tokens; on the rest, for each layer step +//! L→L+1 predict L+1's top-|L+1| experts from L's fired experts and score +//! the hit fraction. Baseline = predict L+1 as the PREVIOUS token's L+1 +//! experts (recency). The gap is the exploitable cross-layer structure. +//! +//! Usage: +//! cooccur-ceiling --trace --synth-layers [--train-frac 0.6] + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +use expert_pager_policy::segment::{parse_records, TkeyTable, TokenSegmenter}; +use expert_pager_policy::ExpertId; + +struct Args { + trace: PathBuf, + synth_layers: u32, + train_frac: f64, +} + +fn parse_args() -> Result { + let mut trace = None; + let mut synth_layers = None; + let mut train_frac = 0.6f64; + let argv: Vec = std::env::args().skip(1).collect(); + let mut i = 0; + while i < argv.len() { + let flag = argv[i].as_str(); + let value = argv + .get(i + 1) + .ok_or_else(|| format!("{flag} needs a value"))?; + match flag { + "--trace" => trace = Some(PathBuf::from(value)), + "--synth-layers" => { + synth_layers = Some(value.parse().map_err(|e| format!("--synth-layers: {e}"))?) + } + "--train-frac" => { + train_frac = value.parse().map_err(|e| format!("--train-frac: {e}"))? + } + other => return Err(format!("unknown flag {other}")), + } + i += 2; + } + Ok(Args { + trace: trace.ok_or("--trace required")?, + synth_layers: synth_layers.ok_or("--synth-layers required")?, + train_frac, + }) +} + +/// Group one token's flat expert set into per-layer expert sets. +fn by_layer(token: &HashSet) -> HashMap> { + let mut m: HashMap> = HashMap::new(); + for e in token { + m.entry(e.layer).or_default().insert(e.expert); + } + m +} + +/// Adjacent-layer co-occurrence tallies: cooccur[(L, p)][n] = times expert +/// `n` in layer L+1 fired in a pass where expert `p` fired in layer L. seen +/// counts the denominator P(n | L, p). +#[derive(Default)] +struct CoOccur { + seen: HashMap<(u32, u32), u64>, + cooccur: HashMap<(u32, u32), HashMap>, +} + +impl CoOccur { + fn learn(&mut self, layers: &HashMap>, n_layers: u32) { + for l in 0..n_layers.saturating_sub(1) { + let (Some(prev), Some(next)) = (layers.get(&l), layers.get(&(l + 1))) else { + continue; + }; + for &p in prev { + *self.seen.entry((l, p)).or_insert(0) += 1; + let row = self.cooccur.entry((l, p)).or_default(); + for &n in next { + *row.entry(n).or_insert(0) += 1; + } + } + } + } + + /// Predict layer L+1's experts (noisy-OR over the fired predecessors in + /// layer L), return the top-`budget` by confidence. + fn predict(&self, l: u32, prev: &HashSet, budget: usize) -> HashSet { + // noisy-OR: P(n) = 1 - Π_p (1 - P(n|p)) + let mut not_fire: HashMap = HashMap::new(); + for &p in prev { + let Some(seen) = self.seen.get(&(l, p)).copied().filter(|&s| s > 0) else { + continue; + }; + if let Some(row) = self.cooccur.get(&(l, p)) { + for (&n, &c) in row { + let cond = c as f64 / seen as f64; + *not_fire.entry(n).or_insert(1.0) *= 1.0 - cond; + } + } + } + let mut scored: Vec<(f64, u32)> = + not_fire.into_iter().map(|(n, nf)| (1.0 - nf, n)).collect(); + let b = budget.min(scored.len()); + if b == 0 { + return HashSet::new(); + } + let idx = (b - 1).min(scored.len() - 1); + scored.select_nth_unstable_by(idx, |a, c| { + c.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal) + }); + scored[..b].iter().map(|x| x.1).collect() + } +} + +fn hit(pred: &HashSet, actual: &HashSet) -> f64 { + if actual.is_empty() { + return 0.0; + } + actual.iter().filter(|e| pred.contains(e)).count() as f64 / actual.len() as f64 +} + +fn main() { + let args = match parse_args() { + Ok(a) => a, + Err(e) => { + eprintln!("cooccur-ceiling: {e}"); + eprintln!("usage: cooccur-ceiling --trace --synth-layers [--train-frac 0.6]"); + std::process::exit(2); + } + }; + let bytes = match std::fs::read(&args.trace) { + Ok(b) => b, + Err(e) => { + eprintln!("cooccur-ceiling: read {}: {e}", args.trace.display()); + std::process::exit(1); + } + }; + let table = TkeyTable::for_layers(args.synth_layers); + let (records, _consumed) = parse_records(&bytes); + + // Segment into completed token expert-sets, grouped per layer. + let mut segmenter = TokenSegmenter::new(); + let mut tokens: Vec>> = Vec::new(); + for rec in records { + if let Some(token) = segmenter.push(rec, &table) { + if !token.is_empty() { + tokens.push(by_layer(&token)); + } + } + } + // The big prefill token skews sizes; drop the first (prefill) token so + // the measurement is on decode-shaped passes. + if !tokens.is_empty() { + tokens.remove(0); + } + let n = tokens.len(); + if n < 4 { + eprintln!("cooccur-ceiling: only {n} tokens — need more trace"); + std::process::exit(1); + } + let split = ((n as f64) * args.train_frac) as usize; + let split = split.clamp(1, n - 1); + + // Train. + let mut model = CoOccur::default(); + for tok in &tokens[..split] { + model.learn(tok, args.synth_layers); + } + + // Test each L→L+1 step on held-out tokens: co-occurrence vs recency. + let mut co_sum = 0.0f64; + let mut rec_sum = 0.0f64; + let mut steps = 0u64; + // The decision-relevant metric: of the experts RECENCY MISSES (the ones + // a prefetch predictor would exist to catch), what fraction does + // co-occurrence recover? High = prefetch has a real niche on top of + // residency; low = co-occurrence is redundant-or-worse than recency. + let mut miss_total = 0u64; + let mut miss_caught = 0u64; + for t in split..n { + let cur = &tokens[t]; + let prevtok = &tokens[t - 1]; + for l in 0..args.synth_layers.saturating_sub(1) { + let (Some(prev_experts), Some(actual)) = (cur.get(&l), cur.get(&(l + 1))) else { + continue; + }; + let budget = actual.len(); // predict exactly as many as fire + let co_pred = model.predict(l, prev_experts, budget); + co_sum += hit(&co_pred, actual); + // Recency baseline: last token's SAME layer L+1 experts. + let empty = HashSet::new(); + let rec_pred = prevtok.get(&(l + 1)).unwrap_or(&empty); + rec_sum += hit(rec_pred, actual); + // Co-occurrence recall on the recency-miss subset. + for e in actual { + if !rec_pred.contains(e) { + miss_total += 1; + if co_pred.contains(e) { + miss_caught += 1; + } + } + } + steps += 1; + } + } + + let scale = if steps > 0 { 1.0 / steps as f64 } else { 0.0 }; + println!( + "# cooccur-ceiling: {n} tokens ({split} train / {} test), {steps} layer-steps scored", + n - split + ); + println!("cross_layer_cooccur_hit {:.4}", co_sum * scale); + println!("recency_same_layer_hit {:.4}", rec_sum * scale); + println!( + "# structure beyond recency: {:+.4} (positive = co-occurrence beats recency overall)", + (co_sum - rec_sum) * scale + ); + let miss_recall = if miss_total > 0 { + miss_caught as f64 / miss_total as f64 + } else { + 0.0 + }; + println!( + "cooccur_recall_on_recency_misses {:.4} ({miss_caught}/{miss_total}) \ + — the prefetch niche: co-occurrence's recall on the experts recency misses", + miss_recall + ); +} diff --git a/core/expert-pager-policy/src/bin/moe-pager-driver.rs b/core/expert-pager-policy/src/bin/moe-pager-driver.rs index dc5ce536f1..a5ebfa8476 100644 --- a/core/expert-pager-policy/src/bin/moe-pager-driver.rs +++ b/core/expert-pager-policy/src/bin/moe-pager-driver.rs @@ -34,7 +34,14 @@ use expert_pager_policy::BanditPlanController; struct Args { trace: PathBuf, - table: PathBuf, + /// Operator-authored tkey→layer JSON. Mutually exclusive with + /// `synth_layers` — exactly one is required. + table: Option, + /// Synthesize the tkey→layer map from layer count alone + /// ([`TkeyTable::for_layers`]) — the zero-config seam `MoeTraceTail` + /// uses. Lets the driver replay a completed trace on any box + /// without an operator JSON (offline warm-coverage measurement). + synth_layers: Option, plan: PathBuf, budget_bytes: u64, window_k: u32, @@ -47,11 +54,20 @@ struct Args { /// Precision-ladder index the unpinned cold tail fetches from (the /// small-quant bank — the beat-WASTE knob). None = container default. default_tier: Option, + /// Offline replay: exit once the trace stops growing (EOF), print a + /// SUMMARY line with mean decode-token serving hit. Without it the + /// driver tails forever (the live-next-to-serve mode). + once: bool, + /// Override the predictor's residency budget (expert slots). Default + /// is auto (first token × 1.5). Set this to measure coverage at a + /// REALISTIC free-VRAM budget — the device-fit tradeoff curve. + budget_slots: Option, } fn parse_args() -> Result { let mut trace = None; let mut table = None; + let mut synth_layers = None; let mut plan = None; let mut budget_bytes = None; let mut window_k = None; @@ -60,16 +76,28 @@ fn parse_args() -> Result { let mut poll_ms = 50u64; let mut pin_tier = None; let mut default_tier = None; + let mut once = false; + let mut budget_slots = None; let argv: Vec = std::env::args().skip(1).collect(); let mut i = 0; while i < argv.len() { let flag = argv[i].as_str(); + // `--once` is a lone toggle (no value); handle it before the + // value-consuming flags so it doesn't swallow the next arg. + if flag == "--once" { + once = true; + i += 1; + continue; + } let value = argv .get(i + 1) .ok_or_else(|| format!("{flag} needs a value"))?; match flag { "--trace" => trace = Some(PathBuf::from(value)), "--table" => table = Some(PathBuf::from(value)), + "--synth-layers" => { + synth_layers = Some(value.parse().map_err(|e| format!("--synth-layers: {e}"))?) + } "--plan" => plan = Some(PathBuf::from(value)), "--budget-bytes" => { budget_bytes = Some(value.parse().map_err(|e| format!("--budget-bytes: {e}"))?) @@ -86,13 +114,20 @@ fn parse_args() -> Result { "--default-tier" => { default_tier = Some(value.parse().map_err(|e| format!("--default-tier: {e}"))?) } + "--budget-slots" => { + budget_slots = Some(value.parse().map_err(|e| format!("--budget-slots: {e}"))?) + } other => return Err(format!("unknown flag {other}")), } i += 2; } + if table.is_some() == synth_layers.is_some() { + return Err("exactly one of --table or --synth-layers is required".into()); + } Ok(Args { trace: trace.ok_or("--trace required")?, - table: table.ok_or("--table required")?, + table, + synth_layers, plan: plan.ok_or("--plan required")?, budget_bytes: budget_bytes.ok_or("--budget-bytes required")?, window_k: window_k.ok_or("--window-k required")?, @@ -101,6 +136,8 @@ fn parse_args() -> Result { poll_ms, pin_tier, default_tier, + once, + budget_slots, }) } @@ -117,19 +154,29 @@ fn main() { } }; - let table_json = match std::fs::read_to_string(&args.table) { - Ok(s) => s, - Err(e) => { - eprintln!("moe-pager-driver: read table {}: {e}", args.table.display()); - std::process::exit(1); + let table = match (&args.table, args.synth_layers) { + (Some(path), _) => { + let table_json = match std::fs::read_to_string(path) { + Ok(s) => s, + Err(e) => { + eprintln!("moe-pager-driver: read table {}: {e}", path.display()); + std::process::exit(1); + } + }; + match TkeyTable::from_json(&table_json) { + Ok(t) => t, + Err(e) => { + eprintln!("moe-pager-driver: {e}"); + std::process::exit(1); + } + } } - }; - let table = match TkeyTable::from_json(&table_json) { - Ok(t) => t, - Err(e) => { - eprintln!("moe-pager-driver: {e}"); - std::process::exit(1); + (None, Some(n)) => { + println!("# synthesizing tkey table for {n} layers (zero-config seam)"); + TkeyTable::for_layers(n) } + // parse_args guarantees exactly one is set. + (None, None) => unreachable!("parse_args enforces --table xor --synth-layers"), }; println!( "# driver up: table {} tkeys, trace {}, plan {}, pin_top {}, rewrite_every {}", @@ -148,17 +195,32 @@ fn main() { let mut offset: u64 = 0; let mut carry: Vec = Vec::new(); let mut token_idx: u64 = 0; + // Offline (`--once`) warm-coverage aggregation: mean serving hit over + // DECODE tokens only (prefill's big batches are not representative of + // the steady-state paging load). `in_decode` flips true at the + // prefill→decode boundary. + let mut in_decode = false; + let mut decode_tokens = 0u64; + let mut hit_sum = 0.0f64; loop { let mut file = match std::fs::File::open(&args.trace) { Ok(f) => f, Err(_) => { + if args.once && offset > 0 { + break; + } // Serve not started yet — wait for the trace to appear. std::thread::sleep(std::time::Duration::from_millis(args.poll_ms)); continue; } }; let len = file.metadata().map(|m| m.len()).unwrap_or(0); + // Offline replay: the trace has stopped growing and we've drained + // it — emit the summary and exit instead of tailing forever. + if args.once && offset > 0 && len <= offset { + break; + } if len < offset { // Truncated: a NEW serve started. Reset the whole loop // state — stale scores belong to the previous generation. @@ -193,6 +255,7 @@ fn main() { // prefill batches churn the cache, so the // boundary is when the hint matters most). if boundary.observe(experts.len()) { + in_decode = true; if let Some(ctl) = controller.as_ref() { match ctl.write_tiered_plan( &args.plan, @@ -213,15 +276,25 @@ fn main() { } } let ctl = controller.get_or_insert_with(|| { - let budget = experts.len() * 3 / 2; + let budget = args.budget_slots.unwrap_or(experts.len() * 3 / 2); println!( - "# first token: {} experts -> budget {budget} slots", - experts.len() + "# first token: {} experts -> budget {budget} slots{}", + experts.len(), + if args.budget_slots.is_some() { + " (operator override)" + } else { + " (auto: ×1.5)" + } ); BanditPlanController::new(budget) }); let hit = ctl.observe_token(&experts); token_idx += 1; + // Warm-coverage aggregation: decode tokens only. + if in_decode { + decode_tokens += 1; + hit_sum += hit; + } let arms: Vec = ctl .per_arm_reward() .iter() @@ -259,4 +332,18 @@ fn main() { } std::thread::sleep(std::time::Duration::from_millis(args.poll_ms)); } + + // Reached only in `--once` (offline) mode: the trace is fully drained. + // The mean decode-token serving hit IS the warm schedulable coverage — + // the fraction of per-token expert H2D copies the pager avoids. + let mean_hit = if decode_tokens > 0 { + hit_sum / decode_tokens as f64 + } else { + 0.0 + }; + println!( + "# SUMMARY tokens_total {token_idx} decode_tokens {decode_tokens} \ + mean_decode_hit {mean_hit:.4} unknown_tkeys {}", + segmenter.unknown_tkeys + ); } diff --git a/core/expert-pager-policy/src/division.rs b/core/expert-pager-policy/src/division.rs new file mode 100644 index 0000000000..8dc4fb9cc5 --- /dev/null +++ b/core/expert-pager-policy/src/division.rs @@ -0,0 +1,384 @@ +//! DivisionPolicy — the governor's VRAM-division RL brain (#2/#3, the second control rung). +//! +//! The pager decides WHICH experts stay resident (reward = per-token hit-rate, cheap → online +//! [`crate::decay::DecayBandit`]). This rung decides HOW TO DIVIDE the card: how much VRAM goes to the +//! resident (non-expert) weights vs the expert cache, to MAXIMIZE tok/s. The reward here — actual +//! tok/s — is EXPENSIVE to sample (a serve + warm ≈ minutes), so a naive online bandit would flail. +//! The fix is SIM-WARM-START: predict tok/s for every candidate division OFFLINE from the measured +//! coverage curve (no serve), pick from that, then a SLOW bandit refines each arm from real measured +//! tok/s. Warm-start from the model, reinforcement-optimize on the real hardware. +//! +//! Same shape as the pager: POLICY lives here (windows-clean, testable standalone); the serving_daemon +//! ACTUATES it (feeds the `--resident-only` tier manifests + live tok/s, applies the chosen +//! {resident_tier, device_budget_bytes} to the plan file). This is the reusable brain of governor #2. +//! +//! The control law is FRACTAL: pager (experts↔hit-rate) → THIS (VRAM split↔tok/s) → grid (node↔throughput). + +/// One resident precision tier, read from a `--resident-only` sidecar manifest +/// (`{tier_label, resident_bytes}`). resident_bytes is the MEASURED footprint at that precision. +#[derive(Debug, Clone)] +pub struct ResidentTier { + pub label: String, + pub resident_bytes: u64, +} + +/// The device's fixed budget the division must fit inside (all VRAM figures, bytes). +#[derive(Debug, Clone, Copy)] +pub struct HardwareBudget { + pub vram_total_bytes: u64, + pub kv_bytes: u64, // KV cache reservation + pub compute_reserve_bytes: u64, // graph scratch + activations headroom +} + +impl HardwareBudget { + /// VRAM left for the expert cache after a given resident tier is placed. Saturating at 0 + /// (a tier whose resident + KV + reserve exceeds VRAM is infeasible → no cache). + pub fn expert_cache_bytes(&self, resident_bytes: u64) -> u64 { + self.vram_total_bytes + .saturating_sub(resident_bytes) + .saturating_sub(self.kv_bytes) + .saturating_sub(self.compute_reserve_bytes) + } +} + +/// The per-model MoE shape that turns a cache size into a fetch cost. +#[derive(Debug, Clone, Copy)] +pub struct MoeShape { + pub expert_bytes: u64, // one expert record (all matrices packed), e.g. 8_093_696 for K3 + pub experts_per_token: u64, // routed activations per decode token (≈ top_k * n_moe_layers) +} + +/// Coverage(cache_slots) = fraction of a token's experts already resident, as a piecewise-linear +/// curve over MEASURED points. Monotone non-decreasing, saturating. Defaults to the K3 trace-replay +/// numbers ([[k3-coverage-vs-vram-curve]]); a model supplies its own points as it's measured. +#[derive(Debug, Clone)] +pub struct CoverageModel { + points: Vec<(u64, f64)>, // (slots, coverage in [0,1]), sorted ascending by slots +} + +impl CoverageModel { + /// `points` need not be pre-sorted; (0,0) is implied. Coverage is clamped to [0,1]. + pub fn new(mut points: Vec<(u64, f64)>) -> Self { + points.push((0, 0.0)); + points.sort_by_key(|p| p.0); + points.dedup_by_key(|p| p.0); + for p in &mut points { + p.1 = p.1.clamp(0.0, 1.0); + } + Self { points } + } + + /// The measured K3 bandit-residency curve (slots → coverage), from real trace replay. + pub fn k3_measured() -> Self { + Self::new(vec![ + (250, 0.138), + (500, 0.235), + (1000, 0.364), + (1500, 0.451), + (2000, 0.513), + (4024, 0.657), + ]) + } + + /// Interpolate coverage at `slots` (piecewise-linear; flat beyond the last measured point — + /// we never extrapolate an optimistic coverage past what was measured). + pub fn coverage(&self, slots: u64) -> f64 { + if self.points.is_empty() { + return 0.0; + } + let last = self.points.last().unwrap(); + if slots >= last.0 { + return last.1; // saturate, don't extrapolate upward + } + // find the bracketing segment + let mut prev = self.points[0]; + for &cur in &self.points[1..] { + if slots < cur.0 { + let span = (cur.0 - prev.0) as f64; + let t = if span > 0.0 { (slots - prev.0) as f64 / span } else { 0.0 }; + return prev.1 + t * (cur.1 - prev.1); + } + prev = cur; + } + prev.1 + } +} + +/// A candidate way to divide the card. +#[derive(Debug, Clone)] +pub struct DivisionConfig { + pub tier_idx: usize, // index into the tier catalog + pub device_budget_bytes: u64, // expert cache budget this tier frees + pub cache_slots: u64, // device_budget_bytes / expert_bytes +} + +/// Predicted decode tok/s for a division, computed OFFLINE (no serve). This is the warm-start prior. +/// +/// coverage = model(slots); a cache HIT skips the per-token H2D, a MISS pays it: +/// h2d_bytes/token = (1 - coverage) * experts_per_token * expert_bytes +/// t_token ≈ h2d_bytes / pcie_h2d_bps + compute_floor_s +/// tok/s ≈ 1 / t_token +/// Higher coverage → less H2D → faster. `compute_floor_s` is the irreducible per-token compute +/// (matmuls + kernel launches) that residency can't remove — the ceiling the curve approaches. +pub fn predict_tok_s( + cfg: &DivisionConfig, + model: &CoverageModel, + shape: &MoeShape, + pcie_h2d_bps: f64, + compute_floor_s: f64, +) -> f64 { + let coverage = model.coverage(cfg.cache_slots); + let h2d_bytes = (1.0 - coverage) * shape.experts_per_token as f64 * shape.expert_bytes as f64; + let t_token = (h2d_bytes / pcie_h2d_bps.max(1.0)) + compute_floor_s.max(0.0); + if t_token > 0.0 { 1.0 / t_token } else { 0.0 } +} + +/// Enumerate the feasible divisions over a tier catalog: each tier that fits VRAM yields one config +/// with the cache budget its resident footprint frees. Infeasible tiers (resident > VRAM) are dropped. +pub fn feasible_divisions( + tiers: &[ResidentTier], + hw: &HardwareBudget, + shape: &MoeShape, +) -> Vec { + let mut out = Vec::new(); + for (i, t) in tiers.iter().enumerate() { + let budget = hw.expert_cache_bytes(t.resident_bytes); + let slots = if shape.expert_bytes > 0 { budget / shape.expert_bytes } else { 0 }; + if budget == 0 || slots == 0 { + continue; // this tier leaves no room for a cache — not a useful division + } + out.push(DivisionConfig { tier_idx: i, device_budget_bytes: budget, cache_slots: slots }); + } + out +} + +/// EMA rate for the measured-tok/s reward — how fast a division's value tracks real serves. +const REWARD_ALPHA: f64 = 0.4; + +/// The slow bandit over divisions. Every arm starts with its OFFLINE predicted tok/s as a prior; +/// the arm that gets SERVED updates its value toward the MEASURED tok/s (EMA). `choose` serves the +/// best current value, so a good prediction is exploited immediately and a wrong one is corrected as +/// soon as it's measured — the expensive reward is spent only on the arm we actually run. +#[derive(Debug, Clone)] +pub struct DivisionBandit { + configs: Vec, + value: Vec, // current tok/s estimate per arm (prior, then EMA of measured) + measured: Vec, // has this arm been served at least once? +} + +impl DivisionBandit { + /// Warm-start every arm from the offline predictor. Empty catalog → an empty bandit (`choose` + /// returns None); the caller falls back to its current fixed division. + pub fn warm_start( + configs: Vec, + model: &CoverageModel, + shape: &MoeShape, + pcie_h2d_bps: f64, + compute_floor_s: f64, + ) -> Self { + let value = configs + .iter() + .map(|c| predict_tok_s(c, model, shape, pcie_h2d_bps, compute_floor_s)) + .collect(); + let measured = vec![false; configs.len()]; + Self { configs, value, measured } + } + + /// The division to serve now: argmax current value (predicted-or-measured). Ties → lowest index. + pub fn choose(&self) -> Option<&DivisionConfig> { + self.value + .iter() + .enumerate() + .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(i, _)| &self.configs[i]) + } + + /// Feed back the MEASURED tok/s for the tier that was served (the reward). First measurement + /// REPLACES the prior (the real number beats the prediction outright); later ones EMA in, so a + /// non-stationary workload keeps the estimate fresh. + pub fn observe(&mut self, tier_idx: usize, measured_tok_s: f64) { + if let Some(pos) = self.configs.iter().position(|c| c.tier_idx == tier_idx) { + if self.measured[pos] { + self.value[pos] = (1.0 - REWARD_ALPHA) * self.value[pos] + REWARD_ALPHA * measured_tok_s; + } else { + self.value[pos] = measured_tok_s; // prior → truth on first real serve + self.measured[pos] = true; + } + } + } + + pub fn predicted_value(&self, tier_idx: usize) -> Option { + self.configs.iter().position(|c| c.tier_idx == tier_idx).map(|p| self.value[p]) + } + pub fn len(&self) -> usize { self.configs.len() } + pub fn is_empty(&self) -> bool { self.configs.is_empty() } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn k3_shape() -> MoeShape { + MoeShape { expert_bytes: 8_093_696, experts_per_token: 1472 } + } + + /// what this catches: the coverage model interpolates between measured points, saturates past the + /// last one (never extrapolates optimistic), and is monotone. A regression here corrupts every + /// tok/s prediction downstream. + #[test] + fn coverage_interpolates_saturates_and_is_monotone() { + let m = CoverageModel::k3_measured(); + assert!((m.coverage(0) - 0.0).abs() < 1e-9); + assert!((m.coverage(250) - 0.138).abs() < 1e-6); + // midpoint 375 slots is between (250,.138) and (500,.235) + let mid = m.coverage(375); + assert!(mid > 0.138 && mid < 0.235, "interp {mid}"); + // saturates, doesn't extrapolate above the last measured coverage + assert!((m.coverage(9999) - 0.657).abs() < 1e-6); + // monotone non-decreasing + let mut prev = 0.0; + for s in (0..4500).step_by(100) { + let c = m.coverage(s); + assert!(c + 1e-9 >= prev, "non-monotone at {s}: {c} < {prev}"); + prev = c; + } + } + + /// what this catches: THE load-bearing property — more expert-cache VRAM (a smaller/cheaper + /// resident) predicts MORE tok/s, because coverage rises and per-token H2D falls. If this inverts, + /// the governor would "optimize" toward starving the cache. Uses the measured K3 curve + shape. + #[test] + fn more_cache_predicts_more_tok_s() { + let model = CoverageModel::k3_measured(); + let shape = k3_shape(); + let (bps, floor) = (25.0e9, 0.010); // ~25 GB/s H2D, 10 ms compute floor + let small = DivisionConfig { tier_idx: 0, device_budget_bytes: 500 * shape.expert_bytes, cache_slots: 500 }; + let big = DivisionConfig { tier_idx: 1, device_budget_bytes: 2000 * shape.expert_bytes, cache_slots: 2000 }; + let t_small = predict_tok_s(&small, &model, &shape, bps, floor); + let t_big = predict_tok_s(&big, &model, &shape, bps, floor); + assert!(t_big > t_small, "more cache must predict faster: {t_big} !> {t_small}"); + } + + /// what this catches: a tier whose resident overflows VRAM is dropped (no negative/huge budget), + /// and a tier that fits yields slots = freed/expert_bytes. + #[test] + fn feasible_divisions_drop_overflow_and_size_cache() { + let hw = HardwareBudget { + vram_total_bytes: 32 * 1024 * 1024 * 1024, + kv_bytes: 1 * 1024 * 1024 * 1024, + compute_reserve_bytes: 2 * 1024 * 1024 * 1024, + }; + let shape = k3_shape(); + let tiers = vec![ + ResidentTier { label: "q6_K".into(), resident_bytes: 33 * 1024 * 1024 * 1024 }, // overflows 32GB + ResidentTier { label: "q4_K".into(), resident_bytes: 25 * 1024 * 1024 * 1024 }, // fits + ResidentTier { label: "q3_K".into(), resident_bytes: 16 * 1024 * 1024 * 1024 }, // fits, more cache + ]; + let divs = feasible_divisions(&tiers, &hw, &shape); + assert_eq!(divs.len(), 2, "q6_K overflows and must be dropped"); + // the q3_K tier frees more VRAM than q4_K → more slots + let q4 = divs.iter().find(|d| d.tier_idx == 1).unwrap(); + let q3 = divs.iter().find(|d| d.tier_idx == 2).unwrap(); + assert!(q3.cache_slots > q4.cache_slots); + } + + /// what this catches: the bandit warm-starts from the predictor and then a MEASURED reward + /// overrides the prior on first serve (real number beats prediction). Encodes the sim-warm-start + /// + slow-refine contract that makes an expensive-reward learner viable. + #[test] + fn bandit_warm_starts_then_measurement_overrides_prior() { + let hw = HardwareBudget { + vram_total_bytes: 32 * 1024 * 1024 * 1024, + kv_bytes: 1 * 1024 * 1024 * 1024, + compute_reserve_bytes: 2 * 1024 * 1024 * 1024, + }; + let shape = k3_shape(); + let model = CoverageModel::k3_measured(); + let tiers = vec![ + ResidentTier { label: "q4_K".into(), resident_bytes: 25 * 1024 * 1024 * 1024 }, + ResidentTier { label: "q3_K".into(), resident_bytes: 16 * 1024 * 1024 * 1024 }, + ]; + let divs = feasible_divisions(&tiers, &hw, &shape); + let mut bandit = DivisionBandit::warm_start(divs, &model, &shape, 25.0e9, 0.010); + assert_eq!(bandit.len(), 2); + // warm-start: the bigger-cache tier (q3_K, idx 1) should predict faster and be chosen + assert_eq!(bandit.choose().unwrap().tier_idx, 1); + // now REALITY says q4_K (idx 0) actually served much faster (e.g. q3_K crushed quality/thrashed) + bandit.observe(0, 5.0); + assert_eq!(bandit.choose().unwrap().tier_idx, 0, "measured reward must override the prior"); + // a second measurement EMAs, not replaces + let before = bandit.predicted_value(0).unwrap(); + bandit.observe(0, 1.0); + let after = bandit.predicted_value(0).unwrap(); + assert!(after < before && after > 1.0, "EMA blend: {after} between prior 5.0 and new 1.0"); + } + + /// what this catches: the RL division bandit, fed the REAL measured V4-Flash residency curve + /// (BigMama RTX 5090, DeepSeek-V4-Flash UD-IQ2_M, `--n-cpu-moe` sweep), learns the SATURATION KNEE + /// from measurement — it picks the tok/s-max division (8 resident layers), NOT the max-residency + /// one, even though the latter pins ~11 GB more VRAM. This is the empirical proof that static + /// residency saturates: 0→8 layers buys +0.30 tok/s, 8→14 buys nothing (1.69 vs 1.68). A bandit + /// that merely maximized residency would waste the VRAM the device cache should own — which is why + /// "minimal static residency + max device cache" is the division the governor must converge to. + /// Measured 2026-08-03; sweep = scratch-v4flash-sweep.sh, curve = division-curve.jsonl. + #[test] + fn bandit_learns_residency_saturation_from_measured_v4flash_curve() { + // Each arm is one `--n-cpu-moe` division. This axis is STATIC layer residency (not the device + // cache), so budget/slots are placeholders and the MEASURED reward — not the coverage prior — + // drives the choice. Flat prior (empty coverage model → equal warm-start) makes that explicit. + let divs = vec![ + DivisionConfig { tier_idx: 0, device_budget_bytes: 0, cache_slots: 0 }, // ncpu=48, 0 resident + DivisionConfig { tier_idx: 1, device_budget_bytes: 0, cache_slots: 0 }, // ncpu=40, 8 resident + DivisionConfig { tier_idx: 2, device_budget_bytes: 0, cache_slots: 0 }, // ncpu=34, 14 resident + ]; + let flat = CoverageModel::new(vec![]); + let shape = MoeShape { expert_bytes: 1, experts_per_token: 1 }; + let mut bandit = DivisionBandit::warm_start(divs, &flat, &shape, 25.0e9, 0.010); + assert_eq!(bandit.len(), 3); + // feed the REAL measured decode tok/s (BigMama 5090, this session) + bandit.observe(0, 1.39); // 0 resident (all experts stream from NVMe) + bandit.observe(1, 1.69); // 8 resident — the knee + bandit.observe(2, 1.68); // 14 resident — saturated, no gain over 8 at +11 GB VRAM + // the bandit converges on the KNEE (idx 1), not the max-residency arm (idx 2) + assert_eq!(bandit.choose().unwrap().tier_idx, 1, "must learn the saturation knee, not max residency"); + // strictly better than the all-stream baseline + assert!(bandit.predicted_value(1).unwrap() > bandit.predicted_value(0).unwrap()); + // saturation: MORE residency (idx 2) is NOT better than the knee (idx 1) + assert!(bandit.predicted_value(2).unwrap() <= bandit.predicted_value(1).unwrap()); + } + + /// what this catches: the load-bearing reason the bandit corrects its OWN prior. The device-cache + /// coverage/tok-s response is NON-MONOTONIC in VRAM budget — measured on V4-Flash (5090, UD-IQ2_M, + /// GGML_MOE_VRAM_CACHE_GB sweep): 6GB/992 slots → 1.80 (undersized, evict-churn), 12GB/1985 → 3.10 + /// (plateau knee), 22GB/3630 → 2.96 (oversized: 100% hit but the O(slots) reserve_slot eviction scan + /// costs). Coverage is 100% from ~12GB up, so predict_tok_s's monotonic "more slots → more tok/s" + /// prior is WRONG past the coverage knee: it would pick 22GB. Only the MEASURED reward finds the 12GB + /// optimum — and 12GB frees ~20GB of a 32GB card for other lanes. If this test inverts, the governor + /// oversizes the cache and starves co-resident models. Measured 2026-08-03; coverage-curve.jsonl. + #[test] + fn bandit_finds_nonmonotonic_device_cache_budget_optimum() { + // arms = device-cache VRAM budgets; tier_idx maps to the sweep point. cache_slots carries the + // measured slot count so a naive coverage prior WOULD rank 22GB highest — the measured reward + // must override that and land on 12GB (the plateau knee, max tok/s at min VRAM). + let divs = vec![ + DivisionConfig { tier_idx: 0, device_budget_bytes: 6 << 30, cache_slots: 992 }, // undersized + DivisionConfig { tier_idx: 1, device_budget_bytes: 12 << 30, cache_slots: 1985 }, // knee + DivisionConfig { tier_idx: 2, device_budget_bytes: 22 << 30, cache_slots: 3630 }, // oversized + ]; + // The monotonic prior (k3 curve) ranks the biggest cache first — set up the wrong belief... + let model = CoverageModel::k3_measured(); + let shape = MoeShape { expert_bytes: 2_163_200, experts_per_token: 774 }; + let mut bandit = DivisionBandit::warm_start(divs, &model, &shape, 25.0e9, 0.010); + assert_eq!(bandit.predicted_value(2).unwrap() >= bandit.predicted_value(1).unwrap(), true, + "prior should (wrongly) favor the biggest cache before measurement"); + // ...then MEASUREMENT corrects it to the real non-monotonic curve. + bandit.observe(0, 1.80); + bandit.observe(1, 3.10); + bandit.observe(2, 2.96); + assert_eq!(bandit.choose().unwrap().tier_idx, 1, "bandit must land on the 12GB plateau knee"); + // oversizing (22GB) is NOT better than the knee — so the governor frees the difference + assert!(bandit.predicted_value(2).unwrap() <= bandit.predicted_value(1).unwrap()); + // undersizing (6GB, evict-churn) is clearly worse despite also showing 100% hit on warm tokens + assert!(bandit.predicted_value(0).unwrap() < bandit.predicted_value(1).unwrap()); + } +} diff --git a/core/expert-pager-policy/src/lib.rs b/core/expert-pager-policy/src/lib.rs index 427ac4a1fd..2e9233253e 100644 --- a/core/expert-pager-policy/src/lib.rs +++ b/core/expert-pager-policy/src/lib.rs @@ -8,11 +8,16 @@ pub mod controller; pub mod decay; +pub mod division; pub mod expert_id; pub mod plan_file; pub mod segment; pub use controller::BanditPlanController; pub use decay::{DecayBandit, EmaScoreboard, DECAY_ARMS, REWARD_ALPHA}; +pub use division::{ + feasible_divisions, predict_tok_s, CoverageModel, DivisionBandit, DivisionConfig, + HardwareBudget, MoeShape, ResidentTier, +}; pub use expert_id::ExpertId; pub use plan_file::{write_plan_file, PlanFileDocument, PlanPin, PLAN_FILE_VERSION}; diff --git a/core/vendor/llama.cpp b/core/vendor/llama.cpp index e3ce51df59..fa7e0d8e9e 160000 --- a/core/vendor/llama.cpp +++ b/core/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3ce51df596547b9540afa6b093559096025f475 +Subproject commit fa7e0d8e9e475387e40d2de02522b90bb22438be diff --git a/docs/architecture/GPU-RESIDENT-HOT-EXPERTS.md b/docs/architecture/GPU-RESIDENT-HOT-EXPERTS.md new file mode 100644 index 0000000000..e19374505a --- /dev/null +++ b/docs/architecture/GPU-RESIDENT-HOT-EXPERTS.md @@ -0,0 +1,85 @@ +# GPU-Resident Hot Experts — "trend to full GPU only" (task #23) + +**Goal (Joel):** the biggest single speedup — the residency system should *trend itself toward full-GPU*: +keep the hot expert working set resident in VRAM so decode's hot path computes fully on GPU with **zero +per-token NVMe fetch and zero host→VRAM copy**. As the pager identifies the stable hot set, it promotes +hot→VRAM / evicts cold, and the working set self-organizes toward VRAM residency. + +## Where the time actually goes (measured, top-8, ~1.9 s/token = 0.53 tok/s) + +The op-offload MoE path (`ggml-backend.cpp` ~1648-1824) does, per layer, per token: +1. **NVMe fetch** of missed experts into the pinned host cache (recency ~62% hit → ~38% miss). ~0.53 s. +2. **host→VRAM copy** of the *used* experts into `input_cpy` (the VRAM tensor the `MUL_MAT_ID` reads), + via `ggml_backend_tensor_set_async`. ~0.2 s. +3. **GPU compute** (the expert matmuls — already GPU-native). The rest. + +Two structural inefficiencies, independent of the fetch: +- **Redundant copies:** `input_cpy` persists across tokens (it's the split's VRAM buffer), but every token + re-copies *all* used experts — including the ~62% that were already resident from last token. +- **Serialization:** `ggml_backend_synchronize` (lines 1660, 1679) forces fetch→copy→compute to run + serially per layer instead of pipelining. + +## Design — three increments, lowest-risk first + +### Increment 1 — copy-skip (VRAM residency tracking on `input_cpy`) +Per split's `input_cpy` VRAM buffer, track the **set of expert ids currently resident** (last copied). Each +token, copy only the experts NOT already resident (the misses vs last token); skip the rest. At ~62% +token-to-token overlap this eliminates ~62% of the host→VRAM copies. No new VRAM — `input_cpy` already +holds them. Care: the sched double-buffers (`sched->cur_copy`) → keep a resident-set **per buffer id**. +Correctness: `MUL_MAT_ID` only reads the selected `ids`, so a skipped-but-still-resident expert is valid. + +### Increment 2 — persistent hot-expert VRAM cache +A device-side residency cache (VRAM analog of the pinned-host `ResidencyCache`): promote the hottest +experts (recency + the plan-file pin list) into a **persistent VRAM region** that survives across layers +and tokens. On expert access the seam checks VRAM-residency first: **hit ⇒ GPU-native, no fetch, no copy**; +miss ⇒ current fetch→copy path. Evict cold by the same score-hint/generation logic as the host cache. +This is where "trend to full GPU" lives: the promotion loop grows the VRAM-resident fraction as the hot +set stabilizes. + +### Increment 3 — pipeline (de-serialize) +Overlap the copy/fetch of the next layer's experts with the current layer's compute (double-buffer + +drop the hard syncs where the dependency allows). Bounded by sequential routing (layer N+1's experts +depend on N's output), but the copy of already-known experts can overlap. + +## The 32 GB constraint — the rate-distortion knob +On a 32 GB card the resident attention (~28-33 GB) fills VRAM, leaving little for expert residency. So the +"trend" is bounded here and becomes a **rate-distortion allocation**: VRAM spent on hot experts vs +high-fidelity attention. The **imatrix** (running now) is what makes that tradeoff intelligent — it lets us +down-quant the resident attention *and* the cold experts by measured importance, freeing VRAM the hot set +promotes into. On a bigger card (or smaller MoE) the trend approaches genuine full-GPU. This is the +misfit-moat fractal: same code, the governor spends the VRAM budget differently per device +([[device-fit-repeatable-primitive]], [[dynamic-precision-tiers-and-diversity]]). + +## Sequence +imatrix keystone (async now) → increment 1 (copy-skip, low-risk, measurable alone) → increment 2 +(VRAM hot cache, the "full-GPU trend") → increment 3 (pipeline). Each increment is measured on its own via +the `k3-bench` harness before stacking. Ref: [[k3-beats-waste-decisively]] (compute/copy-bound at top-8). + +## CRITICAL open question (verify before building Increment 1) +Increment 1 (copy-skip on `input_cpy`) only works if `input_cpy` **persists per (layer,matrix) tensor +across tokens**. But a per-tensor `input_cpy` = 92×3×~2 GB ≫ VRAM, so the sched almost certainly **reuses +a shared scratch `input_cpy`** across all MoE splits — meaning its contents churn *within* one token and +retain nothing tensor-specific for the next. **If shared-scratch (likely), copy-skip is a no-op and +Increment 2 is the real mechanism:** a SEPARATE persistent VRAM hot-expert region (not the scratch buffer), +keyed by stable ExpertId, that the matmul reads from on a hit. VERIFY the `input_cpy` allocation lifetime +(`ggml_backend_sched` reuse) first — it decides whether increment 1 exists or we go straight to increment 2. + +## Modular implementation (rework-proof — mechanism built once, policy injected) +Key: `ResidencyCache` is ALREADY generic over `(ggml_backend_buffer_type_t buft, ExpertFetcher& fetcher)`. +A VRAM hot-expert cache = the SAME class, instantiated with a DEVICE buft + a host→device fetcher. No new +cache class, no hardcoded constants — budget/window/pin arrive on the existing plan-file wire. + +Three small, parameterized pieces (each measured alone via `k3-bench`): +1. **`DeviceUploadFetcher : ExpertFetcher`** — the only genuinely new code. `fetch(dst_dev, host_src, n)` + does a host→device copy via the split backend's buffer-set API (not memcpy). ~15 lines. +2. **Instantiate** `k3_device_cache(budget = GGML_MOE_VRAM_CACHE_GB, device_buft = split backend's buft, + DeviceUploadFetcher)` — parameter-driven; `0` (unset) ⇒ disabled ⇒ current behavior, zero risk. + `get()`'s `memset(pad)` must be guarded for device slots (cudaMemset or skip; padding is CUDA-MMQ NaN + guard — handle once). +3. **Seam hook** (in the copy loop): before the host→VRAM copy, ask `k3_device_cache` for the expert's + device slot. **HIT ⇒ device→`input_cpy` VRAM→VRAM copy** (no NVMe, no host round-trip). MISS ⇒ current + host-cache path, then upload into the device cache. Same score-hint/generation eviction as the host + cache — the plan-file policy drives BOTH tiers with one wire. + +When stats land (imatrix, live hit-rates) NOTHING here reworks: the numbers only set `GGML_MOE_VRAM_CACHE_GB` +and the plan's pin/window — the mechanism is fixed. That is the point of the modular split. diff --git a/docs/architecture/MOE-SERVING-GOVERNED-BUDGET.md b/docs/architecture/MOE-SERVING-GOVERNED-BUDGET.md new file mode 100644 index 0000000000..990f953154 --- /dev/null +++ b/docs/architecture/MOE-SERVING-GOVERNED-BUDGET.md @@ -0,0 +1,151 @@ +# MoE Serving on a Governed Budget + +**Status:** draft (BigMama). The governor-interface sections are marked **[M5 OWNS]** — edit +them directly; they describe her lane and I've only sketched the seam I need from it. + +**Precedence:** this defers to [CONCURRENCY-STYLE-GUIDE](CONCURRENCY-STYLE-GUIDE.md), +[GENOME-FOUNDRY-SENTINEL](GENOME-FOUNDRY-SENTINEL.md) (`SubstrateGovernor`), and +[INFERENCE-LANES-REALISTIC](INFERENCE-LANES-REALISTIC.md). If it disagrees with those on a +substrate question, they win and this gets reconciled. + +## What is proven (keep) + +K3 (2.8T MoE, IQ2) serves coherent tokens on a 32 GB card by streaming experts from NVMe through a +bounded resident cache. The mechanism is real and model-agnostic: + +- The op-offload seam (`ggml-backend.cpp`) copies only the router-selected experts per token. +- `ResidencyCache` (`ggml-moe-residency.hpp`) is pure mechanism: size-classed pinned pools, recency + eviction, fed by an `ExpertFetcher` adapter (`DirectReadFetcher` on Windows NVMe, `MmapFaultFetcher` + portable). Keyed on `ExpertId` = `canonical_name_key(blk.N.ffn_{gate,up,down}_exps)` + index. +- That key is the **universal MoE naming** every one of the 140 `src/models/*.cpp` adapters shares + (glm4-moe, deepseek2, qwen3moe, kimi-k3, ...). Model-specific differences (K3's KDA attention, SITU) + live in the per-model graph adapter, not the paging path. **K3 is one model; the pager serves all.** +- top-k override (`--override-kv .expert_used_count`) cuts fetch + compute; measured `res_exp` + halves as expected. + +## The failure this fixes (the reason for the doc) + +Measured on BigMama, 2026-08-01, with the prototype serve script: + +``` +GGML_MOE_HOST_CACHE_GB=40 (hardcoded in a scratchpad .bat) +-> llama-server private commit 95.9 GB on a 63 GB box (40 GB pinned + model mmap working set) +-> 33 GB overcommitted to the pagefile -> thrash +-> DirectRead bandwidth collapses 3 GB/s -> ~205 MB/s ([FETCH] unbuf_ok, resolve_fail=0: not a fallback, real collapse) +-> [RETAIN] pool 5943/5943, evict=25/token, resident=2 (cache retains nothing; re-fetches the whole working set every token) +-> 36.8 s / decode token = 0.027 tok/s +``` + +The seam was correct (100% intra-token hit, top-8 confirmed). The cause was a **hardcoded residency +budget that overcommitted RAM** — the single failure mode the substrate's pressure/governor +architecture exists to prevent. A budget sized to *fit* free RAM (net of the model's mmap footprint) +never thrashes. On this box that is ~18-22 GB, not 40. + +## Principle: the governor owns the budget. Nothing else sets it. + +``` +SystemProfile (measured free RAM/VRAM, minus model mmap footprint, minus headroom) + -> SubstrateGovernor / MemoryPressureMonitor [M5 OWNS] policy + -> ServingExpertPager (observe pressure + expert trace -> budget_bytes + window_k + pin_list) + -> plan-file (atomic-rename JSON: {budget_bytes, window_k, pin_list}) the ONE wire + -> ResidencyCache (C++) reads budget from the plan-file. Pure mechanism. +``` + +Rules: + +1. **Delete `GGML_MOE_HOST_CACHE_GB` (and any VRAM-cache env) as a budget *input*.** They were the + prototype bypass. The C++ cache takes budget **only** from the plan-file. `MoeServingConfig` keeps + the *operational* knobs (which fetcher, stats/trace/capture sinks) but not the budget. +2. **Budget must be net of the model's mmap working set.** Pinned host cache + mmap resident set must + never exceed the RAM budget. This is the specific accounting that was missing today. +3. **Mechanism vs policy stays split** (already true in the traits): `ResidencyCache` / + `DirectReadFetcher` = mechanism; `ServingExpertPager` / `TierPolicy` / `PagerCaptureSink` = policy, + governed. Pressure rises -> governor shrinks `budget_bytes` -> plan-file updates -> cache evicts. + That is the RTOS control loop, not a new subsystem. +4. **No prototype rigging.** Serve launch, budget, and measurement come from the governed serving lane + in continuum-core, not scratchpad `.bat`s and hand-`curl`. + +## [M5 OWNS] Governor budget interface — answered (M5, 2026-08-01) + +1. **`serving_budget_bytes()` is NOT net of mmap, and never can be by watching free + memory.** It returns `0.80 x min(available_memory(), total_vram)` (`system_profile.rs` + via `host_budget_from` — a headroom figure). The trap: mmap'd weight pages are + file-backed, so the OS reports them "available" while they are load-bearing — + evicting them re-fetches weights, which is the fetch-bandwidth collapse wearing a + healthy free-RAM number. Net-of-mmap is therefore EXPLICIT plan arithmetic at the + reconcile site: `expert_host_cache = f(available, weights_bytes (already on the + planner, serving_daemon.rs), kv_total = kv_at(ctx) x lanes, OS floor)`, plus a + Windows private-commit ceiling (commit must never exceed physical RAM — pagefile + overcommit thrashes silently instead of OOMing loud). This derivation is task #287, + my lane. +2. **Push via `watch`, consume on your tick.** The canonical shape + (CONCURRENCY-STYLE-GUIDE): the governor publishes budget revisions on its own + cadence through its `watch::Sender`; `ServingExpertPager` borrows the + latest at each reconcile tick. Not per-event `PressureBroker` traffic — the broker + is for relief demands; the budget rides the profile snapshot. Sticky hysteresis on + the published value ([[never-thrash]]). +3. **Yes — register the residency cache as a governed pool** with a real + `evict_at_least` (the "every cache class has a decided eviction story" discipline; + the `broker_relieve_actually_deletes_from_an_over_budget_pool` test shape is the + guard). RAM-class pool, `PagedResourcePool` is the right primitive. +4. **(c) both, ordered — and the ordering lives in `TierPolicy`, agreed.** Shrink + `budget_bytes` FIRST (fast: plan-file actuation, next-token effect, no quality + cost), demote precision tier SECOND (slower, quality-affecting, and imatrix-gated + today so it is not yet a live lever). Both directions sticky so recovery grows + back without oscillation (#214 grow-back lesson applies here too). + +The wire needs nothing new: `plan_file.budget_bytes` already exists and your +`ResidencyCache` already honors it — the whole fix is the derivation feeding the +existing field. Cross-link: EXPERT-PAGING-CONTROL-LAW.md S5 (the plan wire), S7 (the +lever stack this composes with). + +## Measurement (VDD, not ad-hoc curl) + +Ad-hoc `curl` fooled us twice this session (ghost server on a shared port; phantom "hang" that was a +detached client failing to submit). Measurement is a tested harness: + +- `tests/test-moe-residency.cpp` — TDD for the keying/config invariants (now includes + `test_serving_config_from_env`) + a **VDD trace-replay** path that asserts a nonzero reuse floor on a + real captured trace (`replay_trace`). This is the regression gate for the pager. +- A serve-smoke that asserts decode produces tokens above a floor tok/s, so client flakiness can never + again read as a code regression. (To build; belongs in the governed serving lane, not a script.) + +## Measured on BigMama (2026-08-01) - inputs for the #287 derivation + +Ungoverned static budgets were run to characterize the box (they thrash by design; do NOT read them +as the governed result - a governed budget flexes with KV and never overcommits): + +| budget (hardcoded) | private commit | fetch | outcome | +|---|---|---|---| +| 40 GiB | 95.9 GB (>63.4 RAM) | 205 MB/s | pagefile thrash, 0.027 tok/s | +| 8 GiB (during load) | 60.6 GB | **2485 MB/s** | fits RAM, fetch fully recovered | +| 8 GiB (during decode, KV grew) | 63.9 GB | 165 MB/s | tipped over -> thrash again | + +Derived inputs for `expert_host_cache = f(available, weights_bytes, kv_total, OS floor)`: +- Total RAM 63.4 GB. Non-cache private footprint ~56 GB (base + KV + CUDA staging). +- K3 top-8 per-token expert working set ~5.5 GB (res_exp 2208). +- So the governed budget on this box is ~6 GB (62 - 56), which holds ~1 token's set -> ~40% recency + retention, misses fetching at the recovered ~2.5 GB/s. That is the regime that should pull tok/s back + toward 0.5 - NOT written off. The governor MUST subtract KV growth continuously (decode tipped 8 GiB + over) and cap on the Windows private-commit ceiling. + +## C++ mechanism - COMPLETE (k3-adopt) + +The cache honors the governed lease fully: `dd8463b74` config-manager + naming; `848f409c7` budget is +plan-file-only in governed mode (env can't overcommit); `b0e877cbd` poll the plan every token so a +governed budget can turn the cache ON (it starts at 0); `8680cefa9` free pools on budget DECREASE so +the governor can flex the cache down under pressure. Grow + shrink + enable-from-plan all validated +(builds green, test 94/0). Partial-evict-keeping-hottest on shrink is the one future refinement. + +## Graduated path: `serving/load kimi-k3` (replaces the rigged .bat) + +Three pieces, then a JOINT governed measurement (no more hand-`curl`): +1. **Catalog row for K3** (BigMama lane, `model_registry/catalog.rs`) - servable metadata + the serving + profile: `--n-cpu-moe 999`, `--override-kv kimi-k3.expert_used_count=int:8`, `-ngl `, + `GGML_MOE_DIRECT_READ=1`, `GGML_MOE_PLAN_FILE=`. Gated on 2+3 so it's not a servable row + that thrashes. +2. **Serving-lane MoE launch** (`inference/llama_server.rs`, M5) - set those flags + the plan-file env + for a streaming-MoE model; today the launcher wires KV/context budget but not the MoE offload env. +3. **#287 budget derivation** (M5) - writes `plan_file.budget_bytes` from the profile above, flexing + with KV. Register the cache as a governed `PagedResourcePool` with `evict_at_least` (the C++ free + path above is its actuator). diff --git a/docs/architecture/STORAGE-SERVING-TIER-GOVERNOR.md b/docs/architecture/STORAGE-SERVING-TIER-GOVERNOR.md new file mode 100644 index 0000000000..63e2dbc43c --- /dev/null +++ b/docs/architecture/STORAGE-SERVING-TIER-GOVERNOR.md @@ -0,0 +1,92 @@ +# Storage serving-tier governor — NVMe↔cold contention, managed like VRAM/RAM + +**Status:** design (2026-08-02, BigMama). Prompted by Joel: *"Like vram and memory, this +contention has to be managed too between cold storage and nvme."* Written because completing the K3 +expert container hit a manual disk fight (delete the 662 GB C: copy? pack from D:?) that the +substrate should have resolved on its own. + +## The gap + +`capacity::system_profile::DriveRole` today has two roles — `System` (OS/working) and `Cold` (the +big offload drive) — and documents `Cold` as *"where MoE expert sets are paged into VRAM on +demand."* **That is wrong for a spinning HDD.** Measured ([[hdd-vs-nvme-is-a-residency-tier]]): +streaming an expert bank off a 130 MB/s HDD = 156–544 s/token = unservable. The model conflates two +physically different tiers: + +- **Hot serving tier** — the artifact being *paged per token* (the K3 expert **container**). MUST + live on NVMe (2.6 GB/s). This is a device-like resource with a **budget** (NVMe free − system + reserve) and **contention** (two big models' containers can't both be resident). +- **Frozen tier** — artifacts NOT streamed per token: source GGUFs, backups, models-not-currently + served. These belong on the Cold drive. A GGUF the foundry already re-packed into a container is + frozen — its NVMe copy is pure duplication. + +Nothing governs the boundary, so a human (or an agent) hand-decides which 662 GB file to delete to +fit a 667 GB container. That is the VRAM-OOM story from a year ago, one tier down. + +## The model: NVMe is a governed hot-serving tier + +Reuse the machinery that already exists for `cargo-target` — do NOT invent a parallel one: + +- `system_resources::disk_reporters::TrackedDir` — a named, measured disk cache class. +- `paging::pool::ResourcePool` (`capacity_bytes` / `usage_bytes` / `evict_at_least`) — the eviction + contract. `CargoTargetPool` is the worked example. +- `disk_eviction::every_cache_class_has_a_decided_eviction_story` — the test that FAILS on an + undecided class (CLAUDE.md disk doctrine). + +Add **one new `ResourcePool`**: `NvmeServingTierPool`. + +- `capacity_bytes` = NVMe (System drive) total − a system reserve (OS/build headroom, governed, not + a magic constant). +- `usage_bytes` = bytes of hot serving artifacts resident on NVMe (containers + device-fit + overrides + primary GGUFs currently served). +- **`evict_at_least(want)` = migrate the coldest FROZEN / DUPLICATE artifact off NVMe to the Cold + drive** (not delete — *migrate*, and a verified duplicate that already exists on Cold is a pure + drop). Coldest-first: models not currently served, then source GGUFs whose experts are already in + a container, then LRU by last-served. Never evict the artifact the active serve is paging. + +The `DriveRole` doc is corrected: `Cold` = frozen storage, **never** the per-token streaming tier. + +## Serving integration (dissolves the manual fight) + +The serving planner already computes what a model needs. Extend it: before a serve, ask the storage +governor to **make the hot tier resident** — + +``` +storage.ensure_hot_resident(model) -> + needs = container_bytes(model) + device_fit_override_bytes(model) // the per-token-paged set + if nvme_free >= needs { place / keep on NVMe; done } + else { NvmeServingTierPool.evict_at_least(needs - nvme_free) } // migrate frozen/dupes to Cold + if still short -> Unfittable: route to grid, LOUD (no silent HDD-stream fallback) +``` + +`capacity::device_fit` already returns `Unfittable` for the VRAM tier; this is the exact same shape +one level down for the **NVMe** tier. The two compose: a model is GPU-servable on this box iff its +resident tier fits VRAM AND its paged tier fits NVMe (after governed frozen-eviction). + +## The K3 worked example (what should have happened today) + +1. Foundry packs the K3 expert container (667 GB, hot) onto NVMe. +2. `ensure_hot_resident(kimi-k3)` sees NVMe short by ~76 GB. +3. `NvmeServingTierPool.evict_at_least(76 GB)` finds the **C: IQ2 GGUF** — its experts are already in + the container (frozen) AND a verified identical copy exists on the Cold drive + (`D:\continuum-cold\…\UD-IQ2_XXS`, 16 shards, 662 GB). It is a pure duplicate → drop from NVMe. +4. Container fits. Serve. **No human deletes anything; the governor tiered it.** + +## Boundaries / ownership + +- The `ResourcePool` + `TrackedDir` + `disk_eviction` machinery is M5's `system_resources` / + governor lane — this pool registers there, same as `CargoTargetPool`. +- Duplicate detection (an NVMe GGUF whose identical twin is on Cold) is a small content/shard + identity check; a hash or (size + shard count + name) match is enough for the "safe to drop" + verdict — never drop an NVMe artifact without a verified Cold twin. +- Migration cost (NVMe→Cold write, or Cold→NVMe promote for a re-served cold model) is real disk + bandwidth; the governor leases it like any other, and a promote from a 130 MB/s HDD is a + first-token latency the planner must surface (it is why "serve a cold model" is not free). + +## Why this is the right shape + +The pager control law is fractal ([[pager-control-law-is-fractal-to-grid]]): VRAM↔RAM↔NVMe↔Cold is +the same predict-recency → allocate-under-budget → evict-coldest loop at every level. The serving +tier on NVMe is just the next rung down from the expert pager on VRAM — and one rung up from the +grid pooling capacity across nodes. Managing it with the SAME `ResourcePool` contract keeps the +substrate coherent instead of growing a bespoke disk-juggler. diff --git a/tools/scripts/start-server.sh b/tools/scripts/start-server.sh index 1ccca4db9e..917913a6f8 100755 --- a/tools/scripts/start-server.sh +++ b/tools/scripts/start-server.sh @@ -295,6 +295,97 @@ if [ -n "$CONTINUUM_RELEASE" ]; then PROFILE_LABEL="release" fi +# ── Windows: import the MSVC toolchain so cargo's CUDA (candle) build finds cl.exe ────────── +# candle compiles CUDA kernels (affine.cu, ...) via nvcc, which needs cl.exe as its host +# compiler plus INCLUDE/LIB. The cargo builds below run in THIS bash shell (unlike the +# llama-server cmake build, which runs inside a vcvars .bat), so without this nvcc fails with +# "Cannot find compiler 'cl.exe' in PATH" and the whole core build dies. Import it once: export +# INCLUDE/LIB verbatim (only cl.exe reads them) and prepend the EXACT MSVC + Windows SDK bin +# dirs (from vcvars, converted to unix) to PATH — nvcc finds cl.exe while bash's own PATH +# resolution stays intact (we add unix dirs, never overwrite PATH with the Windows one). VS2022 +# (14.4x) is selected explicitly: nvcc 12.x rejects the newer 14.5x/VS18 toolset. +if [ "$_mf_os" = windows ] && command -v nvcc >/dev/null 2>&1 && ! command -v cl.exe >/dev/null 2>&1; then + _vswhere="/c/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" + _vs="" + [ -x "$_vswhere" ] && _vs="$("$_vswhere" -version "[17.0,18.0)" -products '*' \ + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \ + -property installationPath 2>/dev/null | head -1)" + _vcvars_u="" + [ -n "$_vs" ] && _vcvars_u="$(cygpath -u "$_vs\\VC\\Auxiliary\\Build\\vcvars64.bat" 2>/dev/null)" + if [ -n "$_vcvars_u" ] && [ -f "$_vcvars_u" ]; then + # Capture vcvars' env via a .bat FILE, not inline `cmd //c "call … && set"`: the vcvars path + # has spaces AND parens, and escaping nested quotes through bash->cmd corrupts it (the same + # reason install-llama-server.sh emits a .bat). The .bat calls vcvars then dumps `set`. + _msbat="$(mktemp --suffix=.bat 2>/dev/null || echo "${TEMP:-/tmp}/_cc_vcvars_$$.bat")" + printf '@echo off\r\ncall "%s" >nul 2>&1\r\nset\r\n' "$(cygpath -w "$_vcvars_u")" > "$_msbat" + _msenv="$(cmd //c "$(cygpath -w "$_msbat")" 2>/dev/null | tr -d '\r')" + rm -f "$_msbat" + export INCLUDE="$(printf '%s\n' "$_msenv" | sed -n 's/^INCLUDE=//Ip' | head -1)" + export LIB="$(printf '%s\n' "$_msenv" | sed -n 's/^LIB=//Ip' | head -1)" + _vct="$(printf '%s\n' "$_msenv" | sed -n 's/^VCToolsInstallDir=//Ip' | head -1)" + _sdkbin="$(printf '%s\n' "$_msenv" | sed -n 's/^WindowsSdkVerBinPath=//Ip' | head -1)" + # Guard each conversion: only prepend a REAL existing directory. An empty/failed cygpath would + # otherwise inject a bogus relative entry that corrupts the Windows-format PATH the cargo build + # subprocess inherits, which silently breaks other tools' resolution (e.g. cmake "program not found"). + if [ -n "$_vct" ]; then _clb="$(cygpath -u "${_vct}bin\\Hostx64\\x64" 2>/dev/null)"; [ -d "$_clb" ] && PATH="$_clb:$PATH"; fi + if [ -n "$_sdkbin" ]; then _sdb="$(cygpath -u "${_sdkbin}x64" 2>/dev/null)"; [ -d "$_sdb" ] && PATH="$_sdb:$PATH"; fi + export PATH + # Pin cmake explicitly: the cmake crate (core/llama build.rs) resolves cmake via the CMAKE env + # var or PATH, but a manifest-provisioned cmake lives at ~/.continuum/tools/cmake/bin and is not + # guaranteed on the build shell's PATH (measured: absent in a clean subshell -> "cmake not + # found"). Point CMAKE at the known install (same resolution as install-llama-server.sh) and put + # its dir on PATH for cmake's own sub-tools. + _ccmk="$(command -v cmake 2>/dev/null || echo "${CONTINUUM_HOME:-$HOME/.continuum}/tools/cmake/bin/cmake.exe")" + if [ -x "$_ccmk" ]; then + export CMAKE="$(cygpath -w "$_ccmk" 2>/dev/null || echo "$_ccmk")" + PATH="$(dirname "$_ccmk"):$PATH"; export PATH + fi + # Force a generator cmake actually knows. The cmake crate auto-picks the newest installed VS; on a + # VS18-2026 box that is "Visual Studio 18 2026" - a generator cmake 3.30.x does NOT define ("Could + # not create named generator"). Ninja is version-agnostic, uses the MSVC env imported above, and + # matches the llama-server build. [[windows-build-env-drift]] + _cninja="$(command -v ninja 2>/dev/null || echo "${CONTINUUM_HOME:-$HOME/.continuum}/tools/ninja/ninja.exe")" + if [ -x "$_cninja" ]; then + export CMAKE_GENERATOR="Ninja" + # ninja must be ON PATH: the cmake crate ignores CMAKE_MAKE_PROGRAM env, so with -G Ninja it + # searches PATH ("unable to find Ninja / CMAKE_MAKE_PROGRAM is not set" otherwise). + PATH="$(dirname "$_cninja"):$PATH"; export PATH + fi + # CUDA_PATH must be set: cudarc/candle/pocket-tts read it to emit their link-search; without it the + # link has NO CUDA search path (measured: LNK1181 cuda.lib). Point it at a cuda-* whose import-lib + # dir actually has the libs (a provisioning split can leave the crate-detected dir EMPTY - cuda-env + # /Library/lib/x64=0 vs cuda-13.2=12; the #6 provisioning fix unifies them, and this node's cuda-env + # was completed by copying the sibling's libs in). candle finds nvcc via PATH independently, so a + # libs-only CUDA_PATH is fine (build proven). Real fix: provision ONE complete toolkit (#6). + if [ -z "$CUDA_PATH" ]; then + for _cand in "${CONTINUUM_HOME:-$HOME/.continuum}"/cuda-env "${CONTINUUM_HOME:-$HOME/.continuum}"/cuda-*; do + for _sub in Library/lib/x64 lib/x64; do + if [ -f "$_cand/$_sub/curand.lib" ] && [ -f "$_cand/$_sub/cuda.lib" ]; then + export CUDA_PATH="$(cygpath -w "$_cand" 2>/dev/null || echo "$_cand")" + _culibw="$(cygpath -w "$_cand/$_sub" 2>/dev/null || echo "$_cand/$_sub")" + # pocket-tts links cuda.lib but (unlike cudarc) emits no rustc-link-search, relying on the + # linker's own search. rustc links with the newest VS's linker + ITS OWN LIB (not ours), so + # the dir must reach link.exe via RUSTFLAGS -L. RUSTFLAGS env REPLACES (not merges) the + # .cargo/config target rustflags, so re-include +crt-static (the task-#4 /MT fix) or the whole + # GPU stack mislinks LNK2038 MT-vs-MD. + case " $RUSTFLAGS " in *"-L native=${_culibw} "*) : ;; *) export RUSTFLAGS="-C target-feature=+crt-static -L native=${_culibw} ${RUSTFLAGS}" ;; esac + echo "▶ CUDA_PATH=$CUDA_PATH + RUSTFLAGS -L $_culibw (import libs in $_sub)" + break 2 + fi + done + done + [ -z "$CUDA_PATH" ] && echo "⚠ no complete CUDA import-lib dir found (cuda-*/**/{cuda,curand}.lib) - core link WILL fail. Provisioning gap (#6)." >&2 + fi + if command -v cl.exe >/dev/null 2>&1; then + echo "▶ MSVC toolchain imported for the CUDA cargo build (cl.exe on PATH for nvcc/candle)" + else + echo "⚠ MSVC import ran but cl.exe still unresolved — the CUDA cargo build will fail" >&2 + fi + else + echo "✗ Windows+CUDA needs VS2022 (14.4x) C++ x64 toolset for nvcc's host compiler; vswhere/vcvars not found — the core build will fail. Install via the 'msvc' module." >&2 + fi +fi + # ── #296 swept-artifact guard ──────────────────────────────────────── # Cache eviction is a SUPPORTED event (CLAUDE.md disk doctrine): the shared # cargo cache's $PROFILE_LABEL/ dir can be swept while build/ + deps/ survive,