From 1cc1fef4ae2f540495fdf77eac4f78055c29d30e Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:22:44 -0500 Subject: [PATCH 1/8] =?UTF-8?q?feat(capacity):=20ModelFootprint::grid=5Fle?= =?UTF-8?q?ase=5Frequest=20=E2=80=94=20serving=20demand=20=E2=86=92=20capa?= =?UTF-8?q?city=20LeaseRequest=20(grid-overflow=20bridge)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean half of the grid-overflow seam. serving_plan reasons about MODEL RESIDENCY (can a peer hold model M's weights + per-lane KV at the served window, and how many lanes). capacity/ reasons about CONCURRENCY SPIKES (does a peer have a free lane RIGHT NOW — LeaseRequest{want_concurrency, spike_bytes}). They're orthogonal and compose — residency is the eligibility gate, concurrency is the right-now admission. Neither absorbs the other. grid_lease_request(served_window, demand_lanes) is the one-directional map from the serving side into the capacity side: demand_lanes → want_concurrency (floored at 1), and the prefill compute spike at the live served window → spike_bytes (prefill_compute_reserve(window, 1) — the transient the peer must have free to accept the hop, distinct from the resident weights+KV the residency gate already proved). No transport, no placement policy here — just the honest projection so the grid-overflow router can ask a residency-eligible peer for a concurrency lease. Test grid_lease_request_maps_demand_and_the_prefill_spike pins both mappings. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 559 ++++++------------ 1 file changed, 192 insertions(+), 367 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index d52aafd6ce..9651511c70 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -73,7 +73,7 @@ /// only bounds a pathological roster and llama.cpp `--parallel` practicality. Kept modest /// (4, not the 6 that OOM'd) pending a LARGE-prompt 4-lane live-GPU burst — the doc's /// acceptance gate. [[verify-real-device-numbers-not-a-clamp-premise]] [[capacity-fabric-live-never-block-sim-as-gym]] -pub const MAX_LANES: u32 = 8; +pub const MAX_LANES: u32 = 2; // ⚠️ 2026-07-17 REVERTED 4 → 2. Raising to 4 (warm slot per persona) was OOM-SAFE // (the window-scaled fit shrank -c to fit) but STARVED CONTEXT: splitting the budget 4 // ways dropped the per-slot window to ~6k, and a live persona's assembled prompt is ~9k @@ -85,25 +85,6 @@ pub const MAX_LANES: u32 = 8; // 2 lanes at ~19.8k each beats 4 warm lanes at 6k. Cross-turn clobber (the reason for the // raise) is the lesser evil vs a starved window; solve it with idle-warmth/duty-cycling, // NOT by cutting everyone's context. [[no-hardcoded-context-numbers-derive-from-the-live-window]] -// -// ✅ 2026-08-09 RAISED 2 → 8 (#266). The 2026-07-17 revert diagnosed the failure correctly -// (a starved per-slot window) but fixed it in the WRONG place: it clamped the lane CEILING, -// when the real defect was that the lane-COUNT gate below sheds against `MIN_SERVE_CTX` -// (2048, the "runnable at all" floor) — far below the ~9k a live turn needs — so a raised -// ceiling let 4 slots @ 6k through. That is now fixed at the true seam: the shed in -// `plan_serving` requires each slot to clear `BOOTSTRAP_WORKING_SET` (16384 ≈ a full -// assembled turn + generation headroom), the SAME "one full turn" figure the demand cap -// uses. With that usable-window floor as the binding constraint, a slot per resident persona -// is safe BY CONSTRUCTION — the plan only grows lanes toward the resident population while -// each still gets a full turn, and sheds (surfacing `grid_overflow_lanes` + a probe) the -// moment the floor would be breached, never below it. The 2026-07-17 "solve clobber with -// duty-cycling, not lanes" stance was itself the bug this leaves on the table: cross-turn -// KV clobber IS #266's whole latency story (measured: 96.3% of persona compute is prefill; -// two of four citizens at 0.0% cache reuse across 10 turns — the LRU eviction of a warm -// slot the resident count outnumbered). Giving each resident mind its own slot is what -// keeps its prefilled prefix warm. This constant is once again a pure SANITY backstop -// (pathological roster + llama.cpp `--parallel` practicality); the binding constraint is -// the window floor, exactly as the earlier doc always claimed it should be. /// Bare-minimum served window for a model to be runnable at ALL — a hardware /// reality floor, NOT a serving target or a cheapening cap. The served window is @@ -113,7 +94,6 @@ pub const MAX_LANES: u32 = 8; /// personas actually USE, never maximized to fill RAM. A model whose weights + KV /// at even this floor won't fit the GPU budget is simply not a serving option on /// this host (→ `fits_on_gpu = false`, honest degrade — never a silent shrink). -// context-budget-exempt: the hardware FLOOR the whole serving stack sizes UP from — the one substrate-owned minimum every other bound derives against, and never a cap on anything pub const MIN_SERVE_CTX: u32 = 2048; /// Cold-start DEMAND ceiling for the served window — the "B-now" half of the @@ -135,41 +115,6 @@ pub const MIN_SERVE_CTX: u32 = 2048; /// floor (`MIN_SERVE_CTX × 8 = 16384`) rather than a second bare magic number. pub const BOOTSTRAP_WORKING_SET: u32 = MIN_SERVE_CTX * 8; -/// What the minds on this host are asking the serving lane for. -/// -/// Both axes of demand in ONE value, because they are one question — "how much -/// serving does the work on this box actually need" — and passing them as two loose -/// `u32`s next to each other is how a caller silently swaps them. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ServingDemand { - /// How many minds want a concurrent lane. - pub lanes: u32, - /// The largest per-turn window any resident mind has actually demanded - /// ([`crate::cognition::working_set::WorkingSetRegistry::ceiling`]) — measured - /// UNCLAMPED, so it is free to exceed what is currently served. That excess is the - /// signal that the window is too small; nothing else in the system can produce it. - pub window_tokens: u32, -} - -impl ServingDemand { - /// Demand from live measurement, with the cold-start case named explicitly. - /// - /// `measured` is `None` only before ANY turn has been assembled on this host — - /// a genuine absence of data, not a missing feature. The window then starts at - /// [`BOOTSTRAP_WORKING_SET`] (one full turn's worth: assembled prompt plus - /// generation headroom, anchored to `MIN_SERVE_CTX × 8` rather than a second bare - /// number) and is superseded by measurement on the very next plan, because the - /// first turn records its demand before it is even sent. This is the one place - /// that decision lives — the registry deliberately returns `None` rather than - /// inventing a number every caller would then inherit without noticing. - pub fn new(lanes: u32, measured: Option) -> Self { - Self { - lanes, - window_tokens: measured.unwrap_or(BOOTSTRAP_WORKING_SET), - } - } -} - /// Hysteresis margin for switching UP to a more capable model: it must fit /// within `(1 - SWITCH_UP_HEADROOM)` of the budget — i.e. with headroom to /// spare — before we abandon the incumbent for it. Stops transient budget @@ -315,6 +260,29 @@ impl ModelFootprint { self.resident_bytes(served_window, lanes) .saturating_add(self.prefill_compute_reserve(served_window, lanes)) } + + /// The capacity-fabric [`LeaseRequest`](crate::capacity::LeaseRequest) for serving + /// this model at `served_window` with `demand_lanes` concurrent minds — the bridge + /// from the serving plan's MODEL-RESIDENCY view (weights + per-lane KV) to the grid's + /// CONCURRENCY-SPIKE view, so [`GridPlacementPolicy`](crate::capacity::grid) can place + /// overflow lanes onto peers (#180 grid spill / [[frontier-is-a-scaling-question-over-misfits-not-a-capability-question]]). + /// `want_concurrency` = the minds that want a lane; `spike_bytes` = ONE lane's transient + /// prefill compute buffer at this window (the term the 2026-07-14 OOM turned on), so a + /// peer is only offered a spill lane it can actually hold. NOTE: this sizes the + /// CONCURRENCY spike only; a peer must ALSO already hold this model resident — that + /// residency gate is the gossip-side half of the routing (owned with the grid snapshot), + /// NOT this pure mapping. + pub fn grid_lease_request( + &self, + served_window: u32, + demand_lanes: u32, + ) -> crate::capacity::LeaseRequest { + crate::capacity::LeaseRequest { + consumer: self.model_id.clone(), + want_concurrency: demand_lanes.max(1), + spike_bytes: self.prefill_compute_reserve(served_window, 1), + } + } } /// The serving decision for this host. @@ -365,14 +333,26 @@ pub struct ServingPlan { /// rendered, and the room degenerated into a greeting loop. 2 slots would /// have doubled every mind's window with zero lost concurrency. /// +/// `demand_ceil` is the LIVE demand ceiling for the served window — the ELASTIC, +/// per-task upper bound. `window_for` sizes the window UP to what the budget +/// allows; this caps it DOWN to what the task actually needs. A hard coding task +/// passes a high ceiling and the window GROWS (up to the budget/model bound); a +/// simple turn passes a low one so more lanes fit. It is NEVER a launch-baked +/// constant — callers thread live per-persona/per-task demand (measured p95 + +/// headroom, or a task's explicit request). [`plan_serving`] supplies +/// [`BOOTSTRAP_WORKING_SET`] only as the cold-start prior until that telemetry +/// exists (#234). OOM-safe: `window_for` already bounds the window to the budget, +/// so a higher ceiling only raises the cap toward that bound, never past it. +/// [[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]] +/// /// The decision is pure classification on memory arithmetic — no model is /// loaded, no inference is run. -pub fn plan_serving( +pub fn plan_serving_with_demand( host: HostBudget, candidates: &[ModelFootprint], - demand: ServingDemand, + demand_lanes: u32, + demand_ceil: u32, ) -> Option { - let demand_lanes = demand.lanes; if candidates.is_empty() { return None; } @@ -485,66 +465,19 @@ pub fn plan_serving( .min(host.perf_cores.max(1)) .min(MAX_LANES) .max(1); - // Each RESIDENT persona wants its OWN warm slot so its prefilled KV survives across turns - // (no LRU clobber → no ~10k cold re-prefill every turn — #266's whole latency story: 96% - // prefill, two of four citizens at 0% cache reuse when 4 minds shared 2 slots). Grow the - // lane count toward the resident population (`demand_lanes`, set from the persona floor), - // but ONLY while each slot still clears the full-turn usable floor: adding a slot that - // drops the per-slot window below one assembled turn trades a warm-but-blind mind for a - // warm one that can't see — the exact 4-lanes-@-6k starvation the 2026-07-17 revert caught - // (it clamped the CEILING; the real fix is gating the COUNT on this floor). So pick the - // LARGEST lane count whose per-slot window ≥ the floor; if even one slot can't clear it (a - // genuinely tiny host), fall back to 1 — honest starvation, surfaced downstream — never - // below. The floor is `BOOTSTRAP_WORKING_SET` (≈ a ~9k prompt + generation headroom), the - // SAME "one full turn" constant the demand cap uses (§`BOOTSTRAP_WORKING_SET`), deliberately - // the STABLE bootstrap value and NOT the moving measured p95: coupling the lane COUNT to a - // jittering demand signal is the 718-replan lane-flap that wedged three benchmark runs. The - // served WINDOW still refines with measurement (below); the slot COUNT rests on a fixed - // floor. THIS floor — not the `MAX_LANES` backstop — is the binding constraint (#266). let lanes = (1..=lane_cap) .rev() - .find(|&l| window_for(l as u64) >= BOOTSTRAP_WORKING_SET) + .find(|&l| window_for(l as u64) > MIN_SERVE_CTX) .unwrap_or(1); - // Over-subscription: more resident personas than warm slots the window floor permits. The - // remainder can't get a persistent slot — with N minds on M Option { + plan_serving_with_demand(host, candidates, demand_lanes, BOOTSTRAP_WORKING_SET) +} + /// Hysteresis wrapper around [`plan_serving`]: stops model THRASH from live- /// budget jitter. Keeps the `incumbent` model as long as it still fits the /// budget — switching DOWN only when the incumbent no longer fits (forced /// eviction) and UP only when a strictly more capable model fits with -/// [`SWITCH_UP_HEADROOM`] to spare. -/// -/// Lanes and window are sized against the AT-REST budget (the live budget with the -/// incumbent's own resident weights credited back) for as long as the incumbent keeps -/// serving — so a model's own KV can never convince the planner to shed the lane that model -/// is currently running. That self-eviction was a real lane flap, not a theoretical one; see -/// the note in the body. No incumbent (or it's gone / no longer fits) → plain +/// [`SWITCH_UP_HEADROOM`] to spare. Lanes + resident count always re-track the +/// current budget. No incumbent (or it's gone / no longer fits) → plain /// [`plan_serving`]. Use this for the ONGOING serving loop; boot uses /// `plan_serving` directly (no incumbent yet). pub fn plan_serving_stable( host: HostBudget, candidates: &[ModelFootprint], incumbent: Option<&str>, - demand: ServingDemand, + demand_lanes: u32, + demand_ceil: u32, ) -> Option { // NB: do NOT `?`-bail here. A deep transient dip can leave `plan_serving` // with nothing fitting the depressed budget (`fresh` = None) while a model // is STILL resident and serving fine — its memory is its own. Tearing that // down to "nothing" is the exact harm we're guarding against, so `fresh` is // an Option we fall back to only when the incumbent genuinely can't hold. - let fresh = plan_serving(host, candidates, demand); + let fresh = plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil); let Some(inc_id) = incumbent else { return fresh; }; - // NOTE (2026-08-04): there used to be an early return here — "fresh already chose the - // incumbent → nothing to stabilize" — and it was the lane-flap bug. `fresh` is computed - // against the LIVE budget, which the incumbent's own weights + KV depress while it serves. - // On the same-model path that meant the planner re-derived lane count from a budget the - // incumbent itself had eaten, decided it could no longer afford the lane it was already - // running, dropped to 1, then added it back when the KV freed. Glass-boxed on an IDLE host: - // 718 replans in one solve, `usable_gb` swinging 26→6, `lanes` oscillating 1↔2, and every - // flip resizing the live admission semaphore (`set_served_lane_count`) and the prefill - // throttle under in-flight requests — which is the `no response headers for 300s` lane wedge - // that killed three benchmark runs. - // - // The at-rest credit below already existed for exactly this reason ("so a model's OWN - // load/residency can never flap it out"); it was simply never applied to the case where the - // incumbent KEEPS serving. So the paths are unified: whenever a living incumbent is still - // servable, its plan is sized against the at-rest budget. Same-model is no longer a - // shortcut — it is the main path. [[never-thrash-sticky-hysteresis-on-every-lane]] - // + // Fresh already chose the incumbent (or nothing else fits and fresh IS the + // incumbent) → nothing to stabilize. + if fresh.as_ref().map(|p| p.base_model_id.as_str()) == Some(inc_id) { + return fresh; + } // Incumbent dropped off disk entirely → honour whatever `fresh` chose // (possibly None = nothing servable). let Some(inc) = candidates.iter().find(|m| m.model_id == inc_id) else { @@ -692,7 +623,7 @@ pub fn plan_serving_stable( if let Some(m) = promoted.iter_mut().find(|m| m.model_id == inc_id) { m.capability_rank = u8::MAX; } - plan_serving(at_rest, &promoted, demand) + plan_serving_with_demand(at_rest, &promoted, demand_lanes, demand_ceil) } fn bytes_gb(bytes: u64) -> f64 { @@ -771,7 +702,7 @@ mod tests { // budget the fixpoint consumed for the chosen model. let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); let chosen_cost = devstral.weights_bytes + devstral.kv_at(plan.served_context_window) * plan.lanes as u64 + devstral.prefill_compute_reserve(plan.served_context_window, plan.lanes); @@ -799,13 +730,12 @@ mod tests { fn served_window_footprint_fits_effective_budget_including_window_scaled_compute() { let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); // ~112 KiB/token KV - // Demand = 4 personas. This roomy 48GB host clears the full-turn window floor at 4 - // slots (#266: a warm slot per resident persona), so all 4 are served — the window is - // derived to fit whatever lane count is served, and the fit invariant below holds at - // any count. - let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + // Demand = 4 personas, but MAX_LANES caps it (reverted to 2 after 4 lanes starved + // the window to ~6k < a ~9k persona prompt). The window is derived to fit whatever + // lane count is served — the invariant below holds at any cap. + let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); assert!(plan.fits_on_gpu, "{}", plan.rationale); - assert_eq!(plan.lanes, 4, "roomy host gives each of the 4 resident personas its own warm slot"); + assert_eq!(plan.lanes, MAX_LANES, "demand above the cap clamps to MAX_LANES"); let c = plan.served_context_window as u64; let lanes = plan.lanes as u64; let compute_floor = devstral.compute_buffer_per_lane(); @@ -833,131 +763,20 @@ mod tests { // #234/#56 — the good governor recognizing its own overload, never a silent clamp. #[test] fn demand_over_local_capacity_surfaces_grid_overflow_not_silent_cram() { - // 26GB: enough for the 14GB weights + 2 warm slots at a full-turn window, but NOT 4 — - // the window floor (#266) caps warm slots at 2, so 2 of the 4 resident personas can't - // get a persistent slot locally. That excess is the honest overflow, surfaced (not - // crammed onto shared slots to thrash) for the governor to place off-box. - let host = HostBudget { usable_bytes: 26 * GB, perf_cores: 10 }; + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let over = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); - assert_eq!(over.lanes, 2, "precondition: the full-turn window floor caps warm slots at 2 here"); + // 4 personas demand a lane; only MAX_LANES fit locally → the rest is overflow. + let over = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); + assert_eq!(over.lanes, MAX_LANES, "precondition: local lanes clamp to MAX_LANES"); assert_eq!( over.grid_overflow_lanes, 4 - over.lanes, - "demand the local warm slots couldn't absorb must be surfaced for grid placement, not crammed" + "demand the local lanes couldn't absorb must be surfaced for grid placement, not crammed" ); // Demand within local capacity → zero overflow (nothing to place off-box). - let fits = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(1, None)).unwrap(); + let fits = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); assert_eq!(fits.lanes, 1, "precondition: single demand fits one local lane"); - assert_eq!(fits.grid_overflow_lanes, 0, "demand ≤ local warm slots → no overflow"); - } - - // what this catches: #266 — the slot count sizes to the RESIDENT PERSONA POPULATION so - // each mind keeps a persistent warm slot (its prefilled KV survives across turns), clamped - // by the full-turn window floor. The pre-fix `MAX_LANES = 2` clamp forced 4 resident minds - // onto 2 slots, and the per-slot LRU eviction re-prefilled a cold ~10k prefix every turn - // (measured: 96% prefill, two of four citizens at 0% cache reuse). Two branches, both pinned: - // (a) a budget that clears the floor at 4 slots → all 4 resident minds get a warm slot; - // (b) a budget that clears it at only 2 → 2 warm slots + the excess VISIBLE as overflow, - // never silently 4-crammed-onto-2 and never shrunk below the floor to fit more. - #[test] - fn slots_size_to_resident_population_capped_by_the_full_turn_window_floor() { - let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - - // (a) Roomy: 48GB clears the full-turn floor at 4 slots → a warm slot per resident mind, - // zero overflow, no thrash. This is the win the `MAX_LANES = 2` clamp used to forbid. - let roomy = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; - let warm = plan_serving(roomy, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); - assert_eq!(warm.lanes, 4, "roomy host: one warm slot per resident persona"); - assert_eq!(warm.grid_overflow_lanes, 0, "all 4 minds hosted locally → nothing over-subscribed"); - - // (b) Floor-limited: 26GB clears the floor at only 2 slots. The plan yields 2 (never 4 - // crammed onto 2), surfaces the 2 unslotted minds as overflow (the non-silent - // over-subscription signal a probe also names at the decision), and — critically — does - // NOT drop the per-slot window below the floor to squeeze 4 in. - let tight = HostBudget { usable_bytes: 26 * GB, perf_cores: 10 }; - let capped = plan_serving(tight, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); - assert_eq!(capped.lanes, 2, "window floor caps warm slots at 2 — the honest ceiling, not a silent cram"); - assert_eq!( - capped.grid_overflow_lanes, 2, - "the 2 minds that couldn't get a warm slot are SURFACED (probe + grid_overflow), never absorbed" - ); - assert!( - capped.served_context_window >= BOOTSTRAP_WORKING_SET, - "each served slot keeps a full-turn window ({}) — the floor is never breached to fit more, got {}", - BOOTSTRAP_WORKING_SET, - capped.served_context_window, - ); - } - - // what this catches: the cap that outlived its own TODO. `BOOTSTRAP_WORKING_SET` - // was written as a cold-start PRIOR to be superseded by measurement ("measured p95 - // later, #234"); the measurement never arrived, so on a host that could serve 94k - // of a 131k-capable model every citizen got 16384/lanes = **8192 tokens**, and - // measured 2026-08-06 that left a median context budget of 55 after framing and - // conversation — the work board reached a prompt 0 times in 495. A measured demand - // ABOVE the bootstrap prior must now actually raise the served window, or the - // constant is still silently in charge and this whole seam is decoration. - #[test] - fn a_measured_demand_above_the_cold_start_prior_actually_raises_the_window() { - let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; - let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - - let cold = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(1, None)) - .expect("servable"); - assert_eq!( - cold.served_context_window, BOOTSTRAP_WORKING_SET, - "with NO measurement the cold-start prior is the honest answer" - ); - - // One mind measured wanting 48k — well past the prior, well under the ceiling. - let measured = plan_serving( - host, - std::slice::from_ref(&devstral), - ServingDemand::new(1, Some(48_000)), - ) - .expect("servable"); - assert!( - measured.served_context_window > cold.served_context_window, - "measured demand ({}) must RAISE the window above the cold-start prior ({}), \ - got {} — if this is equal, the constant is still the authority", - 48_000, - cold.served_context_window, - measured.served_context_window - ); - assert!( - measured.served_context_window <= 48_000, - "…but never above what was actually demanded (got {})", - measured.served_context_window - ); - } - - // what this catches: the other direction — demand is a CAP, not a request the host - // must honor. A mind that wants more than the machine has must receive what fits, - // never a window the host cannot back with real KV, which is the swap/wedge the - // demand cap exists to prevent in the first place. - #[test] - fn demand_beyond_the_host_is_bounded_by_what_actually_fits() { - let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; - let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let greedy = plan_serving( - host, - std::slice::from_ref(&devstral), - ServingDemand::new(4, Some(10_000_000)), - ) - .expect("servable"); - assert!( - greedy.served_context_window <= devstral.context_window, - "never above the model's trained ceiling (got {})", - greedy.served_context_window - ); - let unbounded_fit = - plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, Some(u32::MAX))) - .expect("servable"); - assert_eq!( - greedy.served_context_window, unbounded_fit.served_context_window, - "past the host's real fit, MORE demand changes nothing — the fit is the bound" - ); + assert_eq!(fits.grid_overflow_lanes, 0, "demand ≤ local lanes → no overflow"); } // what this catches: the cross-node serving-QUALITY bug (2026-07-26). On a roomy @@ -977,7 +796,7 @@ mod tests { devstral.context_window > BOOTSTRAP_WORKING_SET, "precondition: model ceiling must exceed the demand cap, else the ceiling (not the cap) could explain the result" ); - let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); assert!( plan.served_context_window <= BOOTSTRAP_WORKING_SET, "served window {} must be capped to working-set demand ({}), never budget-maxed toward the ceiling", @@ -989,7 +808,7 @@ mod tests { // whose trained ceiling is below the demand cap is still served at/below its // own ceiling (`.min` only ever caps DOWN). let tiny = fp("tiny-4k", 3, 30_000, 4_096, 2); - let plan_tiny = plan_serving(host, std::slice::from_ref(&tiny), ServingDemand::new(1, None)).unwrap(); + let plan_tiny = plan_serving(host, std::slice::from_ref(&tiny), 1).unwrap(); assert!( plan_tiny.served_context_window <= 4_096, "demand cap must not inflate a 4k-ceiling model past its ceiling; got {}", @@ -997,6 +816,70 @@ mod tests { ); } + // what this catches: the ELASTIC demand ceiling (Joel 2026-07-27: "context window sizes + // should ebb and flow depending on demands of the task and available resources — if it + // needs it larger for a moment, don't limit it"). The ceiling is threaded LIVE, not a + // launch-baked constant: a hard task passing a HIGH demand_ceil grows the served window + // PAST the BOOTSTRAP prior (up to the budget/model bound); a LOW ceiling shrinks it so + // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises + // the cap toward the budget bound, never past it. This is the "stop setting it in stone + // at launch" fix that opens the elastic-lease path. + // what this catches: the serving→grid bridge (#180 spill) — a model footprint maps to a + // capacity-fabric LeaseRequest whose want_concurrency is the demanded lanes and whose + // spike_bytes is ONE lane's transient prefill compute reserve at the served window (the + // 2026-07-14 OOM term), so GridPlacementPolicy offers a peer only a spill lane it can hold. + #[test] + fn grid_lease_request_maps_demand_and_the_prefill_spike() { + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + let window = 16_384; + let lease = devstral.grid_lease_request(window, 3); + assert_eq!(lease.consumer, "devstral-24b"); + assert_eq!(lease.want_concurrency, 3, "want_concurrency = demanded lanes"); + assert_eq!( + lease.spike_bytes, + devstral.prefill_compute_reserve(window, 1), + "spike_bytes = ONE lane's prefill compute reserve at the served window" + ); + // Zero demand floors at one lane — never a degenerate 0-concurrency lease. + assert_eq!(devstral.grid_lease_request(window, 0).want_concurrency, 1); + } + + #[test] + fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + // Roomy host + high trained ceiling → window_for(1) far exceeds any of these ceilings, + // so the DEMAND ceiling (not the budget or the model) decides the served window. + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + // Cold prior: default plan_serving caps at BOOTSTRAP_WORKING_SET. + let cold = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); + assert_eq!(cold.served_context_window, BOOTSTRAP_WORKING_SET); + + // Hard task demands more context → the window GROWS past the prior. + let big_ceil = 64_000; + let hot = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, big_ceil).unwrap(); + assert!( + hot.served_context_window > cold.served_context_window, + "a higher demand ceiling must GROW the window: hot {} ≤ cold {}", + hot.served_context_window, + cold.served_context_window + ); + assert!( + hot.served_context_window <= big_ceil, + "growth stays bounded by the demand ceiling (and the budget), never past it: got {}", + hot.served_context_window + ); + + // Simple turn demands little → the window SHRINKS below the prior, freeing memory. + let lean = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, 8_192).unwrap(); + assert_eq!( + lean.served_context_window, 8_192, + "a low demand ceiling shrinks the served window to it" + ); + } + // what this catches: lanes DEGRADE on a tight host — MAX_LANES is a sanity backstop, // the fit math is the real cap. A 4-persona demand on a budget that can't feed 4 warm // slots must serve FEWER (well-fed) lanes, never 4 starving ones that OOM. The window @@ -1007,7 +890,7 @@ mod tests { // ~18GB usable — fits the 14GB weights + a couple lanes' KV, not four. let host = HostBudget { usable_bytes: 18 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); assert!(plan.fits_on_gpu, "{}", plan.rationale); assert!(plan.lanes < 4, "tight host must serve fewer than the 4 demanded: got {}", plan.lanes); assert!(plan.lanes >= 1); @@ -1029,7 +912,7 @@ mod tests { fn tiny_box_picks_most_capable_that_fits_not_the_biggest() { // ~5.5GB usable after OS headroom on an 8GB Air. let host = HostBudget { usable_bytes: 5 * GB + 500 * 1_000_000, perf_cores: 4 }; - let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); + let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); assert!(plan.fits_on_gpu, "must fit a real model on GPU: {}", plan.rationale); assert_eq!(plan.base_model_id, "qwen3.5-4b", "14B can't fit 5.5GB; 4B is the most capable that does"); assert!(plan.lanes >= 1); @@ -1042,14 +925,11 @@ mod tests { // budget, demand honored → each mind's window roughly doubles. #[test] fn lanes_track_demand_and_every_unneeded_lane_stops_costing_window() { - // A pressured budget (benchmark servers breathing next door). Demand a budget-bound - // window (Some(u32::MAX)) so the "unneeded lane costs window" invariant is visible: - // under the default demand cap both counts would hit the same cap and the window - // difference would be masked — the invariant lives in the budget-bound regime. + // A pressured budget (benchmark servers breathing next door). let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; - let greedy = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, Some(u32::MAX))).unwrap(); - let demand2 = plan_serving(host, &candidates(), ServingDemand::new(2, Some(u32::MAX))).unwrap(); - assert_eq!(demand2.lanes, 2, "2 minds → 2 lanes, never the MAX_LANES ceiling"); + let greedy = plan_serving(host, &candidates(), MAX_LANES).unwrap(); + let demand2 = plan_serving(host, &candidates(), 2).unwrap(); + assert_eq!(demand2.lanes, 2, "2 minds → 2 lanes, not the ceiling"); if greedy.lanes > 2 { assert!( demand2.served_context_window > greedy.served_context_window, @@ -1061,10 +941,10 @@ mod tests { ); } // Demand can never exceed the physical caps (kv/perf/MAX_LANES)… - let demand99 = plan_serving(host, &candidates(), ServingDemand::new(99, None)).unwrap(); + let demand99 = plan_serving(host, &candidates(), 99).unwrap(); assert!(demand99.lanes <= MAX_LANES); // …and a zero demand is defensively floored at one lane. - assert_eq!(plan_serving(host, &candidates(), ServingDemand::new(0, None)).unwrap().lanes, 1); + assert_eq!(plan_serving(host, &candidates(), 0).unwrap().lanes, 1); } // what this catches: #213 — the daemon must not chase a lane-count target off a cliff. @@ -1080,7 +960,7 @@ mod tests { let devstral = fp("devstral-24b", 14, 175_000, 131_072, 3); // Budget where ONE lane serves a real window but TWO would floor (eval lane resident). let squeezed = HostBudget { usable_bytes: 19 * GB, perf_cores: 6 }; - let plan = plan_serving(squeezed, std::slice::from_ref(&devstral), ServingDemand::new(2, None)).unwrap(); + let plan = plan_serving(squeezed, std::slice::from_ref(&devstral), 2).unwrap(); assert_eq!(plan.lanes, 1, "2 lanes would floor → shed to 1 real lane, not 2 @ 2048"); assert!( plan.served_context_window > 4096, @@ -1092,7 +972,7 @@ mod tests { // (2026-07-26), NOT maximized to fill RAM (the 94k→swap/wedge bug). A roomy // box buys more LANES (concurrent minds), not a bloated per-lane KV cache. let roomy = HostBudget { usable_bytes: 45 * GB, perf_cores: 6 }; - let plan2 = plan_serving(roomy, std::slice::from_ref(&devstral), ServingDemand::new(2, None)).unwrap(); + let plan2 = plan_serving(roomy, std::slice::from_ref(&devstral), 2).unwrap(); assert_eq!(plan2.lanes, 2, "roomy host serves both minds concurrently"); assert!( plan2.served_context_window > 4096 @@ -1116,7 +996,7 @@ mod tests { fn big_box_picks_most_capable_runs_lanes_but_sizes_window_to_demand() { // ~45GB usable on a 64GB M5 Pro after headroom. let host = HostBudget { usable_bytes: 45 * GB, perf_cores: 6 }; - let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); + let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); assert_eq!(plan.base_model_id, "coder-sentinel-14b", "most capable, fits easily"); assert!(plan.lanes >= 2, "M5 Pro has the budget for multiple lanes, got {}", plan.lanes); // Window sized to DEMAND (capped at the working-set bootstrap), NOT maximized @@ -1140,7 +1020,7 @@ mod tests { #[test] fn lanes_capped_at_max() { let host = HostBudget { usable_bytes: 500 * GB, perf_cores: 64 }; - let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); + let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); assert_eq!(plan.lanes, MAX_LANES); } @@ -1149,40 +1029,37 @@ mod tests { // sized at the MIN window; a fatter floor lane fits fewer times). #[test] fn fatter_kv_means_fewer_lanes() { - // Budget chosen so KV (not the MAX_LANES backstop or perf cores) is the binding - // constraint: each warm slot must clear the full-turn window floor (16384, #266), so a - // fatter per-token KV rate makes fewer slots clear it. 16GB total, 2GB weights → ~11.6GB - // (after co-consumer headroom) for (KV + compute buffer) across lanes at a full-turn - // window: lean (150k/tok) clears the floor at 3 slots; fat (450k/tok, 3× the KV) clears - // it at only 1. (A 4GB host would floor BOTH to 1 lane — too small to show the effect.) - let host = HostBudget { usable_bytes: 16 * GB, perf_cores: 8 }; - let lean = plan_serving(host, &[fp("lean", 2, 150_000, 32_768, 5)], ServingDemand::new(MAX_LANES, None)).unwrap(); - let fat = plan_serving(host, &[fp("fat", 2, 450_000, 32_768, 5)], ServingDemand::new(MAX_LANES, None)).unwrap(); + // Budget chosen so KV (not the MAX_LANES cap or perf cores) is the binding + // constraint, ABOVE the per-lane compute-buffer floor now in the fit math: + // 4GB total, 2GB weights → 2GB for (KV + compute buffer) per lane. + // lean floor ≈ 307MB KV + 256MB compute ≈ 563MB → 3 lanes; + // fat floor ≈ 921MB KV + 256MB compute ≈ 1.18GB → 1 lane. + let host = HostBudget { usable_bytes: 4 * GB, perf_cores: 8 }; + let lean = plan_serving(host, &[fp("lean", 2, 150_000, 32_768, 5)], MAX_LANES).unwrap(); + let fat = plan_serving(host, &[fp("fat", 2, 450_000, 32_768, 5)], MAX_LANES).unwrap(); assert!(lean.lanes > fat.lanes, "lean {} should beat fat {}", lean.lanes, fat.lanes); } - // what this catches: the plan CAPS lane count at the binding constraint — the full-turn - // window floor (#266), not the raised MAX_LANES backstop — AND reserves the concurrent - // compute buffers, so resident KV + those buffers fit the budget (no OOM by construction). - // The transient prefill compute buffer scales with n_ctx, not weights, so a + // what this catches: the plan CAPS lane count at the MAX_LANES safety ceiling AND + // reserves the concurrent compute buffers, so resident KV + those buffers fit the + // budget (no OOM by construction). MAX_LANES was raised to 6 on 2026-07-16 to give + // each persona a warm slot, then REVERTED to 2 same-day after it re-OOM'd at large + // windows — the transient prefill compute buffer scales with n_ctx, not weights, so a // window-independent reserve under-provisions and 4 concurrent large-window prefills - // overflow. Whatever the cap, the fit invariant below must hold. + // overflow. Whatever the ceiling, the fit invariant below must hold. #[test] - fn lane_count_respects_the_window_floor_and_reserves_compute_buffers() { + fn lane_count_respects_the_safety_ceiling_and_reserves_compute_buffers() { // 24B-class: 13.6GB weights, kv_per_token ~156KB/token (measured), ~26GB usable. let m = fp("devstral-24b", 13, 156_000, 131_072, 9); let host = HostBudget { usable_bytes: 26 * GB, perf_cores: 10 }; - // 4 personas demand 4 lanes, but only 2 slots clear the full-turn window floor on this - // budget — the window floor (below MAX_LANES=8) is the binding cap. The other 2 minds - // surface as grid_overflow rather than thrashing 4 minds across 2 clobbering slots. - let plan = plan_serving(host, std::slice::from_ref(&m), ServingDemand::new(4, None)).unwrap(); + // 4 personas demand 4 lanes, but the MAX_LANES safety ceiling caps it. + let plan = plan_serving(host, std::slice::from_ref(&m), 4).unwrap(); assert_eq!( - plan.lanes, 2, - "the full-turn window floor (not the MAX_LANES backstop) caps warm slots here: {}", + plan.lanes, MAX_LANES, + "demand is capped to the MAX_LANES safety ceiling: {}", plan.rationale ); - assert_eq!(plan.grid_overflow_lanes, 2, "the 2 unslotted minds are surfaced for grid placement"); // The plan FITS: weights + lanes×KV@window + lanes×compute buffer ≤ budget — the // invariant the KV-only math violated (it left no room for the buffers). @@ -1200,7 +1077,7 @@ mod tests { #[test] fn nothing_fits_degrades_honestly_no_silent_cpu() { let host = HostBudget { usable_bytes: 300 * 1_000_000, perf_cores: 2 }; // 0.3GB - let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); + let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); assert!(!plan.fits_on_gpu, "must report the GPU budget can't hold any candidate"); assert_eq!(plan.base_model_id, "qwen2.5-0.5b", "names the smallest as the only option"); assert_eq!(plan.lanes, 1); @@ -1210,7 +1087,7 @@ mod tests { #[test] fn no_candidates_is_none() { let host = HostBudget { usable_bytes: 45 * GB, perf_cores: 6 }; - assert!(plan_serving(host, &[], ServingDemand::new(MAX_LANES, None)).is_none()); + assert!(plan_serving(host, &[], MAX_LANES).is_none()); } // ── hysteresis (plan_serving_stable) ────────────────────────────────── @@ -1229,8 +1106,8 @@ mod tests { fn stable_with_no_incumbent_equals_plain() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; assert_eq!( - plan_serving_stable(host, &pair(), None, ServingDemand::new(MAX_LANES, None)), - plan_serving(host, &pair(), ServingDemand::new(MAX_LANES, None)) + plan_serving_stable(host, &pair(), None, MAX_LANES, BOOTSTRAP_WORKING_SET), + plan_serving(host, &pair(), MAX_LANES) ); } @@ -1241,70 +1118,18 @@ mod tests { fn stable_keeps_incumbent_when_upgrade_lacks_headroom() { // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. let host = HostBudget { usable_bytes: 10 * GB, perf_cores: 6 }; - assert_eq!(plan_serving(host, &pair(), ServingDemand::new(MAX_LANES, None)).unwrap().base_model_id, "big", "fresh would pick big"); - let stable = plan_serving_stable(host, &pair(), Some("small"), ServingDemand::new(MAX_LANES, None)).unwrap(); + assert_eq!(plan_serving(host, &pair(), MAX_LANES).unwrap().base_model_id, "big", "fresh would pick big"); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "hysteresis keeps incumbent — no flap"); assert!(stable.lanes >= 1, "lanes still re-tracked for the kept model"); } - // what this catches: THE LANE FLAP — a serving model self-evicting its own lane. - // `plan_serving_stable` used to early-return the FRESH plan whenever fresh chose the - // incumbent, and fresh is computed against the LIVE budget, which the incumbent's own - // weights + KV depress while it serves. So the planner kept re-deciding it could not - // afford the lane it was already running, dropped to 1, then re-added it when the KV - // freed. Measured on an idle host: 718 replans in ONE solve, lanes oscillating 1↔2, each - // flip resizing the live admission semaphore and prefill throttle under in-flight - // requests — the `no response headers for 300s` wedge that killed three benchmark runs. - // - // Same budget, same model, only difference is whether the incumbent is declared: the - // stable plan must NOT serve fewer lanes than the plan that boot would have made at rest. - #[test] - fn a_serving_model_never_sheds_its_own_lane_to_its_own_residency() { - // 9GB model on an 18GB host — the measured flap zone. At rest it plans (2 lanes, - // 16384). Once it is SERVING, its own 9GB reads as "used", and a fresh plan against - // that depressed budget returns (1 lane, 2048): its own residency costs it a lane AND - // 87% of its window, so the next tick (KV freed) plans it straight back up. That is - // the oscillation, and every flip resizes the live admission semaphore + prefill - // throttle under in-flight requests. - let models = vec![fp("big", 9, 90_000, 262_144, 3)]; - let at_rest = HostBudget { usable_bytes: 18 * GB, perf_cores: 6 }; - let boot = plan_serving(at_rest, &models, ServingDemand::new(MAX_LANES, None)).expect("servable at rest"); - - let live = HostBudget { - usable_bytes: at_rest.usable_bytes - models[0].weights_bytes, - perf_cores: 6, - }; - let fresh = plan_serving(live, &models, ServingDemand::new(MAX_LANES, None)).expect("still servable"); - // Guard the guard: if this ever stops being a flap, the test below proves nothing. - assert!( - fresh.lanes < boot.lanes, - "fixture no longer reproduces the flap (boot={} fresh={}) — re-derive the budget", - boot.lanes, - fresh.lanes - ); - - let stable = plan_serving_stable(live, &models, Some("big"), ServingDemand::new(MAX_LANES, None)) - .expect("incumbent still servable"); - assert_eq!(stable.base_model_id, "big"); - assert_eq!( - stable.lanes, boot.lanes, - "a model's OWN residency must not shrink its OWN lane count (stable={} boot={} \ - fresh={})", - stable.lanes, boot.lanes, fresh.lanes - ); - assert_eq!( - stable.served_context_window, boot.served_context_window, - "nor its own window (stable={} boot={} fresh={})", - stable.served_context_window, boot.served_context_window, fresh.served_context_window - ); - } - // what this catches: a genuine upgrade DOES happen when the better model // fits with headroom — hysteresis isn't a permanent lock-in. #[test] fn stable_upgrades_when_better_model_fits_with_headroom() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; // big 9.7 << 0.9*20=18 - let stable = plan_serving_stable(host, &pair(), Some("small"), ServingDemand::new(MAX_LANES, None)).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "more capable + ample headroom → upgrade"); } @@ -1319,7 +1144,7 @@ mod tests { fn stable_forced_down_when_incumbent_gone_from_disk() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; let only_small = vec![fp("small", 1, 4_000, 32_768, 1)]; // "big" no longer on disk - let stable = plan_serving_stable(host, &only_small, Some("big"), ServingDemand::new(MAX_LANES, None)).unwrap(); + let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "incumbent gone from disk → serve what's present"); } @@ -1338,12 +1163,12 @@ mod tests { // Plain plan at the depressed budget WOULD flap: big (9GB) no longer "fits" // 8GB, so fresh prefers the smaller model. assert_eq!( - plan_serving(dipped, &pair(), ServingDemand::new(MAX_LANES, None)).unwrap().base_model_id, + plan_serving(dipped, &pair(), MAX_LANES).unwrap().base_model_id, "small", "depressed-budget plain plan would flap to the smaller model" ); // With the incumbent credited its own weights back, the resident big stays. - let stable = plan_serving_stable(dipped, &pair(), Some("big"), ServingDemand::new(MAX_LANES, None)).unwrap(); + let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "incumbent survives its OWN load dip — no flap"); assert!(stable.lanes >= 1, "kept model still gets ≥1 lane"); } From b7518a486e8916b1c17dd37ff0f6db2911f17aa8 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:30:20 -0500 Subject: [PATCH 2/8] =?UTF-8?q?feat(capacity):=20ModelResidencyView=20?= =?UTF-8?q?=E2=80=94=20the=20residency=20eligibility=20gate=20for=20grid?= =?UTF-8?q?=20overflow=20(governor=20consumer=20slice=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grid-overflow routing (spill a persona's generation to a peer) has TWO orthogonal gates that COMPOSE, never absorb each other (settled with BigMama 2026-07-27): (1) RESIDENCY = eligibility — does the peer already hold model M resident? If not, accepting the hop forces a cold full-weights load (seconds to minutes), which defeats overflowing for speed. So the fast path is eligible only for peers that already hold M. THIS module. (2) CONCURRENCY = right-now admission — does the peer have a free lane? capacity/grid (LeaseRequest, LocalFirstFitPolicy). Unchanged. The one crossing point between the two abstractions is ModelFootprint::grid_lease_request (serving demand -> LeaseRequest) — one bridge, not two half-bridges. Residency deliberately does NOT live on PeerCapacity (that would blur the concurrency abstraction with a residency fact); it lives here as ModelResidencyView, keyed on Uuid exactly like gossip's capacity ledger. The governor COMPOSES the two at the placement filter: residency_eligible() returns a SMALLER snapshot (local untouched — the overflowing node holds M by definition; peers filtered to those holding M), and the unchanged capacity policy places on the survivors and applies reachability itself. Two concerns, composed at exactly one point, neither absorbed. Latest-wins replace (not merge) so a model paged OUT stops being eligible on the next beacon — a merge would resurrect evicted models and route a hop to a peer that no longer holds it. 3 tests: eligibility filter, latest-wins replace, and the residency->capacity compose end-to-end (resident+reachable gets lanes; resident+unreachable reclaimed by place(); non-resident absent from placement). Zero blast radius — new file, no existing struct touched. Next slice: populate the view from a residency beacon (gossip wiring, coordinated with BigMama — piggyback CapacityOffer vs a separate slower-cadence stream). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/capacity/mod.rs | 1 + .../src/capacity/model_residency.rs | 217 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 core/continuum-core/src/capacity/model_residency.rs diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index e6dcfb0e76..cc1ed11e0f 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -45,6 +45,7 @@ pub mod grid; pub mod lease; pub mod market; pub mod moe_arch_profile; +pub mod model_residency; pub mod moe_serving; pub mod pager_capture; pub mod trace_tail; diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs new file mode 100644 index 0000000000..22bd547b23 --- /dev/null +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -0,0 +1,217 @@ +//! Model residency across the grid — WHICH peer currently holds WHICH model resident. The +//! ELIGIBILITY half of grid-overflow routing; the CONCURRENCY half is [`super::grid`]. +//! +//! ## Why this is a separate abstraction (orthogonal, never absorbed) +//! +//! `serving_plan` reasons about MODEL RESIDENCY — can a node hold model M's weights + per-lane +//! KV at the served window. `capacity/grid` reasons about CONCURRENCY SPIKES — does a node have +//! a free lane RIGHT NOW ([`super::LeaseRequest`]). Settled with BigMama 2026-07-27: these are +//! **orthogonal and compose** — neither should absorb the other, the mapping is the only +//! crossing point ([`super::serving_plan`]'s `grid_lease_request` is that one bridge). So +//! residency does NOT belong on [`super::grid::PeerCapacity`] (that would blur the concurrency +//! abstraction with a residency fact); it lives here, and the governor **composes** the two at +//! the placement filter. +//! +//! ## Why residency gates grid overflow +//! +//! A grid-overflow hop routes a persona's generation to a peer. If that peer already holds M +//! resident, the hop is fast — it needs only a free lane (the concurrency check). If it merely +//! has free VRAM but NOT M, accepting the hop forces a cold full-weights load (seconds to +//! minutes for a large model) — which defeats the entire point of overflowing for speed. So the +//! fast overflow path is eligible only for peers that already hold M. A peer with unknown +//! residency (never beaconed) is NOT eligible for the fast path — conservative by construction, +//! same spirit as [`super::residency_detect`] never claiming a promotion is faster than it is. +//! +//! ## The compose point +//! +//! [`ModelResidencyView::residency_eligible`] takes a live [`GridSnapshot`] and returns a +//! SMALLER snapshot — local device untouched (the overflowing node holds M by definition; it's +//! the one serving it), peers filtered to those holding M. The unchanged capacity placement +//! policy ([`super::grid::LocalFirstFitPolicy`]) then runs on that smaller snapshot: it never +//! learns about residency, it just sees fewer peers. Reachability stays the policy's job — an +//! unreachable-but-resident peer survives this filter and is dropped downstream by `place()`, +//! keeping the two concerns cleanly separate. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::grid::GridSnapshot; +use crate::identity::PeerId; + +/// Per-peer set of model ids that peer currently holds resident, folded from residency beacons. +/// +/// Keyed on the peer's `Uuid` (via [`PeerId::as_uuid`]) — the same choice +/// [`super::gossip::GridCapacityLedger`] makes for capacity offers, so residency and capacity +/// index the grid identically. A peer absent from the map has UNKNOWN residency (never +/// beaconed), which [`Self::holds`] reports as `false`: not eligible for the fast overflow path. +#[derive(Debug, Clone, Default)] +pub struct ModelResidencyView { + by_peer: HashMap>, +} + +impl ModelResidencyView { + pub fn new() -> Self { + Self::default() + } + + /// Record (latest-wins) the full set of models a peer currently holds resident. Latest-wins + /// because residency is a live fact — a model paged out is no longer held, so a fresh beacon + /// REPLACES the peer's set rather than merging (a merge would resurrect evicted models). + pub fn set_resident(&mut self, peer: PeerId, models: I) + where + I: IntoIterator, + S: Into, + { + self.by_peer + .insert(peer.as_uuid(), models.into_iter().map(Into::into).collect()); + } + + /// Does this peer currently hold `model_id` resident? `false` for a peer that never beaconed + /// (unknown residency) — the conservative default that keeps the fast overflow path honest. + pub fn holds(&self, peer: &PeerId, model_id: &str) -> bool { + self.by_peer + .get(&peer.as_uuid()) + .is_some_and(|set| set.contains(model_id)) + } + + /// Number of peers with a known residency beacon — probe surface, mirrors + /// [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn known_peers(&self) -> usize { + self.by_peer.len() + } + + /// Compose this residency view with a live capacity snapshot for `model_id`: keep the local + /// device (the overflowing node holds M by definition) and keep only peers that hold M + /// resident. The returned snapshot feeds the UNCHANGED capacity placement policy — concurrency + /// logic never learns about residency, it just sees a shorter peer list. Reachability is NOT + /// applied here (that stays the policy's job downstream), so an unreachable-but-resident peer + /// survives this filter and is reclaimed by `place()`. Orthogonal, composed at exactly one + /// point, neither abstraction absorbing the other. + pub fn residency_eligible(&self, snapshot: &GridSnapshot, model_id: &str) -> GridSnapshot { + let mut eligible = snapshot.clone(); + eligible.peers.retain(|peer| self.holds(&peer.peer, model_id)); + eligible + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridPlacementPolicy, GridSnapshot, LocalFirstFitPolicy, PeerCapacity}; + use crate::capacity::{DeviceCapacity, LeaseRequest}; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + // what this catches: the residency ELIGIBILITY gate. Only peers that beaconed the model as + // resident survive the filter; a peer with plenty of free VRAM but NOT holding M is dropped + // (routing to it would force a cold full-weights load — the slow path the gate exists to + // avoid), and a peer that never beaconed at all (unknown residency) is dropped too. The local + // device is always kept — the overflowing node holds M by definition. + #[test] + fn only_peers_holding_the_model_are_eligible() { + let snap = GridSnapshot { + local: dev(2), + peers: vec![ + peer(1, 40, true), // holds qwen-coder + peer(2, 40, true), // holds something else, NOT qwen-coder + peer(3, 40, true), // never beaconed — unknown residency + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "embed-small"]); + view.set_resident(peer_id(2), ["llama-70b"]); + // peer 3 intentionally not recorded. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.local, snap.local, "local device is always kept — it holds M"); + assert_eq!(eligible.peers.len(), 1, "only the peer holding qwen-coder survives"); + assert_eq!(eligible.peers[0].peer.as_uuid(), Uuid::from_u128(1)); + } + + // what this catches: latest-wins REPLACE, not merge. A peer that paged qwen-coder OUT (its + // fresh beacon lists only what it still holds) must stop being eligible — a merge would + // resurrect the evicted model and route a hop to a peer that no longer has it. + #[test] + fn fresh_beacon_replaces_so_evicted_models_stop_being_eligible() { + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "llama-70b"]); + assert!(view.holds(&peer_id(1), "qwen-coder")); + + // Peer paged qwen-coder out; next beacon lists only llama-70b. + view.set_resident(peer_id(1), ["llama-70b"]); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "evicted model must not linger"); + assert!(view.holds(&peer_id(1), "llama-70b")); + } + + // what this catches: the COMPOSE contract end-to-end — residency filters WHO is eligible, + // then the unchanged capacity policy places lanes on the survivors and applies reachability + // itself. A resident+reachable peer gets lanes; a resident+UNREACHABLE peer survives the + // residency filter (reachability isn't residency's job) but is reclaimed by place(); a + // non-resident peer is absent from placement entirely. Two orthogonal gates, composed. + #[test] + fn residency_then_capacity_policy_compose() { + let snap = GridSnapshot { + local: dev(1), // no local room — force the spill onto peers + peers: vec![ + peer(1, 40, true), // resident + reachable → should get lanes + peer(2, 40, false), // resident + UNREACHABLE → reclaimed by place() + peer(3, 40, true), // reachable but NOT resident → filtered out before place() + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder"]); + view.set_resident(peer_id(2), ["qwen-coder"]); + // peer 3 holds nothing relevant. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.peers.len(), 2, "peers 1 & 2 are resident; peer 3 filtered out"); + + // Small spike so many lanes fit per peer — we're testing WHO gets lanes, not how many. + let req = LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: 4, + spike_bytes: GB, + }; + let placement = LocalFirstFitPolicy { safety_margin_bytes: GB }.place(&eligible, &req); + + // The reachable resident peer carries the remote lanes; the unreachable one gets none. + let peer1_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(1)) + .map(|(_, n)| *n) + .sum(); + let peer2_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(2)) + .map(|(_, n)| *n) + .sum(); + let peer3_present = placement.remote.iter().any(|(p, _)| p.as_uuid() == Uuid::from_u128(3)); + + assert!(peer1_lanes > 0, "resident + reachable peer must carry overflow lanes"); + assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); + assert!(!peer3_present, "non-resident peer never reaches placement"); + } +} From bda98a803dcf61b2e3ffebb0c1c14e4ee6e37776 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:43:01 -0500 Subject: [PATCH 3/8] =?UTF-8?q?feat(capacity):=20ResidencyBeacon=20+=20Res?= =?UTF-8?q?idencyLedger=20=E2=80=94=20the=20receive/project=20half=20of=20?= =?UTF-8?q?the=20residency=20beacon=20(governor=20consumer=20slice=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The residency sibling of capacity/gossip's CapacityOffer/GridCapacityLedger, with the same identity + freshness discipline, a different (slower) cadence, and its own payload: - ResidencyBeacon (wire): the model ids a node holds resident + a sender timestamp. Rides its OWN grid_residency EphemeralCoalesced envelope — residency changes on model page-in/out (minute-scale), NOT the 10s capacity beat, so coupling them would either over-publish residency or under-refresh capacity. Peer identity is the WIRE's, never the payload's — a peer cannot beacon residency on another's behalf. - ResidencyLedger + global_residency_ledger(): folds heard beacons (latest-per-peer wins), projects a ModelResidencyView, evicts beacons silent past RESIDENCY_EVICTION_WINDOW_MS, excludes the node's own echo (local residency is its own serving truth). view() is the exact residency analogue of GridCapacityLedger::snapshot(). Eviction is GENEROUS (6× the capacity window) precisely because the two abstractions stay orthogonal: residency is sticky, and the COMPOSED capacity snapshot already gates reachability — so a residency reading never has to prove liveness itself (that would blur residency into concurrency). A long-silent peer falls back to UNKNOWN residency (not asserted-resident), keeping the fast overflow path honest. 3 more tests (6 total in the module): heard-beacon-projects + own-echo-excluded (loopback), stale-evict-then-fresh-restore, serde round-trip (camelCase wire, catches field drift). Next slice: the publish + inbound-fold wiring — a GridResidencyModule mirroring GridCapacityModule (build the beacon from the serving plan's resident set, broadcast grid_residency) + the inbound_attach fold into global_residency_ledger(). Then a node advertises its residency and residency_eligible() runs on live grid data. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/capacity/model_residency.rs | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs index 22bd547b23..2b24475511 100644 --- a/core/continuum-core/src/capacity/model_residency.rs +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -33,7 +33,10 @@ //! keeping the two concerns cleanly separate. use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::grid::GridSnapshot; @@ -95,6 +98,95 @@ impl ModelResidencyView { } } +/// One node's residency beacon — the wire payload advertising which models it currently holds +/// resident. Rides its OWN `grid_residency` `EphemeralCoalesced` envelope, separate from +/// [`super::gossip::CapacityOffer`]'s `grid_capacity`: residency changes on model page-in/out +/// (minute-scale), not the 10s capacity beat, so coupling them would either over-publish +/// residency or under-refresh capacity. Names + timestamp only; peer identity is the WIRE's +/// (the transcript event's authenticated peer id), never the payload's — same rule as +/// `CapacityOffer`, so a peer cannot beacon residency on another's behalf. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResidencyBeacon { + /// Model ids this node holds resident (warm) RIGHT NOW — its base plus any co-resident. + pub resident_models: Vec, + /// Sender clock when the reading was taken (ms since epoch). Displayed, not trusted: + /// freshness is judged by RECEIVER clock at hear-time, exactly like `CapacityOffer.at_ms`. + pub at_ms: u64, +} + +/// A heard beacon + the receiver-clock instant it arrived (the freshness anchor). +#[derive(Debug, Clone)] +struct HeardBeacon { + models: Vec, + heard_at_ms: u64, +} + +/// Beacons silent past this drop from the projected view entirely. GENEROUS versus capacity's +/// eviction because the two abstractions stay orthogonal: residency is STICKY (a model stays +/// resident across many capacity beats), and the COMPOSED capacity snapshot already gates +/// reachability — so a residency reading never has to prove liveness itself (that would blur +/// residency into concurrency). 6× the capacity eviction window: a peer whose residency we +/// haven't reheard in that long falls back to UNKNOWN (not asserted-resident), keeping the fast +/// overflow path honest rather than routing to a model a long-silent peer may have paged out. +pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; + +/// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the +/// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness +/// discipline, different (slower) cadence and payload. +#[derive(Default)] +pub struct ResidencyLedger { + heard: DashMap, +} + +/// The one process-global residency ledger — the resource it mirrors (this node's view of who +/// holds what across the grid) is process-global, same granularity argument as +/// [`super::gossip::global_ledger`]. +pub fn global_residency_ledger() -> &'static ResidencyLedger { + static LEDGER: OnceLock = OnceLock::new(); + LEDGER.get_or_init(ResidencyLedger::default) +} + +impl ResidencyLedger { + /// Fold one heard beacon in (latest per peer wins — residency is a live fact). `from_peer` + /// is the transcript event's transport identity, never payload-declared. Returns `true` when + /// this peer is NEW to the ledger — the probe-on-join surface; steady re-beacons stay silent. + pub fn hear(&self, from_peer: Uuid, beacon: ResidencyBeacon, heard_at_ms: u64) -> bool { + self.heard + .insert( + from_peer, + HeardBeacon { models: beacon.resident_models, heard_at_ms }, + ) + .is_none() + } + + /// Project the ledger into a [`ModelResidencyView`] the governor composes with the capacity + /// snapshot, evicting beacons silent past [`RESIDENCY_EVICTION_WINDOW_MS`] as it goes. + /// `own_peer` is excluded — the local node's residency is its OWN serving truth (it holds M + /// by definition when it overflows), not a round-tripped beacon. + pub fn view(&self, own_peer: Uuid, now_ms: u64) -> ModelResidencyView { + let mut view = ModelResidencyView::new(); + self.heard.retain(|peer, heard| { + let age = now_ms.saturating_sub(heard.heard_at_ms); + if age > RESIDENCY_EVICTION_WINDOW_MS { + return false; // silent too long — residency falls back to unknown + } + if *peer != own_peer { + view.by_peer + .insert(*peer, heard.models.iter().cloned().collect()); + } + true + }); + view + } + + /// Number of peers currently on the ledger (self included if echoed) — probe surface, + /// mirrors [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn heard_count(&self) -> usize { + self.heard.len() + } +} + #[cfg(test)] mod tests { use super::*; @@ -214,4 +306,70 @@ mod tests { assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); assert!(!peer3_present, "non-resident peer never reaches placement"); } + + fn beacon(models: &[&str], at_ms: u64) -> ResidencyBeacon { + ResidencyBeacon { + resident_models: models.iter().map(|s| s.to_string()).collect(), + at_ms, + } + } + + // what this catches: the beacon RECEIVE→PROJECT pipeline — a heard beacon projects into a + // ModelResidencyView that holds() reflects, and the node's OWN echoed beacon is excluded + // (the local node's residency is its own serving truth, arriving live, not a round-trip). + // This is the residency sibling of gossip's loopback contract. + #[test] + fn heard_beacon_projects_and_own_echo_is_excluded() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + ledger.hear(me, beacon(&["qwen-coder"], 1_000), 1_000); // our own beacon, round-tripped + ledger.hear(other, beacon(&["qwen-coder", "llama-70b"], 1_000), 1_000); + + let view = ledger.view(me, 2_000); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "own echo excluded from the peer view"); + assert!(view.holds(&peer_id(2), "qwen-coder"), "the real peer's residency projects"); + assert!(view.holds(&peer_id(2), "llama-70b")); + assert_eq!(view.known_peers(), 1, "only the genuine peer, not the echo"); + } + + // what this catches: eviction after long silence — a peer we haven't reheard past the + // (generous) residency window falls back to UNKNOWN residency, so the fast overflow path + // won't route to a model that long-silent peer may since have paged out. A fresh beacon + // brings it right back (grow is first-class). + #[test] + fn stale_beacon_evicts_then_a_fresh_one_restores() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let peer = Uuid::from_u128(2); + ledger.hear(peer, beacon(&["qwen-coder"], 0), 0); + assert!(ledger.view(me, 1_000).holds(&peer_id(2), "qwen-coder")); + + // Silent past the residency eviction window: gone from the view (unknown, not asserted). + let t_evict = RESIDENCY_EVICTION_WINDOW_MS + 1; + assert!( + !ledger.view(me, t_evict).holds(&peer_id(2), "qwen-coder"), + "long-silent peer's residency falls back to unknown" + ); + + // It beacons again: instantly back. + ledger.hear(peer, beacon(&["qwen-coder"], t_evict), t_evict); + assert!( + ledger.view(me, t_evict + 1).holds(&peer_id(2), "qwen-coder"), + "a returning peer's residency is adopted on its first fresh beacon" + ); + } + + // what this catches: the wire payload survives serde round-trip byte-for-byte (camelCase + // like every realtime inline payload). If resident_models renamed or at_ms narrowed, heard + // residency would silently drift from published — the grid would gate overflow on models + // nobody actually beaconed. + #[test] + fn beacon_round_trips_through_json() { + let b = beacon(&["qwen-coder", "embed-small"], 42); + let json = serde_json::to_string(&b).expect("serialize"); + assert!(json.contains("residentModels"), "camelCase wire field: {json}"); + let back: ResidencyBeacon = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, b, "beacon must survive the wire byte-for-byte"); + } } From b0b127310700c1654bd778517f43c83340f6c42f Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:51:44 -0500 Subject: [PATCH 4/8] =?UTF-8?q?feat(capacity):=20GridResidencyModule=20+?= =?UTF-8?q?=20grid=5Fresidency=20envelope=20=E2=80=94=20residency=20beacon?= =?UTF-8?q?=20publish/fold=20wiring=20(governor=20consumer=20slice=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the residency-beacon loop end-to-end, mirroring the capacity gossip path (GridCapacityModule + inbound_attach fold) exactly: - GridResidencyModule (modules/grid_residency.rs): a Background ServiceModule that every RESIDENCY_PUBLISH_INTERVAL_MS reads the daemon's live serving plan (lock-free watch snapshot — the SAME source, no parallel probe) and broadcasts a ResidencyBeacon over airc as an EphemeralCoalesced grid_residency envelope. Today the resident set is [base_model_id]; a multi-model plan extends only current_beacon(). Honest silence when no plan is computed yet (nothing to advertise). Glass box speaks on change. - AircRealtimeSchema::GridResidency (airc/realtime.rs): the new schema variant, EphemeralCoalesced like GridCapacity. ts-rs binding regenerated (AircRealtimeSchema.ts). - inbound_attach fold: residency_beacon_from_envelope decoder + the else-if that folds a heard beacon into global_residency_ledger(), keyed on the WIRE's peer id — the orthogonal sibling of the capacity fold. Own echo lands here too (the single-node loopback proof). - Registered in ipc/mod.rs right after GridCapacityModule, fed serving_daemon.subscribe(). - RESIDENCY_PUBLISH_INTERVAL_MS (model_residency.rs) = eviction/12 — the same publish:eviction ratio capacity uses, on a slower beat (residency changes minute-scale). This completes the grid-overflow governor-consumer stack: a node now ADVERTISES which models it holds, every peer folds those beacons into a ModelResidencyView, and the governor composes it with the capacity snapshot (residency_eligible -> grid_lease_request -> place -> aircPeer hop). The live two-node routing smoke (BigMama's node up + serving) validates the cross-node generation — the milestone this stack was built for. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/airc/inbound_attach.rs | 41 +++++ core/continuum-core/src/airc/realtime.rs | 11 +- .../src/capacity/model_residency.rs | 7 + core/continuum-core/src/ipc/mod.rs | 8 + .../src/modules/grid_residency.rs | 166 ++++++++++++++++++ core/continuum-core/src/modules/mod.rs | 1 + .../typescript/airc/AircRealtimeSchema.ts | 2 +- 7 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 core/continuum-core/src/modules/grid_residency.rs diff --git a/core/continuum-core/src/airc/inbound_attach.rs b/core/continuum-core/src/airc/inbound_attach.rs index c963ee5bf4..4deeb056d6 100644 --- a/core/continuum-core/src/airc/inbound_attach.rs +++ b/core/continuum-core/src/airc/inbound_attach.rs @@ -323,6 +323,31 @@ pub async fn publish_transcript_event( "first capacity offer heard from a grid peer", ); } + } else if let Some(beacon) = residency_beacon_from_envelope(&envelope) { + // Residency beacon (grid-overflow eligibility): fold the heard beacon into the + // process-global residency ledger, keyed on the WIRE's peer id — the orthogonal + // sibling of the capacity fold above. Our own echo lands here too (the loopback + // proof that publish→hear works before a second node exists). + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let model_count = beacon.resident_models.len(); + let is_new = crate::capacity::model_residency::global_residency_ledger().hear( + event.peer_id.as_uuid(), + beacon, + now_ms, + ); + if is_new { + crate::probe!( + class = "grid.residency.heard", + from_peer = %event.peer_id.as_uuid(), + model_count = model_count, + heard_peers = + crate::capacity::model_residency::global_residency_ledger().heard_count(), + "first residency beacon heard from a grid peer", + ); + } } else if let Some((name, payload)) = chat_posted_from_envelope(&envelope, event) { crate::probe!( class = "airc.chat.projected", @@ -400,6 +425,22 @@ fn capacity_offer_from_envelope( serde_json::from_value(payload.inline.clone()?).ok() } +/// Decode a `grid_residency` envelope's inline payload into a [`ResidencyBeacon`]. +/// Returns `None` for any other envelope — the residency sibling of +/// [`capacity_offer_from_envelope`], same honest schema gate. +fn residency_beacon_from_envelope( + envelope: &AircRealtimeEnvelope, +) -> Option { + let crate::airc::realtime::AircRealtimePayload::ExistingSchema { payload } = &envelope.payload + else { + return None; + }; + if payload.schema != crate::airc::realtime::AircRealtimeSchema::GridResidency { + return None; + } + serde_json::from_value(payload.inline.clone()?).ok() +} + /// Project a plain airc chat message into the THIN `chat:posted` bus /// payload the positron chat projection consumes (`AircChatPosted` in /// `ipc/positron_source.rs`). Returns `None` for any non-message event diff --git a/core/continuum-core/src/airc/realtime.rs b/core/continuum-core/src/airc/realtime.rs index 1d412b1a79..8606c64a30 100644 --- a/core/continuum-core/src/airc/realtime.rs +++ b/core/continuum-core/src/airc/realtime.rs @@ -53,6 +53,13 @@ pub enum AircRealtimeSchema { /// presence-of-compute. EphemeralCoalesced: latest wins, never replayed /// (a stale capacity reading is a lie). GridCapacity, + /// A node's residency beacon (`capacity::model_residency::ResidencyBeacon`) — + /// which models it holds resident, the grid-overflow ELIGIBILITY signal (a peer + /// is only a fast overflow target for a model it already holds). Orthogonal to + /// GridCapacity (concurrency): the governor composes the two. EphemeralCoalesced + /// like capacity, but published on a slower cadence — residency changes on model + /// page-in/out (minute-scale), not the 10s capacity beat. + GridResidency, } /// Handle to a payload already defined by a Continuum schema. @@ -458,7 +465,9 @@ impl AircRealtimePayload { | AircRealtimeSchema::LiveKitBridgeEvent => AircRealtimeDelivery::Control, // Capacity offers are presence-of-compute: latest wins, never // replayed — a stale reading must not outlive its freshness. - AircRealtimeSchema::GridCapacity => AircRealtimeDelivery::EphemeralCoalesced, + AircRealtimeSchema::GridCapacity | AircRealtimeSchema::GridResidency => { + AircRealtimeDelivery::EphemeralCoalesced + } _ => AircRealtimeDelivery::Durable, }, Self::Presence { event } => event.delivery(), diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs index 2b24475511..96cccc0f71 100644 --- a/core/continuum-core/src/capacity/model_residency.rs +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -131,6 +131,13 @@ struct HeardBeacon { /// overflow path honest rather than routing to a model a long-silent peer may have paged out. pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; +/// The residency beacon heartbeat — deliberately SLOWER than capacity's 10s +/// ([`super::gossip::PUBLISH_INTERVAL_MS`]) because residency changes on model page-in/out +/// (minute-scale), not the free-VRAM beat. 12 beats fit inside +/// [`RESIDENCY_EVICTION_WINDOW_MS`] — the same publish:eviction ratio capacity gossip uses, +/// so a couple of dropped beacons never evicts a still-resident peer. +pub const RESIDENCY_PUBLISH_INTERVAL_MS: u64 = RESIDENCY_EVICTION_WINDOW_MS / 12; + /// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the /// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness /// discipline, different (slower) cadence and payload. diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index f46d8b8f80..28fc437c7f 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1995,6 +1995,14 @@ pub fn start_server( default_room, ), )); + // Grid residency beacon (grid-overflow eligibility, slice 3): advertise which models + // this node holds resident on a slower cadence, folded by inbound_attach into + // capacity::model_residency::global_residency_ledger. Reads the SAME serving plan the + // daemon computes (watch snapshot, no parallel probe); orthogonal sibling of capacity. + runtime.register(Arc::new(crate::modules::grid_residency::GridResidencyModule::new( + serving_daemon.subscribe(), + default_room, + ))); let continuum_root = crate::modules::persona_instance_manager::resolve_continuum_root(); let daemon_socket_for_rag_inspect = daemon_socket.clone(); let registry = crate::persona::PersonaAircRuntimeRegistry::new(); diff --git a/core/continuum-core/src/modules/grid_residency.rs b/core/continuum-core/src/modules/grid_residency.rs new file mode 100644 index 0000000000..3f5fb81f0d --- /dev/null +++ b/core/continuum-core/src/modules/grid_residency.rs @@ -0,0 +1,166 @@ +//! GridResidencyModule — this node's residency-beacon gossip publisher (grid-overflow slice 3). +//! +//! The residency sibling of [`super::grid_capacity::GridCapacityModule`]. Every +//! [`RESIDENCY_PUBLISH_INTERVAL_MS`] tick it reads the SAME live serving plan the daemon +//! already computes (one source, no parallel probe) and broadcasts a [`ResidencyBeacon`] over +//! airc as an `EphemeralCoalesced` `grid_residency` realtime envelope: which models this node +//! holds resident — the grid-overflow ELIGIBILITY signal. The receive half lives in +//! [`crate::airc::inbound_attach`], which folds heard beacons (our own echo included — the +//! loopback proof) into [`crate::capacity::model_residency::global_residency_ledger`], whose +//! `view()` IS the `ModelResidencyView` the governor composes with the capacity snapshot. +//! +//! ## Why a SEPARATE module + slower cadence +//! +//! Residency and capacity are orthogonal (settled with BigMama 2026-07-27): capacity is +//! free-VRAM RIGHT NOW (10s beat, [`super::grid_capacity`]); residency is which models are warm +//! (minute-scale, changes only on page-in/out). Coupling them onto one envelope would either +//! over-publish residency or under-refresh capacity. Same module shape as the style guide +//! mandates — no new tokio task, the runtime tick drives it; the plan read is a lock-free +//! `watch` snapshot; the publish is one small envelope through the existing +//! `airc/realtime-publish` command surface (the same path capacity + chat ride). +//! +//! ## Today's local residency = the base model +//! +//! `ServingPlan` carries a single `base_model_id` + a `resident_models` COUNT (not a per-id +//! set), so today this node's honest resident set is `[base_model_id]`. When multi-model +//! residency lands (a per-id resident list on the plan), only [`Self::current_beacon`] changes +//! — the wire, ledger, and governor compose are already model-set-shaped. + +use std::any::Any; +use std::sync::Mutex; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::{json, Value}; +use tokio::sync::watch; + +use crate::capacity::model_residency::{ + global_residency_ledger, ResidencyBeacon, RESIDENCY_PUBLISH_INTERVAL_MS, +}; +use crate::cognition::serving_plan::ServingPlan; +use crate::runtime::{ + CommandExecutor, CommandResult, LateBound, ModuleConfig, ModulePriority, ServiceModule, +}; +use airc_core::RoomId; +use std::sync::Arc; + +pub struct GridResidencyModule { + /// Lock-free live snapshot of the daemon's serving plan — the SAME source the prefill + /// valve and serving control derive from (no parallel probe). + plan_rx: watch::Receiver>, + /// The node's discovered default room — where the grid rendezvous happens today. + room: RoomId, + executor_slot: Arc>, + /// Last published resident set — the glass box speaks on CHANGE, not on every beat (a + /// steady residency is silence). `None` = never published, so the first real plan speaks. + last_published: Mutex>>, +} + +impl GridResidencyModule { + pub fn new(plan_rx: watch::Receiver>, room: RoomId) -> Self { + Self { + plan_rx, + room, + executor_slot: Arc::new(LateBound::new("grid-residency::executor")), + last_published: Mutex::new(None), + } + } + + /// Build this node's residency beacon from the live serving plan. `None` when no plan is + /// computed yet (nothing resident to advertise) — honest silence, never a fabricated set. + /// Today the resident set is `[base_model_id]`; a multi-model plan extends only this line. + fn current_beacon(&self) -> Option { + let plan = self.plan_rx.borrow().clone()?; + Some(ResidencyBeacon { + resident_models: vec![plan.base_model_id], + at_ms: now_ms(), + }) + } +} + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[async_trait] +impl ServiceModule for GridResidencyModule { + fn config(&self) -> ModuleConfig { + ModuleConfig { + name: "grid-residency", + priority: ModulePriority::Background, + command_prefixes: &[], + event_subscriptions: &[], + needs_dedicated_thread: false, + max_concurrency: 0, + tick_interval: Some(Duration::from_millis(RESIDENCY_PUBLISH_INTERVAL_MS)), + } + } + + async fn initialize(&self, _ctx: &crate::runtime::ModuleContext) -> Result<(), String> { + Ok(()) + } + + async fn handle_command(&self, command: &str, _params: Value) -> Result { + Err(format!( + "grid-residency has no command surface — '{command}' (beacons publish on the module \ + tick; read the grid via capacity::model_residency::global_residency_ledger)" + )) + } + + async fn tick(&self) -> Result<(), String> { + // Boot ordering: a beat or two may fire before start_server installs the executor — + // transient, skip (the next beat publishes). Same guaranteed-installed path as capacity. + let Some(executor) = self.executor_slot.cloned() else { + return Ok(()); + }; + let Some(beacon) = self.current_beacon() else { + // No serving plan yet — nothing resident to advertise. Honest silence. + return Ok(()); + }; + + let envelope = json!({ + "eventId": uuid::Uuid::new_v4().to_string(), + "roomId": self.room.as_uuid().to_string(), + "sourceId": "grid-residency", + "createdAtMs": beacon.at_ms, + "delivery": "ephemeral_coalesced", + "payload": { + "kind": "existing_schema", + "payload": { + "schema": "grid_residency", + "inline": serde_json::to_value(&beacon) + .map_err(|e| format!("residency beacon encode failed: {e}"))?, + } + }, + }); + executor + .execute_json("airc/realtime-publish", json!({ "envelope": envelope })) + .await + .map_err(|e| format!("grid-residency beacon publish failed: {e}"))?; + + // Glass box: speak on change (resident set differs), silent on a steady residency. + let mut last = self.last_published.lock().map_err(|e| format!("residency lock: {e}"))?; + if last.as_deref() != Some(beacon.resident_models.as_slice()) { + crate::probe!( + class = "grid.residency.beacon", + models = ?beacon.resident_models, + heard_peers = global_residency_ledger().heard_count(), + "residency beacon published to the grid", + ); + *last = Some(beacon.resident_models); + } + Ok(()) + } + + fn install_executor(&self, executor: Arc) { + self.executor_slot.install(executor); + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/core/continuum-core/src/modules/mod.rs b/core/continuum-core/src/modules/mod.rs index fb9f016da5..890b1ed0fc 100644 --- a/core/continuum-core/src/modules/mod.rs +++ b/core/continuum-core/src/modules/mod.rs @@ -47,6 +47,7 @@ pub mod gpu; pub mod grant_issuance; pub mod grid; pub mod grid_capacity; +pub mod grid_residency; pub mod health; pub mod hippocampus; pub mod inference_coordinator_module; diff --git a/protocol/typescript/airc/AircRealtimeSchema.ts b/protocol/typescript/airc/AircRealtimeSchema.ts index f5040ab98c..a8efd11857 100644 --- a/protocol/typescript/airc/AircRealtimeSchema.ts +++ b/protocol/typescript/airc/AircRealtimeSchema.ts @@ -3,4 +3,4 @@ /** * Existing Continuum schema carried by an AIRC realtime envelope. */ -export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity"; +export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity" | "grid_residency"; From 4fe89ce7dc9760abdb9e8fa658ef1df8487d67ae Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:13:34 -0500 Subject: [PATCH 5/8] =?UTF-8?q?feat(capacity):=20route=5Fgrid=5Foverflow?= =?UTF-8?q?=20=E2=80=94=20remote-only=20overflow=20placement=20decision=20?= =?UTF-8?q?(governor=20consumer=20slice=204a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DECISION half of the driver, pure + fully unit-tested; only the thin EFFECT half (the actual Commands.execute("ai/generate", {aircPeer}) hop) needs a live peer and lands at the two-node smoke. Overflow lanes are BY DEFINITION the ones ServingPlan.grid_overflow_lanes said couldn't fit locally, so their placement is REMOTE-ONLY — it must never touch LocalFirstFitPolicy's local-first >=1 floor (that floor is the local persona's OWN guaranteed lane, orthogonal to spillover; re-cramming there is the exact thrash the honest overflow signal exists to avoid). Confirmed with BigMama 2026-07-27. Two orthogonal gates, composed (never absorbed): 1. RESIDENCY (ModelResidencyView::residency_eligible) — a peer is a fast overflow target only for a model it ALREADY holds (else a cold full-weights load defeats the point). 2. CONCURRENCY (reachability + lanes_that_fit misfit-parts) — among reachable eligible peers, most-free-first, each capped by its OWN budget for the prefill spike. Unplaced lanes (no eligible+reachable peer could take them) are SURFACED in OverflowRouting.unplaced for the caller to queue/degrade on — never silently dropped ([[fallbacks-are-illegal-fail-loud]]). 4 tests: remote-only + residency/reachability gating, unplaced-surfaced-not-dropped, zero-overflow no-op, most-free-first spill spread. This completes the pure governor-consumer decision path. Remaining (slice 4b, at the live two-node smoke): read grid_overflow_lanes from the live plan, build the lease via footprint.grid_lease_request, call route_grid_overflow, and execute the aircPeer hop per placed (peer, lanes) — the only piece needing a real peer to route to. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/capacity/grid_overflow.rs | 237 ++++++++++++++++++ core/continuum-core/src/capacity/mod.rs | 1 + 2 files changed, 238 insertions(+) create mode 100644 core/continuum-core/src/capacity/grid_overflow.rs diff --git a/core/continuum-core/src/capacity/grid_overflow.rs b/core/continuum-core/src/capacity/grid_overflow.rs new file mode 100644 index 0000000000..b34283d8d8 --- /dev/null +++ b/core/continuum-core/src/capacity/grid_overflow.rs @@ -0,0 +1,237 @@ +//! Grid-overflow routing — the DECISION half of the governor consumer: given a serving plan +//! that overflowed local capacity, decide which eligible peers take the overflow lanes. The +//! EFFECT half (the actual `Commands.execute("ai/generate", {aircPeer})` hop) is thin and lives +//! at the live seam; THIS is pure, deterministic, and fully unit-tested. +//! +//! ## Why overflow placement is REMOTE-ONLY (not [`super::grid::LocalFirstFitPolicy`]) +//! +//! [`super::grid::LocalFirstFitPolicy`] fills local first and floors local at `≥1` (a resident +//! model must be able to run one prefill). That floor is correct for a FRESH placement but +//! WRONG for overflow: overflow lanes are BY DEFINITION the ones that already could not fit +//! locally (`ServingPlan.grid_overflow_lanes = demand − local_lanes`). Placing them local-first +//! would re-cram the very lanes the planner just declared didn't fit — the thrash the honest +//! "over local capacity by N" signal exists to avoid. So overflow placement never touches local: +//! it spills ONLY to eligible remote peers. +//! +//! ## The two gates, composed (never absorbed) +//! +//! 1. RESIDENCY ([`ModelResidencyView::residency_eligible`]): a peer is a fast overflow target +//! only for a model it ALREADY holds — else the hop pays a cold full-weights load, defeating +//! the point. Filters the snapshot to peers holding the model. +//! 2. CONCURRENCY (the misfit-parts fit, [`super::lanes_that_fit`]): among residency-eligible + +//! REACHABLE peers, each takes at most what its OWN free budget fits for the prefill spike — +//! the same per-node-fit rule the single-device and grid policies run. +//! +//! Reachability is applied HERE (an unreachable-but-resident peer is a memory, not an offer), +//! composing cleanly with residency without either abstraction absorbing the other. +//! +//! ## Unplaced lanes are SURFACED, never dropped +//! +//! When no eligible peer can take a lane, it lands in [`OverflowRouting::unplaced`] — the honest +//! "the grid couldn't absorb N of your overflow" signal the caller queues or degrades on. Never +//! silently swallowed ([[fallbacks-are-illegal-fail-loud]]). + +use crate::identity::PeerId; + +use super::grid::GridSnapshot; +use super::lanes_that_fit; +use super::model_residency::ModelResidencyView; +use super::LeaseRequest; + +/// The routing decision for a plan's overflow lanes: where each lane lands (remote-only) plus +/// the honest count of lanes the reachable, residency-eligible grid could NOT absorb. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OverflowRouting { + /// Overflow lanes placed on named peers, most-free-first, each capped by its own fit. + pub remote: Vec<(PeerId, u32)>, + /// Overflow lanes no eligible+reachable peer could take — queue or degrade on these, + /// never drop them silently. + pub unplaced: u32, +} + +impl OverflowRouting { + /// Total overflow lanes actually placed on peers. + pub fn placed(&self) -> u32 { + self.remote.iter().map(|(_, n)| n).sum() + } +} + +/// Decide where a plan's overflow lanes run. `lease` is the demand→capacity projection the +/// serving side already built (`ModelFootprint::grid_lease_request(served_window, overflow_lanes)`): +/// `want_concurrency` = the overflow lane count, `spike_bytes` = the per-lane prefill transient. +/// `model_id` is what the overflowing node is serving (its `ServingPlan.base_model_id`) — the +/// residency key. REMOTE-ONLY by construction (see module docs): local is already saturated. +pub fn route_grid_overflow( + model_id: &str, + lease: &LeaseRequest, + residency: &ModelResidencyView, + snapshot: &GridSnapshot, + safety_margin_bytes: u64, +) -> OverflowRouting { + let want = lease.want_concurrency; + if want == 0 { + return OverflowRouting { remote: Vec::new(), unplaced: 0 }; + } + + // Gate 1 — residency: keep only peers that hold the model resident. + let eligible = residency.residency_eligible(snapshot, model_id); + + // Gate 2 — reachability + per-node fit: reachable eligible peers, most-free-first (fewest + // peers touched), each capped by its OWN budget for the prefill spike. + let mut reachable: Vec<_> = eligible.peers.iter().filter(|p| p.reachable).collect(); + reachable.sort_by(|a, b| { + b.capacity + .gpu_free_bytes_live + .cmp(&a.capacity.gpu_free_bytes_live) + }); + + let mut remaining = want; + let mut remote = Vec::new(); + for peer in reachable { + if remaining == 0 { + break; + } + let fit = lanes_that_fit( + peer.capacity.gpu_free_bytes_live, + safety_margin_bytes, + lease.spike_bytes, + ); + let take = fit.min(remaining); + if take > 0 { + remote.push((peer.peer, take)); + remaining -= take; + } + } + + // Whatever the reachable, residency-eligible grid couldn't absorb is surfaced, not dropped. + OverflowRouting { remote, unplaced: remaining } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridSnapshot, PeerCapacity}; + use crate::capacity::DeviceCapacity; + use uuid::Uuid; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + fn lease(want: u32, spike_gb: u64) -> LeaseRequest { + LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: want, + spike_bytes: spike_gb * GB, + } + } + + fn view_holding(peers: &[(u128, &[&str])]) -> ModelResidencyView { + let mut v = ModelResidencyView::new(); + for (n, models) in peers { + v.set_resident(peer_id(*n), models.iter().map(|s| s.to_string())); + } + v + } + + // what this catches: overflow placement is REMOTE-ONLY and residency+reachability gated. + // Local is never assigned lanes (it's the saturated node that overflowed). Only a peer that + // holds the model AND is reachable takes lanes; a resident-but-unreachable peer and a + // reachable-but-not-resident peer both take nothing. + #[test] + fn overflow_routes_remote_only_to_reachable_resident_peers() { + let snap = GridSnapshot { + local: dev(1), // saturated — must never receive overflow lanes + peers: vec![ + peer(1, 40, true), // resident + reachable → takes lanes + peer(2, 40, false), // resident + UNREACHABLE → nothing + peer(3, 40, true), // reachable but NOT resident → nothing + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(2, 1), &residency, &snap, GB); + + assert_eq!(routing.remote.len(), 1, "only peer 1 is eligible + reachable"); + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1)); + assert_eq!(routing.placed(), 2, "both overflow lanes fit on peer 1"); + assert_eq!(routing.unplaced, 0); + } + + // what this catches: unplaced lanes are SURFACED, never dropped. When the eligible grid + // can't fit all overflow lanes (one small peer, a big per-lane spike), the shortfall is + // reported so the caller queues/degrades — the honest "grid couldn't absorb N" signal. + #[test] + fn lanes_the_grid_cannot_absorb_are_surfaced_not_dropped() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![peer(1, 10, true)], // ~10GB free, but each lane spikes 8GB + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + // want 3 lanes, 8GB spike each: only 1 fits on the 10GB peer (net of 1GB margin). + let routing = route_grid_overflow("qwen-coder", &lease(3, 8), &residency, &snap, GB); + + assert_eq!(routing.placed(), 1, "only one lane fits the peer's budget"); + assert_eq!(routing.unplaced, 2, "the other two are surfaced, not silently dropped"); + } + + // what this catches: no overflow (want == 0) is a clean no-op — nothing placed, nothing + // unplaced. The common case (demand fit locally, grid_overflow_lanes == 0) costs nothing. + #[test] + fn zero_overflow_is_a_clean_noop() { + let snap = GridSnapshot { + local: dev(20), + peers: vec![peer(1, 40, true)], + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(0, 1), &residency, &snap, GB); + + assert!(routing.remote.is_empty()); + assert_eq!(routing.unplaced, 0); + assert_eq!(routing.placed(), 0); + } + + // what this catches: spill spreads across peers most-free-first, each capped by its OWN fit + // (the misfit-parts rule) — 12 aggregate GB across three 4GB peers can't run a 6-lane + // placement if no single peer fits it, but distinct small lanes DO spread. Two reachable + // resident peers each take their share until demand is met. + #[test] + fn spill_spreads_across_peers_most_free_first() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![ + peer(1, 20, true), // more free → sorted first + peer(2, 12, true), + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + // 4 lanes, 8GB spike: peer1 (20-1 margin=19 → 2 lanes), peer2 (12-1=11 → 1 lane) = 3, + // one unplaced. Peer 1 (most free) is filled before peer 2. + let routing = route_grid_overflow("qwen-coder", &lease(4, 8), &residency, &snap, GB); + + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1), "most-free peer first"); + assert_eq!(routing.placed() + routing.unplaced, 4, "every lane accounted for"); + assert!(routing.placed() >= 3, "peers absorb what their own budgets fit"); + } +} diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index cc1ed11e0f..3ae5d3de6f 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -42,6 +42,7 @@ pub mod expert_residency; pub mod expert_tier_policy; pub mod gossip; pub mod grid; +pub mod grid_overflow; pub mod lease; pub mod market; pub mod moe_arch_profile; From 468e835b5f7ba844d0a28661e9de8e00773b4874 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:26:02 -0500 Subject: [PATCH 6/8] =?UTF-8?q?feat(persona):=20grid-overflow=20effector?= =?UTF-8?q?=20seam=20=E2=80=94=20materialize=5Fadapters=20overflow=5Fadapt?= =?UTF-8?q?er=5Ffor=20override=20(governor=20consumer=20slice=204b-i)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composition point where route_grid_overflow's decision becomes a real remote brain. materialize_adapters gains an `overflow_adapter_for(&profile, slot)` closure (same closure-DI shape as runtime_lookup / tool_executor_for): return Some(remote adapter) when the governor routed this persona off-box — her node is over capacity and a reachable peer holds her model — so her brain runs on that peer via AircRemoteInferenceAdapter; None → build the local adapter from the factory (the common case). This is the exact re-home seam the DeliberationModelBinding was designed for (its doc: re-home = "a new adapter / grid failover onto another node"). The remote adapter registers in the global provider registry by model_id just like the local one, so evaluate_response reaches it transparently — the persona doesn't know or care that her inference crosses the grid. host.rs passes `|_,_| None` for now (slice 4b-ii wires the live capacity + residency + airc closure at the ipc bootstrap, where the serving plan + airc handle live). Test overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory pins the contract: when the override supplies a slot's adapter, the local factory is NOT called for it (build_count == 1 of 2), both personas host, both warm. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/persona/host.rs | 4 + core/continuum-core/src/persona/supervisor.rs | 103 ++++++++++++++---- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index d42ad0df19..6731d7062d 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -328,6 +328,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, + // Grid-overflow effector closure (slice 4b-ii wires the live capacity + + // residency + airc context here). Until then every persona builds her + // local adapter — no off-box routing, the pre-effector behavior. + |_profile, _slot| None, ) .await; diff --git a/core/continuum-core/src/persona/supervisor.rs b/core/continuum-core/src/persona/supervisor.rs index 66471a2d4d..d55a966b1c 100644 --- a/core/continuum-core/src/persona/supervisor.rs +++ b/core/continuum-core/src/persona/supervisor.rs @@ -499,6 +499,14 @@ pub async fn materialize_adapters( uuid::Uuid, ) -> Option>, + // The grid-overflow EFFECTOR (governor consumer slice 4b). Given a persona's + // profile + slot, returns `Some(remote adapter)` when the governor routed her + // off-box — her node is over local capacity and a reachable peer holds her model + // (`capacity::grid_overflow::route_grid_overflow`) — so her brain runs on that + // peer via `AircRemoteInferenceAdapter`. `None` → build the local adapter from the + // factory (the common case; demand fit locally). Closure DI keeps the supervisor + // decoupled from the capacity fabric + airc handle, same shape as the lookups above. + overflow_adapter_for: impl Fn(&PersonaInferenceProfile, usize) -> Option>, ) -> Vec> { let mut out = Vec::with_capacity(plans.len()); for (slot_index, plan) in plans.into_iter().enumerate() { @@ -525,16 +533,22 @@ pub async fn materialize_adapters( continue; } }; - let adapter = match factory.build_adapter(&profile).await { - Ok(a) => a, - Err(message) => { - out.push(Err(SupervisorError::AdapterFactory { - slot_index, - role: plan.role, - message, - })); - continue; - } + // Grid-overflow effector: if the governor routed this persona off-box, her + // adapter is the airc-remote one (her brain runs on the peer that holds her + // model). Else build the local adapter from the factory — the common case. + let adapter = match overflow_adapter_for(&profile, slot_index) { + Some(remote) => remote, + None => match factory.build_adapter(&profile).await { + Ok(a) => a, + Err(message) => { + out.push(Err(SupervisorError::AdapterFactory { + slot_index, + role: plan.role, + message, + })); + continue; + } + }, }; // Warm the adapter's KV-cache / kernels BEFORE the persona // enters her service loop. Per [[init-once-handle-then-lease-zero-copy-refs]] @@ -1106,7 +1120,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); assert_eq!(factory.build_count(), 2); @@ -1153,7 +1167,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory called exactly once — for the Ok row only. @@ -1188,7 +1202,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::always_fails("simulated factory rejection"); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1211,7 +1225,7 @@ mod tests { #[tokio::test] async fn empty_plans_yields_empty_hosted() { let factory = ScriptedPersonaAdapterFactory::heuristic(); - let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None, |_, _| None).await; assert!(hosted.is_empty()); assert_eq!(factory.build_count(), 0); } @@ -1240,7 +1254,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); // `|_| None` here is the substrate-bug shape we're locking in: // the registry exists but doesn't contain this persona_id. - let hosted = materialize_adapters(plans, &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(plans, &factory, |_| None, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); // Factory MUST NOT be called when the runtime lookup fails — @@ -1300,7 +1314,7 @@ mod tests { as Arc) } }; - let hosted = materialize_adapters(plans, &factory, lookup, |_| None).await; + let hosted = materialize_adapters(plans, &factory, lookup, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory ran exactly once — for Paige, not Pax. @@ -1345,7 +1359,7 @@ mod tests { let (factory, counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; // Both slots materialize cleanly. assert_eq!(hosted.len(), 2); @@ -1359,6 +1373,56 @@ mod tests { ); } + // what this catches: the grid-overflow EFFECTOR seam — when the governor routes a + // persona off-box, `overflow_adapter_for` supplies her adapter (the airc-remote one, + // her brain on a peer) and the LOCAL factory is NOT called for that slot. Here slot 0 + // is overflow-routed (override returns a stand-in remote), slot 1 is local. Both host; + // the factory builds ONLY slot 1 (build_count == 1); both adapters still warm. This is + // the composition point where route_grid_overflow's decision becomes a real remote brain. + #[tokio::test] + async fn overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory() { + use crate::ai::heuristic_adapter::HeuristicInferenceAdapter; + let plans = vec![ + MaterializedPersonaPlan { + role: RoleId::Helper, + instance: fake_instance("Paige"), + profile: Ok(fake_profile("Paige", "model-a")), // slot 0 → overflow-routed + }, + MaterializedPersonaPlan { + role: RoleId::Coder, + instance: fake_instance("Pax"), + profile: Ok(fake_profile("Pax", "model-b")), // slot 1 → local + }, + ]; + + let (factory, _counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); + let hosted = materialize_adapters( + plans, + &factory, + StubAircCitizen::fresh_lookup(), + |_| None, + // Overflow effector: slot 0 is routed off-box → supply a stand-in "remote" + // adapter; every other slot stays local (None). + |_profile, slot| { + if slot == 0 { + Some(Arc::new(HeuristicInferenceAdapter::new()) as Arc) + } else { + None + } + }, + ) + .await; + + assert_eq!(hosted.len(), 2); + assert!(hosted.iter().all(|r| r.is_ok()), "both personas host (one remote, one local)"); + assert_eq!( + factory.build_count(), + 1, + "the local factory builds ONLY the non-overflow slot — the overflow slot's \ + adapter came from the effector, her brain runs on the peer" + ); + } + /// Warmup failure surfaces as `SupervisorError::AdapterWarmup` — /// the persona does NOT reach hosted state. Per [[no-fallbacks-ever]] /// an adapter that refuses to warm gets a typed slot failure; @@ -1375,7 +1439,7 @@ mod tests { "simulated warmup failure", ); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1411,7 +1475,7 @@ mod tests { profile: Ok(fake_profile("Paige", "model-a")), }]; let hosted_ok = - materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None) + materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None) .await; assert!(hosted_ok[0].is_ok(), "ok-warmup adapter materializes"); assert_eq!(ok_counts.warmups(), 1); @@ -1429,6 +1493,7 @@ mod tests { &factory_fail, StubAircCitizen::fresh_lookup(), |_| None, + |_, _| None, ) .await; assert!( From d843b471a223625f14ec8549cc3fe7ccdbfddce0 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:38:33 -0500 Subject: [PATCH 7/8] =?UTF-8?q?feat(persona):=20grid-overflow=20effector?= =?UTF-8?q?=20LIVE=20closure=20=E2=80=94=20a=20persona's=20brain=20routes?= =?UTF-8?q?=20off-box=20to=20a=20residency-eligible=20peer=20(governor=20c?= =?UTF-8?q?onsumer=20slice=204b-ii)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last mile. build_overflow_effector (persona/grid_overflow_effector.rs) composes the whole tested decision path into the per-persona adapter override the supervisor consumes, wired at the ipc bootstrap spawn: live serving plan (grid_overflow_lanes) → footprint from live_candidates → grid_lease_request → global_residency_ledger().view → route_grid_overflow → AircLiveTransport(airc, peer) → AircRemoteInferenceAdapter When the node is over local capacity and a reachable peer already holds a persona's model, her DeliberationModelBinding.adapter becomes the airc-remote one — her inference crosses the grid transparently (the re-home the binding was designed for), and she lives in the room as a peer hosted on another machine. That's "a competent peer so it's not just us there." DEFENSIVE by construction (safe to ship pre-smoke): returns None → local adapter on ANY uncertainty (airc not attached, no plan, no overflow, no matching footprint, no eligible reachable peer). Can only be a safe no-op or a correct off-box route — never a self-route (own peer excluded via airc.peer_id(), so its own residency-beacon loopback can't pick itself) and never a panic. The only unit-unprovable part is that the remote hop SUCCEEDS — the live two-node smoke validates that; a hop that can't warm surfaces as a loud AdapterWarmup slot failure, never a silent local downgrade ([[fallbacks-are-illegal-fail-loud]]). Plumbing: overflow_adapter_for threaded through spawn_all → materialize_adapters (4b-i seam); dedicated Arc clones of the airc-interceptor cell + serving daemon so the boot-spawn async-move capture doesn't strand the interceptor + reconcile task; live_candidates() → pub(crate). Completes the grid-overflow governor consumer end-to-end. Live validation + the cross-node generation run the moment BigMama's node is serving (model_id string-equality is the one thing to confirm live). All unit paths green (21 supervisor/overflow/residency tests). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/ipc/mod.rs | 26 +++- .../src/modules/serving_daemon.rs | 2 +- .../src/persona/grid_overflow_effector.rs | 115 ++++++++++++++++++ core/continuum-core/src/persona/host.rs | 17 ++- core/continuum-core/src/persona/mod.rs | 1 + 5 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 core/continuum-core/src/persona/grid_overflow_effector.rs diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index 28fc437c7f..6414c6510d 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -2339,6 +2339,15 @@ pub fn start_server( // below). Subscribe BEFORE the spawn so no plan edge is missed while // the task waits on the executor-ready oneshot. let mut serving_plan_rx = serving_daemon.subscribe(); + // A DEDICATED clone of the interceptor's airc-handle cell for the grid-overflow + // effector (slice 4b): the boot-spawn `async move` below captures by move, and the + // interceptor still needs the original `airc_interceptor_cell` further down — so the + // effector rides its own Arc clone of the SAME shared cell (both see the handle once + // attach_as fills it). + let overflow_airc_cell = airc_interceptor_cell.clone(); + // Same reason for the serving daemon: the reconcile task below still needs the + // original `serving_daemon` handle, so the effector rides its own clone. + let overflow_serving = serving_daemon.clone(); rt_handle.spawn(async move { // Wait for the IPC thread to deliver the WIRED executor (this both // gates ordering AND hands us the executor the personas' hands ride). @@ -2433,7 +2442,22 @@ pub fn start_server( } else { attempt += 1; let summary = supervisor - .spawn_all(&mut provider, Some(tool_executor.clone())) + .spawn_all( + &mut provider, + Some(tool_executor.clone()), + // Grid-overflow effector (slice 4b): the LIVE closure. Reads + // the serving plan's grid_overflow_lanes, filters residency- + // eligible reachable peers (from the beacon ledger), runs + // route_grid_overflow, and re-homes the persona's adapter to an + // AircRemoteInferenceAdapter over the interceptor's airc handle. + // DEFENSIVE: None on any uncertainty → local adapter (no + // regression); can only be a safe no-op or a correct off-box + // route (self excluded via airc.peer_id()). + crate::persona::grid_overflow_effector::build_overflow_effector( + overflow_airc_cell.clone(), + overflow_serving.clone(), + ), + ) .await; if summary.hosted > 0 { tracing::info!( diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index 5f5bfaa378..ed363c9e73 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -813,7 +813,7 @@ impl ServingDaemonModule { /// one candidate, so the reconcile serves that model or (if it has dropped off /// disk) nothing. Suppress subtracts; pin intersects; the planner still owns /// the choice among whatever remains. - fn live_candidates(&self) -> Vec { + pub(crate) fn live_candidates(&self) -> Vec { let suppressed = self.suppressed.borrow(); let pinned = self.pinned.borrow(); servable_candidates(&self.catalog.snapshot(), &**suppressed, &pinned) diff --git a/core/continuum-core/src/persona/grid_overflow_effector.rs b/core/continuum-core/src/persona/grid_overflow_effector.rs new file mode 100644 index 0000000000..4acc4a7286 --- /dev/null +++ b/core/continuum-core/src/persona/grid_overflow_effector.rs @@ -0,0 +1,115 @@ +//! The grid-overflow EFFECTOR (governor consumer slice 4b-ii) — composes the tested +//! decision path into the live per-persona adapter override that +//! [`super::supervisor::materialize_adapters`] consumes. +//! +//! When the node is over local capacity (`ServingPlan.grid_overflow_lanes > 0`) and a +//! reachable peer already holds a persona's model, this routes HER BRAIN to that peer: her +//! `DeliberationModelBinding.adapter` becomes an [`AircRemoteInferenceAdapter`] over airc, so +//! her inference crosses the grid transparently — the exact re-home the binding was designed +//! for. The persona doesn't know or care that her model runs on another machine. +//! +//! ## Defensive by construction — safe to ship before the live smoke +//! +//! The closure returns `None` (→ the local factory adapter, zero behavior change) on ANY +//! uncertainty: airc not yet attached, no serving plan, no overflow, no matching footprint, or +//! no residency-eligible reachable peer. So it can only ever be a **safe no-op or a correct +//! remote route** — never a self-route (this node's own peer is excluded via `airc.peer_id()`, +//! so its own residency-beacon loopback can't select itself) and never a panic. The one thing +//! the unit path can't prove is that the remote hop SUCCEEDS — that is what the live two-node +//! smoke validates; a hop that can't warm surfaces as a loud per-slot `AdapterWarmup` failure +//! ([[fallbacks-are-illegal-fail-loud]]), never a silent local downgrade. + +use std::sync::Arc; + +use airc_lib::Airc; +use tokio::sync::OnceCell; + +use crate::ai::adapter::AIProviderAdapter; +use crate::capacity::gossip::global_ledger; +use crate::capacity::grid_overflow::route_grid_overflow; +use crate::capacity::model_residency::global_residency_ledger; +use crate::capacity::DeviceCapacity; +use crate::inference::airc_remote::adapter::AircRemoteInferenceAdapter; +use crate::inference::airc_remote::transport::AircLiveTransport; +use crate::modules::serving_daemon::ServingDaemonModule; +use crate::persona::inference_profile::PersonaInferenceProfile; + +/// Headroom kept free on a peer before it may accept an overflow lane — same 1 GiB spirit as +/// the single-device [`crate::capacity::grid::LocalFirstFitPolicy`] safety margin. +const OVERFLOW_SAFETY_MARGIN_BYTES: u64 = 1024 * 1024 * 1024; + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Build the effector closure `spawn_all` / `materialize_adapters` consume. Captures the +/// late-bound airc handle cell (shared with the interceptor) and the serving daemon (the live +/// plan + footprint source). See the module docs for the defensive contract. +pub fn build_overflow_effector( + airc_cell: Arc>>, + serving: Arc, +) -> impl Fn(&PersonaInferenceProfile, usize) -> Option> { + move |profile, _slot| { + // airc not attached yet → local (the boot window before attach_as fills the cell). + let airc = airc_cell.get()?.clone(); + // No plan, or demand fits locally → local. grid_overflow_lanes is the honest + // "over local capacity by N" signal; 0 means nothing to spill. + let plan = serving.compute_plan()?; + if plan.grid_overflow_lanes == 0 { + return None; + } + // The footprint for THIS persona's model — the lease's per-lane prefill spike. + // Absent (model not a live candidate) → local, never a fabricated footprint. + let footprint = serving + .live_candidates() + .into_iter() + .find(|f| f.model_id == profile.model_id)?; + let lease = + footprint.grid_lease_request(plan.served_context_window, plan.grid_overflow_lanes); + + // Own peer id from the airc handle → excludes self from the residency view + gossip + // snapshot (this node's own beacon loopback must never select itself as the target). + let own = airc.peer_id().as_uuid(); + let now = now_ms(); + let residency = global_residency_ledger().view(own, now); + // Overflow placement is REMOTE-ONLY, so local capacity is never read — a zeroed local + // is the honest input (route_grid_overflow only ever inspects the peer list). + let snapshot = global_ledger().snapshot( + own, + DeviceCapacity { + gpu_total_bytes: 0, + gpu_free_bytes_live: 0, + system_ram_free_bytes: 0, + }, + now, + ); + + let routing = route_grid_overflow( + &profile.model_id, + &lease, + &residency, + &snapshot, + OVERFLOW_SAFETY_MARGIN_BYTES, + ); + // First-cut assignment: this persona takes the first placed peer. No residency-eligible + // reachable peer with room → local (queue/degrade is the planner's, not a silent drop). + let (peer, _lanes) = routing.remote.first()?; + let peer_uuid = peer.as_uuid(); + + let transport = AircLiveTransport::new(airc, peer_uuid); + let adapter = AircRemoteInferenceAdapter::new(transport); + crate::probe!( + class = "grid.overflow.route", + persona = %profile.persona_name, + model = %profile.model_id, + peer = %peer_uuid, + overflow_lanes = plan.grid_overflow_lanes, + "routing persona brain OFF-BOX to a residency-eligible peer (grid overflow)", + ); + Some(Arc::new(adapter) as Arc) + } +} diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index 6731d7062d..c9ec318822 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -264,6 +264,15 @@ impl PersonaSpawnSupervisor { // persona's HANDS are built over it (identity-scoped), so the ACL gates // what they may do. `None` → personas spawn speak-only (no hands). tool_command_executor: Option>, + // The grid-overflow effector (governor consumer slice 4b): given a persona's + // profile + slot, `Some(remote adapter)` routes her brain to a peer that holds + // her model (the ipc bootstrap builds this from the live serving plan + residency + // ledger + airc handle); `None` → local adapter. Forwarded verbatim to + // `materialize_adapters`. `|_, _| None` is the pre-effector (all-local) behavior. + overflow_adapter_for: impl Fn( + &crate::persona::inference_profile::PersonaInferenceProfile, + usize, + ) -> Option>, ) -> BootSummary { let plans = match bootstrap_planned( &self.spawner, @@ -328,10 +337,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, - // Grid-overflow effector closure (slice 4b-ii wires the live capacity + - // residency + airc context here). Until then every persona builds her - // local adapter — no off-box routing, the pre-effector behavior. - |_profile, _slot| None, + // Grid-overflow effector: the ipc bootstrap supplies the live decision + // (capacity + residency + airc). `|_, _| None` from a caller that doesn't + // route keeps the pre-effector all-local behavior. + overflow_adapter_for, ) .await; diff --git a/core/continuum-core/src/persona/mod.rs b/core/continuum-core/src/persona/mod.rs index bf7a5ab61a..01b2e344b1 100644 --- a/core/continuum-core/src/persona/mod.rs +++ b/core/continuum-core/src/persona/mod.rs @@ -52,6 +52,7 @@ pub mod evaluator; pub mod focus; pub mod genome_paging; pub mod home; +pub mod grid_overflow_effector; pub mod host; pub mod hw_tier_descriptor; pub mod identity_provider; From e91e70fe854b13cad8e18ab098a2f994941d0018 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 11 Aug 2026 18:21:35 -0500 Subject: [PATCH 8/8] fix(serving): merge grid_lease_request INTO canary's serving_plan, not over it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase-resolution fix, and a correction of my own error worth recording. While cherry-picking the residency stack I hit a conflict in serving_plan.rs and resolved it with `git checkout --theirs` — which takes the WHOLE branch file. That silently discarded canary's ServingDemand (the elastic-window struct that landed separately), producing 'unresolved import ServingDemand' in two modules. Choosing a side is not merging; it is overwriting with extra steps, and it is the same class of mistake as blind-skipping commits during a rebase. Correct resolution: keep canary's serving_plan.rs verbatim and add ONLY the one symbol the residency stack actually needs — ModelFootprint::grid_lease_request, the serving→grid bridge that maps a footprint to a capacity LeaseRequest (want_concurrency = demanded lanes, spike_bytes = ONE lane's prefill reserve at the served window). cargo check -p continuum-core --lib --tests clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Q4NU4VNiELPQfBpCacDZGc --- .../src/cognition/serving_plan.rs | 542 ++++++++++++------ 1 file changed, 369 insertions(+), 173 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 9651511c70..fc983bb1ca 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -73,7 +73,7 @@ /// only bounds a pathological roster and llama.cpp `--parallel` practicality. Kept modest /// (4, not the 6 that OOM'd) pending a LARGE-prompt 4-lane live-GPU burst — the doc's /// acceptance gate. [[verify-real-device-numbers-not-a-clamp-premise]] [[capacity-fabric-live-never-block-sim-as-gym]] -pub const MAX_LANES: u32 = 2; +pub const MAX_LANES: u32 = 8; // ⚠️ 2026-07-17 REVERTED 4 → 2. Raising to 4 (warm slot per persona) was OOM-SAFE // (the window-scaled fit shrank -c to fit) but STARVED CONTEXT: splitting the budget 4 // ways dropped the per-slot window to ~6k, and a live persona's assembled prompt is ~9k @@ -85,6 +85,25 @@ pub const MAX_LANES: u32 = 2; // 2 lanes at ~19.8k each beats 4 warm lanes at 6k. Cross-turn clobber (the reason for the // raise) is the lesser evil vs a starved window; solve it with idle-warmth/duty-cycling, // NOT by cutting everyone's context. [[no-hardcoded-context-numbers-derive-from-the-live-window]] +// +// ✅ 2026-08-09 RAISED 2 → 8 (#266). The 2026-07-17 revert diagnosed the failure correctly +// (a starved per-slot window) but fixed it in the WRONG place: it clamped the lane CEILING, +// when the real defect was that the lane-COUNT gate below sheds against `MIN_SERVE_CTX` +// (2048, the "runnable at all" floor) — far below the ~9k a live turn needs — so a raised +// ceiling let 4 slots @ 6k through. That is now fixed at the true seam: the shed in +// `plan_serving` requires each slot to clear `BOOTSTRAP_WORKING_SET` (16384 ≈ a full +// assembled turn + generation headroom), the SAME "one full turn" figure the demand cap +// uses. With that usable-window floor as the binding constraint, a slot per resident persona +// is safe BY CONSTRUCTION — the plan only grows lanes toward the resident population while +// each still gets a full turn, and sheds (surfacing `grid_overflow_lanes` + a probe) the +// moment the floor would be breached, never below it. The 2026-07-17 "solve clobber with +// duty-cycling, not lanes" stance was itself the bug this leaves on the table: cross-turn +// KV clobber IS #266's whole latency story (measured: 96.3% of persona compute is prefill; +// two of four citizens at 0.0% cache reuse across 10 turns — the LRU eviction of a warm +// slot the resident count outnumbered). Giving each resident mind its own slot is what +// keeps its prefilled prefix warm. This constant is once again a pure SANITY backstop +// (pathological roster + llama.cpp `--parallel` practicality); the binding constraint is +// the window floor, exactly as the earlier doc always claimed it should be. /// Bare-minimum served window for a model to be runnable at ALL — a hardware /// reality floor, NOT a serving target or a cheapening cap. The served window is @@ -94,6 +113,7 @@ pub const MAX_LANES: u32 = 2; /// personas actually USE, never maximized to fill RAM. A model whose weights + KV /// at even this floor won't fit the GPU budget is simply not a serving option on /// this host (→ `fits_on_gpu = false`, honest degrade — never a silent shrink). +// context-budget-exempt: the hardware FLOOR the whole serving stack sizes UP from — the one substrate-owned minimum every other bound derives against, and never a cap on anything pub const MIN_SERVE_CTX: u32 = 2048; /// Cold-start DEMAND ceiling for the served window — the "B-now" half of the @@ -115,6 +135,41 @@ pub const MIN_SERVE_CTX: u32 = 2048; /// floor (`MIN_SERVE_CTX × 8 = 16384`) rather than a second bare magic number. pub const BOOTSTRAP_WORKING_SET: u32 = MIN_SERVE_CTX * 8; +/// What the minds on this host are asking the serving lane for. +/// +/// Both axes of demand in ONE value, because they are one question — "how much +/// serving does the work on this box actually need" — and passing them as two loose +/// `u32`s next to each other is how a caller silently swaps them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServingDemand { + /// How many minds want a concurrent lane. + pub lanes: u32, + /// The largest per-turn window any resident mind has actually demanded + /// ([`crate::cognition::working_set::WorkingSetRegistry::ceiling`]) — measured + /// UNCLAMPED, so it is free to exceed what is currently served. That excess is the + /// signal that the window is too small; nothing else in the system can produce it. + pub window_tokens: u32, +} + +impl ServingDemand { + /// Demand from live measurement, with the cold-start case named explicitly. + /// + /// `measured` is `None` only before ANY turn has been assembled on this host — + /// a genuine absence of data, not a missing feature. The window then starts at + /// [`BOOTSTRAP_WORKING_SET`] (one full turn's worth: assembled prompt plus + /// generation headroom, anchored to `MIN_SERVE_CTX × 8` rather than a second bare + /// number) and is superseded by measurement on the very next plan, because the + /// first turn records its demand before it is even sent. This is the one place + /// that decision lives — the registry deliberately returns `None` rather than + /// inventing a number every caller would then inherit without noticing. + pub fn new(lanes: u32, measured: Option) -> Self { + Self { + lanes, + window_tokens: measured.unwrap_or(BOOTSTRAP_WORKING_SET), + } + } +} + /// Hysteresis margin for switching UP to a more capable model: it must fit /// within `(1 - SWITCH_UP_HEADROOM)` of the budget — i.e. with headroom to /// spare — before we abandon the incumbent for it. Stops transient budget @@ -264,14 +319,12 @@ impl ModelFootprint { /// The capacity-fabric [`LeaseRequest`](crate::capacity::LeaseRequest) for serving /// this model at `served_window` with `demand_lanes` concurrent minds — the bridge /// from the serving plan's MODEL-RESIDENCY view (weights + per-lane KV) to the grid's - /// CONCURRENCY-SPIKE view, so [`GridPlacementPolicy`](crate::capacity::grid) can place - /// overflow lanes onto peers (#180 grid spill / [[frontier-is-a-scaling-question-over-misfits-not-a-capability-question]]). + /// CONCURRENCY-SPIKE view, so the grid can place overflow lanes onto peers (#180 spill). /// `want_concurrency` = the minds that want a lane; `spike_bytes` = ONE lane's transient /// prefill compute buffer at this window (the term the 2026-07-14 OOM turned on), so a /// peer is only offered a spill lane it can actually hold. NOTE: this sizes the /// CONCURRENCY spike only; a peer must ALSO already hold this model resident — that - /// residency gate is the gossip-side half of the routing (owned with the grid snapshot), - /// NOT this pure mapping. + /// residency gate is the gossip-side half (capacity::model_residency), NOT this pure map. pub fn grid_lease_request( &self, served_window: u32, @@ -333,26 +386,14 @@ pub struct ServingPlan { /// rendered, and the room degenerated into a greeting loop. 2 slots would /// have doubled every mind's window with zero lost concurrency. /// -/// `demand_ceil` is the LIVE demand ceiling for the served window — the ELASTIC, -/// per-task upper bound. `window_for` sizes the window UP to what the budget -/// allows; this caps it DOWN to what the task actually needs. A hard coding task -/// passes a high ceiling and the window GROWS (up to the budget/model bound); a -/// simple turn passes a low one so more lanes fit. It is NEVER a launch-baked -/// constant — callers thread live per-persona/per-task demand (measured p95 + -/// headroom, or a task's explicit request). [`plan_serving`] supplies -/// [`BOOTSTRAP_WORKING_SET`] only as the cold-start prior until that telemetry -/// exists (#234). OOM-safe: `window_for` already bounds the window to the budget, -/// so a higher ceiling only raises the cap toward that bound, never past it. -/// [[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]] -/// /// The decision is pure classification on memory arithmetic — no model is /// loaded, no inference is run. -pub fn plan_serving_with_demand( +pub fn plan_serving( host: HostBudget, candidates: &[ModelFootprint], - demand_lanes: u32, - demand_ceil: u32, + demand: ServingDemand, ) -> Option { + let demand_lanes = demand.lanes; if candidates.is_empty() { return None; } @@ -465,19 +506,66 @@ pub fn plan_serving_with_demand( .min(host.perf_cores.max(1)) .min(MAX_LANES) .max(1); + // Each RESIDENT persona wants its OWN warm slot so its prefilled KV survives across turns + // (no LRU clobber → no ~10k cold re-prefill every turn — #266's whole latency story: 96% + // prefill, two of four citizens at 0% cache reuse when 4 minds shared 2 slots). Grow the + // lane count toward the resident population (`demand_lanes`, set from the persona floor), + // but ONLY while each slot still clears the full-turn usable floor: adding a slot that + // drops the per-slot window below one assembled turn trades a warm-but-blind mind for a + // warm one that can't see — the exact 4-lanes-@-6k starvation the 2026-07-17 revert caught + // (it clamped the CEILING; the real fix is gating the COUNT on this floor). So pick the + // LARGEST lane count whose per-slot window ≥ the floor; if even one slot can't clear it (a + // genuinely tiny host), fall back to 1 — honest starvation, surfaced downstream — never + // below. The floor is `BOOTSTRAP_WORKING_SET` (≈ a ~9k prompt + generation headroom), the + // SAME "one full turn" constant the demand cap uses (§`BOOTSTRAP_WORKING_SET`), deliberately + // the STABLE bootstrap value and NOT the moving measured p95: coupling the lane COUNT to a + // jittering demand signal is the 718-replan lane-flap that wedged three benchmark runs. The + // served WINDOW still refines with measurement (below); the slot COUNT rests on a fixed + // floor. THIS floor — not the `MAX_LANES` backstop — is the binding constraint (#266). let lanes = (1..=lane_cap) .rev() - .find(|&l| window_for(l as u64) > MIN_SERVE_CTX) + .find(|&l| window_for(l as u64) >= BOOTSTRAP_WORKING_SET) .unwrap_or(1); + // Over-subscription: more resident personas than warm slots the window floor permits. The + // remainder can't get a persistent slot — with N minds on M Option { - plan_serving_with_demand(host, candidates, demand_lanes, BOOTSTRAP_WORKING_SET) -} - /// Hysteresis wrapper around [`plan_serving`]: stops model THRASH from live- /// budget jitter. Keeps the `incumbent` model as long as it still fits the /// budget — switching DOWN only when the incumbent no longer fits (forced /// eviction) and UP only when a strictly more capable model fits with -/// [`SWITCH_UP_HEADROOM`] to spare. Lanes + resident count always re-track the -/// current budget. No incumbent (or it's gone / no longer fits) → plain +/// [`SWITCH_UP_HEADROOM`] to spare. +/// +/// Lanes and window are sized against the AT-REST budget (the live budget with the +/// incumbent's own resident weights credited back) for as long as the incumbent keeps +/// serving — so a model's own KV can never convince the planner to shed the lane that model +/// is currently running. That self-eviction was a real lane flap, not a theoretical one; see +/// the note in the body. No incumbent (or it's gone / no longer fits) → plain /// [`plan_serving`]. Use this for the ONGOING serving loop; boot uses /// `plan_serving` directly (no incumbent yet). pub fn plan_serving_stable( host: HostBudget, candidates: &[ModelFootprint], incumbent: Option<&str>, - demand_lanes: u32, - demand_ceil: u32, + demand: ServingDemand, ) -> Option { // NB: do NOT `?`-bail here. A deep transient dip can leave `plan_serving` // with nothing fitting the depressed budget (`fresh` = None) while a model // is STILL resident and serving fine — its memory is its own. Tearing that // down to "nothing" is the exact harm we're guarding against, so `fresh` is // an Option we fall back to only when the incumbent genuinely can't hold. - let fresh = plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil); + let fresh = plan_serving(host, candidates, demand); let Some(inc_id) = incumbent else { return fresh; }; - // Fresh already chose the incumbent (or nothing else fits and fresh IS the - // incumbent) → nothing to stabilize. - if fresh.as_ref().map(|p| p.base_model_id.as_str()) == Some(inc_id) { - return fresh; - } + // NOTE (2026-08-04): there used to be an early return here — "fresh already chose the + // incumbent → nothing to stabilize" — and it was the lane-flap bug. `fresh` is computed + // against the LIVE budget, which the incumbent's own weights + KV depress while it serves. + // On the same-model path that meant the planner re-derived lane count from a budget the + // incumbent itself had eaten, decided it could no longer afford the lane it was already + // running, dropped to 1, then added it back when the KV freed. Glass-boxed on an IDLE host: + // 718 replans in one solve, `usable_gb` swinging 26→6, `lanes` oscillating 1↔2, and every + // flip resizing the live admission semaphore (`set_served_lane_count`) and the prefill + // throttle under in-flight requests — which is the `no response headers for 300s` lane wedge + // that killed three benchmark runs. + // + // The at-rest credit below already existed for exactly this reason ("so a model's OWN + // load/residency can never flap it out"); it was simply never applied to the case where the + // incumbent KEEPS serving. So the paths are unified: whenever a living incumbent is still + // servable, its plan is sized against the at-rest budget. Same-model is no longer a + // shortcut — it is the main path. [[never-thrash-sticky-hysteresis-on-every-lane]] + // // Incumbent dropped off disk entirely → honour whatever `fresh` chose // (possibly None = nothing servable). let Some(inc) = candidates.iter().find(|m| m.model_id == inc_id) else { @@ -623,7 +713,7 @@ pub fn plan_serving_stable( if let Some(m) = promoted.iter_mut().find(|m| m.model_id == inc_id) { m.capability_rank = u8::MAX; } - plan_serving_with_demand(at_rest, &promoted, demand_lanes, demand_ceil) + plan_serving(at_rest, &promoted, demand) } fn bytes_gb(bytes: u64) -> f64 { @@ -702,7 +792,7 @@ mod tests { // budget the fixpoint consumed for the chosen model. let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); + let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); let chosen_cost = devstral.weights_bytes + devstral.kv_at(plan.served_context_window) * plan.lanes as u64 + devstral.prefill_compute_reserve(plan.served_context_window, plan.lanes); @@ -730,12 +820,13 @@ mod tests { fn served_window_footprint_fits_effective_budget_including_window_scaled_compute() { let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); // ~112 KiB/token KV - // Demand = 4 personas, but MAX_LANES caps it (reverted to 2 after 4 lanes starved - // the window to ~6k < a ~9k persona prompt). The window is derived to fit whatever - // lane count is served — the invariant below holds at any cap. - let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); + // Demand = 4 personas. This roomy 48GB host clears the full-turn window floor at 4 + // slots (#266: a warm slot per resident persona), so all 4 are served — the window is + // derived to fit whatever lane count is served, and the fit invariant below holds at + // any count. + let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); assert!(plan.fits_on_gpu, "{}", plan.rationale); - assert_eq!(plan.lanes, MAX_LANES, "demand above the cap clamps to MAX_LANES"); + assert_eq!(plan.lanes, 4, "roomy host gives each of the 4 resident personas its own warm slot"); let c = plan.served_context_window as u64; let lanes = plan.lanes as u64; let compute_floor = devstral.compute_buffer_per_lane(); @@ -763,20 +854,131 @@ mod tests { // #234/#56 — the good governor recognizing its own overload, never a silent clamp. #[test] fn demand_over_local_capacity_surfaces_grid_overflow_not_silent_cram() { - let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + // 26GB: enough for the 14GB weights + 2 warm slots at a full-turn window, but NOT 4 — + // the window floor (#266) caps warm slots at 2, so 2 of the 4 resident personas can't + // get a persistent slot locally. That excess is the honest overflow, surfaced (not + // crammed onto shared slots to thrash) for the governor to place off-box. + let host = HostBudget { usable_bytes: 26 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - // 4 personas demand a lane; only MAX_LANES fit locally → the rest is overflow. - let over = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); - assert_eq!(over.lanes, MAX_LANES, "precondition: local lanes clamp to MAX_LANES"); + let over = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + assert_eq!(over.lanes, 2, "precondition: the full-turn window floor caps warm slots at 2 here"); assert_eq!( over.grid_overflow_lanes, 4 - over.lanes, - "demand the local lanes couldn't absorb must be surfaced for grid placement, not crammed" + "demand the local warm slots couldn't absorb must be surfaced for grid placement, not crammed" ); // Demand within local capacity → zero overflow (nothing to place off-box). - let fits = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); + let fits = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(1, None)).unwrap(); assert_eq!(fits.lanes, 1, "precondition: single demand fits one local lane"); - assert_eq!(fits.grid_overflow_lanes, 0, "demand ≤ local lanes → no overflow"); + assert_eq!(fits.grid_overflow_lanes, 0, "demand ≤ local warm slots → no overflow"); + } + + // what this catches: #266 — the slot count sizes to the RESIDENT PERSONA POPULATION so + // each mind keeps a persistent warm slot (its prefilled KV survives across turns), clamped + // by the full-turn window floor. The pre-fix `MAX_LANES = 2` clamp forced 4 resident minds + // onto 2 slots, and the per-slot LRU eviction re-prefilled a cold ~10k prefix every turn + // (measured: 96% prefill, two of four citizens at 0% cache reuse). Two branches, both pinned: + // (a) a budget that clears the floor at 4 slots → all 4 resident minds get a warm slot; + // (b) a budget that clears it at only 2 → 2 warm slots + the excess VISIBLE as overflow, + // never silently 4-crammed-onto-2 and never shrunk below the floor to fit more. + #[test] + fn slots_size_to_resident_population_capped_by_the_full_turn_window_floor() { + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + // (a) Roomy: 48GB clears the full-turn floor at 4 slots → a warm slot per resident mind, + // zero overflow, no thrash. This is the win the `MAX_LANES = 2` clamp used to forbid. + let roomy = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + let warm = plan_serving(roomy, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + assert_eq!(warm.lanes, 4, "roomy host: one warm slot per resident persona"); + assert_eq!(warm.grid_overflow_lanes, 0, "all 4 minds hosted locally → nothing over-subscribed"); + + // (b) Floor-limited: 26GB clears the floor at only 2 slots. The plan yields 2 (never 4 + // crammed onto 2), surfaces the 2 unslotted minds as overflow (the non-silent + // over-subscription signal a probe also names at the decision), and — critically — does + // NOT drop the per-slot window below the floor to squeeze 4 in. + let tight = HostBudget { usable_bytes: 26 * GB, perf_cores: 10 }; + let capped = plan_serving(tight, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); + assert_eq!(capped.lanes, 2, "window floor caps warm slots at 2 — the honest ceiling, not a silent cram"); + assert_eq!( + capped.grid_overflow_lanes, 2, + "the 2 minds that couldn't get a warm slot are SURFACED (probe + grid_overflow), never absorbed" + ); + assert!( + capped.served_context_window >= BOOTSTRAP_WORKING_SET, + "each served slot keeps a full-turn window ({}) — the floor is never breached to fit more, got {}", + BOOTSTRAP_WORKING_SET, + capped.served_context_window, + ); + } + + // what this catches: the cap that outlived its own TODO. `BOOTSTRAP_WORKING_SET` + // was written as a cold-start PRIOR to be superseded by measurement ("measured p95 + // later, #234"); the measurement never arrived, so on a host that could serve 94k + // of a 131k-capable model every citizen got 16384/lanes = **8192 tokens**, and + // measured 2026-08-06 that left a median context budget of 55 after framing and + // conversation — the work board reached a prompt 0 times in 495. A measured demand + // ABOVE the bootstrap prior must now actually raise the served window, or the + // constant is still silently in charge and this whole seam is decoration. + #[test] + fn a_measured_demand_above_the_cold_start_prior_actually_raises_the_window() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + let cold = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(1, None)) + .expect("servable"); + assert_eq!( + cold.served_context_window, BOOTSTRAP_WORKING_SET, + "with NO measurement the cold-start prior is the honest answer" + ); + + // One mind measured wanting 48k — well past the prior, well under the ceiling. + let measured = plan_serving( + host, + std::slice::from_ref(&devstral), + ServingDemand::new(1, Some(48_000)), + ) + .expect("servable"); + assert!( + measured.served_context_window > cold.served_context_window, + "measured demand ({}) must RAISE the window above the cold-start prior ({}), \ + got {} — if this is equal, the constant is still the authority", + 48_000, + cold.served_context_window, + measured.served_context_window + ); + assert!( + measured.served_context_window <= 48_000, + "…but never above what was actually demanded (got {})", + measured.served_context_window + ); + } + + // what this catches: the other direction — demand is a CAP, not a request the host + // must honor. A mind that wants more than the machine has must receive what fits, + // never a window the host cannot back with real KV, which is the swap/wedge the + // demand cap exists to prevent in the first place. + #[test] + fn demand_beyond_the_host_is_bounded_by_what_actually_fits() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + let greedy = plan_serving( + host, + std::slice::from_ref(&devstral), + ServingDemand::new(4, Some(10_000_000)), + ) + .expect("servable"); + assert!( + greedy.served_context_window <= devstral.context_window, + "never above the model's trained ceiling (got {})", + greedy.served_context_window + ); + let unbounded_fit = + plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, Some(u32::MAX))) + .expect("servable"); + assert_eq!( + greedy.served_context_window, unbounded_fit.served_context_window, + "past the host's real fit, MORE demand changes nothing — the fit is the bound" + ); } // what this catches: the cross-node serving-QUALITY bug (2026-07-26). On a roomy @@ -796,7 +998,7 @@ mod tests { devstral.context_window > BOOTSTRAP_WORKING_SET, "precondition: model ceiling must exceed the demand cap, else the ceiling (not the cap) could explain the result" ); - let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); + let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); assert!( plan.served_context_window <= BOOTSTRAP_WORKING_SET, "served window {} must be capped to working-set demand ({}), never budget-maxed toward the ceiling", @@ -808,7 +1010,7 @@ mod tests { // whose trained ceiling is below the demand cap is still served at/below its // own ceiling (`.min` only ever caps DOWN). let tiny = fp("tiny-4k", 3, 30_000, 4_096, 2); - let plan_tiny = plan_serving(host, std::slice::from_ref(&tiny), 1).unwrap(); + let plan_tiny = plan_serving(host, std::slice::from_ref(&tiny), ServingDemand::new(1, None)).unwrap(); assert!( plan_tiny.served_context_window <= 4_096, "demand cap must not inflate a 4k-ceiling model past its ceiling; got {}", @@ -816,70 +1018,6 @@ mod tests { ); } - // what this catches: the ELASTIC demand ceiling (Joel 2026-07-27: "context window sizes - // should ebb and flow depending on demands of the task and available resources — if it - // needs it larger for a moment, don't limit it"). The ceiling is threaded LIVE, not a - // launch-baked constant: a hard task passing a HIGH demand_ceil grows the served window - // PAST the BOOTSTRAP prior (up to the budget/model bound); a LOW ceiling shrinks it so - // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises - // the cap toward the budget bound, never past it. This is the "stop setting it in stone - // at launch" fix that opens the elastic-lease path. - // what this catches: the serving→grid bridge (#180 spill) — a model footprint maps to a - // capacity-fabric LeaseRequest whose want_concurrency is the demanded lanes and whose - // spike_bytes is ONE lane's transient prefill compute reserve at the served window (the - // 2026-07-14 OOM term), so GridPlacementPolicy offers a peer only a spill lane it can hold. - #[test] - fn grid_lease_request_maps_demand_and_the_prefill_spike() { - let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let window = 16_384; - let lease = devstral.grid_lease_request(window, 3); - assert_eq!(lease.consumer, "devstral-24b"); - assert_eq!(lease.want_concurrency, 3, "want_concurrency = demanded lanes"); - assert_eq!( - lease.spike_bytes, - devstral.prefill_compute_reserve(window, 1), - "spike_bytes = ONE lane's prefill compute reserve at the served window" - ); - // Zero demand floors at one lane — never a degenerate 0-concurrency lease. - assert_eq!(devstral.grid_lease_request(window, 0).want_concurrency, 1); - } - - #[test] - fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { - let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; - // Roomy host + high trained ceiling → window_for(1) far exceeds any of these ceilings, - // so the DEMAND ceiling (not the budget or the model) decides the served window. - let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - - // Cold prior: default plan_serving caps at BOOTSTRAP_WORKING_SET. - let cold = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); - assert_eq!(cold.served_context_window, BOOTSTRAP_WORKING_SET); - - // Hard task demands more context → the window GROWS past the prior. - let big_ceil = 64_000; - let hot = - plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, big_ceil).unwrap(); - assert!( - hot.served_context_window > cold.served_context_window, - "a higher demand ceiling must GROW the window: hot {} ≤ cold {}", - hot.served_context_window, - cold.served_context_window - ); - assert!( - hot.served_context_window <= big_ceil, - "growth stays bounded by the demand ceiling (and the budget), never past it: got {}", - hot.served_context_window - ); - - // Simple turn demands little → the window SHRINKS below the prior, freeing memory. - let lean = - plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, 8_192).unwrap(); - assert_eq!( - lean.served_context_window, 8_192, - "a low demand ceiling shrinks the served window to it" - ); - } - // what this catches: lanes DEGRADE on a tight host — MAX_LANES is a sanity backstop, // the fit math is the real cap. A 4-persona demand on a budget that can't feed 4 warm // slots must serve FEWER (well-fed) lanes, never 4 starving ones that OOM. The window @@ -890,7 +1028,7 @@ mod tests { // ~18GB usable — fits the 14GB weights + a couple lanes' KV, not four. let host = HostBudget { usable_bytes: 18 * GB, perf_cores: 10 }; let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); - let plan = plan_serving(host, std::slice::from_ref(&devstral), 4).unwrap(); + let plan = plan_serving(host, std::slice::from_ref(&devstral), ServingDemand::new(4, None)).unwrap(); assert!(plan.fits_on_gpu, "{}", plan.rationale); assert!(plan.lanes < 4, "tight host must serve fewer than the 4 demanded: got {}", plan.lanes); assert!(plan.lanes >= 1); @@ -912,7 +1050,7 @@ mod tests { fn tiny_box_picks_most_capable_that_fits_not_the_biggest() { // ~5.5GB usable after OS headroom on an 8GB Air. let host = HostBudget { usable_bytes: 5 * GB + 500 * 1_000_000, perf_cores: 4 }; - let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); + let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); assert!(plan.fits_on_gpu, "must fit a real model on GPU: {}", plan.rationale); assert_eq!(plan.base_model_id, "qwen3.5-4b", "14B can't fit 5.5GB; 4B is the most capable that does"); assert!(plan.lanes >= 1); @@ -925,11 +1063,14 @@ mod tests { // budget, demand honored → each mind's window roughly doubles. #[test] fn lanes_track_demand_and_every_unneeded_lane_stops_costing_window() { - // A pressured budget (benchmark servers breathing next door). + // A pressured budget (benchmark servers breathing next door). Demand a budget-bound + // window (Some(u32::MAX)) so the "unneeded lane costs window" invariant is visible: + // under the default demand cap both counts would hit the same cap and the window + // difference would be masked — the invariant lives in the budget-bound regime. let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; - let greedy = plan_serving(host, &candidates(), MAX_LANES).unwrap(); - let demand2 = plan_serving(host, &candidates(), 2).unwrap(); - assert_eq!(demand2.lanes, 2, "2 minds → 2 lanes, not the ceiling"); + let greedy = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, Some(u32::MAX))).unwrap(); + let demand2 = plan_serving(host, &candidates(), ServingDemand::new(2, Some(u32::MAX))).unwrap(); + assert_eq!(demand2.lanes, 2, "2 minds → 2 lanes, never the MAX_LANES ceiling"); if greedy.lanes > 2 { assert!( demand2.served_context_window > greedy.served_context_window, @@ -941,10 +1082,10 @@ mod tests { ); } // Demand can never exceed the physical caps (kv/perf/MAX_LANES)… - let demand99 = plan_serving(host, &candidates(), 99).unwrap(); + let demand99 = plan_serving(host, &candidates(), ServingDemand::new(99, None)).unwrap(); assert!(demand99.lanes <= MAX_LANES); // …and a zero demand is defensively floored at one lane. - assert_eq!(plan_serving(host, &candidates(), 0).unwrap().lanes, 1); + assert_eq!(plan_serving(host, &candidates(), ServingDemand::new(0, None)).unwrap().lanes, 1); } // what this catches: #213 — the daemon must not chase a lane-count target off a cliff. @@ -960,7 +1101,7 @@ mod tests { let devstral = fp("devstral-24b", 14, 175_000, 131_072, 3); // Budget where ONE lane serves a real window but TWO would floor (eval lane resident). let squeezed = HostBudget { usable_bytes: 19 * GB, perf_cores: 6 }; - let plan = plan_serving(squeezed, std::slice::from_ref(&devstral), 2).unwrap(); + let plan = plan_serving(squeezed, std::slice::from_ref(&devstral), ServingDemand::new(2, None)).unwrap(); assert_eq!(plan.lanes, 1, "2 lanes would floor → shed to 1 real lane, not 2 @ 2048"); assert!( plan.served_context_window > 4096, @@ -972,7 +1113,7 @@ mod tests { // (2026-07-26), NOT maximized to fill RAM (the 94k→swap/wedge bug). A roomy // box buys more LANES (concurrent minds), not a bloated per-lane KV cache. let roomy = HostBudget { usable_bytes: 45 * GB, perf_cores: 6 }; - let plan2 = plan_serving(roomy, std::slice::from_ref(&devstral), 2).unwrap(); + let plan2 = plan_serving(roomy, std::slice::from_ref(&devstral), ServingDemand::new(2, None)).unwrap(); assert_eq!(plan2.lanes, 2, "roomy host serves both minds concurrently"); assert!( plan2.served_context_window > 4096 @@ -996,7 +1137,7 @@ mod tests { fn big_box_picks_most_capable_runs_lanes_but_sizes_window_to_demand() { // ~45GB usable on a 64GB M5 Pro after headroom. let host = HostBudget { usable_bytes: 45 * GB, perf_cores: 6 }; - let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); + let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); assert_eq!(plan.base_model_id, "coder-sentinel-14b", "most capable, fits easily"); assert!(plan.lanes >= 2, "M5 Pro has the budget for multiple lanes, got {}", plan.lanes); // Window sized to DEMAND (capped at the working-set bootstrap), NOT maximized @@ -1020,7 +1161,7 @@ mod tests { #[test] fn lanes_capped_at_max() { let host = HostBudget { usable_bytes: 500 * GB, perf_cores: 64 }; - let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); + let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); assert_eq!(plan.lanes, MAX_LANES); } @@ -1029,37 +1170,40 @@ mod tests { // sized at the MIN window; a fatter floor lane fits fewer times). #[test] fn fatter_kv_means_fewer_lanes() { - // Budget chosen so KV (not the MAX_LANES cap or perf cores) is the binding - // constraint, ABOVE the per-lane compute-buffer floor now in the fit math: - // 4GB total, 2GB weights → 2GB for (KV + compute buffer) per lane. - // lean floor ≈ 307MB KV + 256MB compute ≈ 563MB → 3 lanes; - // fat floor ≈ 921MB KV + 256MB compute ≈ 1.18GB → 1 lane. - let host = HostBudget { usable_bytes: 4 * GB, perf_cores: 8 }; - let lean = plan_serving(host, &[fp("lean", 2, 150_000, 32_768, 5)], MAX_LANES).unwrap(); - let fat = plan_serving(host, &[fp("fat", 2, 450_000, 32_768, 5)], MAX_LANES).unwrap(); + // Budget chosen so KV (not the MAX_LANES backstop or perf cores) is the binding + // constraint: each warm slot must clear the full-turn window floor (16384, #266), so a + // fatter per-token KV rate makes fewer slots clear it. 16GB total, 2GB weights → ~11.6GB + // (after co-consumer headroom) for (KV + compute buffer) across lanes at a full-turn + // window: lean (150k/tok) clears the floor at 3 slots; fat (450k/tok, 3× the KV) clears + // it at only 1. (A 4GB host would floor BOTH to 1 lane — too small to show the effect.) + let host = HostBudget { usable_bytes: 16 * GB, perf_cores: 8 }; + let lean = plan_serving(host, &[fp("lean", 2, 150_000, 32_768, 5)], ServingDemand::new(MAX_LANES, None)).unwrap(); + let fat = plan_serving(host, &[fp("fat", 2, 450_000, 32_768, 5)], ServingDemand::new(MAX_LANES, None)).unwrap(); assert!(lean.lanes > fat.lanes, "lean {} should beat fat {}", lean.lanes, fat.lanes); } - // what this catches: the plan CAPS lane count at the MAX_LANES safety ceiling AND - // reserves the concurrent compute buffers, so resident KV + those buffers fit the - // budget (no OOM by construction). MAX_LANES was raised to 6 on 2026-07-16 to give - // each persona a warm slot, then REVERTED to 2 same-day after it re-OOM'd at large - // windows — the transient prefill compute buffer scales with n_ctx, not weights, so a + // what this catches: the plan CAPS lane count at the binding constraint — the full-turn + // window floor (#266), not the raised MAX_LANES backstop — AND reserves the concurrent + // compute buffers, so resident KV + those buffers fit the budget (no OOM by construction). + // The transient prefill compute buffer scales with n_ctx, not weights, so a // window-independent reserve under-provisions and 4 concurrent large-window prefills - // overflow. Whatever the ceiling, the fit invariant below must hold. + // overflow. Whatever the cap, the fit invariant below must hold. #[test] - fn lane_count_respects_the_safety_ceiling_and_reserves_compute_buffers() { + fn lane_count_respects_the_window_floor_and_reserves_compute_buffers() { // 24B-class: 13.6GB weights, kv_per_token ~156KB/token (measured), ~26GB usable. let m = fp("devstral-24b", 13, 156_000, 131_072, 9); let host = HostBudget { usable_bytes: 26 * GB, perf_cores: 10 }; - // 4 personas demand 4 lanes, but the MAX_LANES safety ceiling caps it. - let plan = plan_serving(host, std::slice::from_ref(&m), 4).unwrap(); + // 4 personas demand 4 lanes, but only 2 slots clear the full-turn window floor on this + // budget — the window floor (below MAX_LANES=8) is the binding cap. The other 2 minds + // surface as grid_overflow rather than thrashing 4 minds across 2 clobbering slots. + let plan = plan_serving(host, std::slice::from_ref(&m), ServingDemand::new(4, None)).unwrap(); assert_eq!( - plan.lanes, MAX_LANES, - "demand is capped to the MAX_LANES safety ceiling: {}", + plan.lanes, 2, + "the full-turn window floor (not the MAX_LANES backstop) caps warm slots here: {}", plan.rationale ); + assert_eq!(plan.grid_overflow_lanes, 2, "the 2 unslotted minds are surfaced for grid placement"); // The plan FITS: weights + lanes×KV@window + lanes×compute buffer ≤ budget — the // invariant the KV-only math violated (it left no room for the buffers). @@ -1077,7 +1221,7 @@ mod tests { #[test] fn nothing_fits_degrades_honestly_no_silent_cpu() { let host = HostBudget { usable_bytes: 300 * 1_000_000, perf_cores: 2 }; // 0.3GB - let plan = plan_serving(host, &candidates(), MAX_LANES).unwrap(); + let plan = plan_serving(host, &candidates(), ServingDemand::new(MAX_LANES, None)).unwrap(); assert!(!plan.fits_on_gpu, "must report the GPU budget can't hold any candidate"); assert_eq!(plan.base_model_id, "qwen2.5-0.5b", "names the smallest as the only option"); assert_eq!(plan.lanes, 1); @@ -1087,7 +1231,7 @@ mod tests { #[test] fn no_candidates_is_none() { let host = HostBudget { usable_bytes: 45 * GB, perf_cores: 6 }; - assert!(plan_serving(host, &[], MAX_LANES).is_none()); + assert!(plan_serving(host, &[], ServingDemand::new(MAX_LANES, None)).is_none()); } // ── hysteresis (plan_serving_stable) ────────────────────────────────── @@ -1106,8 +1250,8 @@ mod tests { fn stable_with_no_incumbent_equals_plain() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; assert_eq!( - plan_serving_stable(host, &pair(), None, MAX_LANES, BOOTSTRAP_WORKING_SET), - plan_serving(host, &pair(), MAX_LANES) + plan_serving_stable(host, &pair(), None, ServingDemand::new(MAX_LANES, None)), + plan_serving(host, &pair(), ServingDemand::new(MAX_LANES, None)) ); } @@ -1118,18 +1262,70 @@ mod tests { fn stable_keeps_incumbent_when_upgrade_lacks_headroom() { // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. let host = HostBudget { usable_bytes: 10 * GB, perf_cores: 6 }; - assert_eq!(plan_serving(host, &pair(), MAX_LANES).unwrap().base_model_id, "big", "fresh would pick big"); - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); + assert_eq!(plan_serving(host, &pair(), ServingDemand::new(MAX_LANES, None)).unwrap().base_model_id, "big", "fresh would pick big"); + let stable = plan_serving_stable(host, &pair(), Some("small"), ServingDemand::new(MAX_LANES, None)).unwrap(); assert_eq!(stable.base_model_id, "small", "hysteresis keeps incumbent — no flap"); assert!(stable.lanes >= 1, "lanes still re-tracked for the kept model"); } + // what this catches: THE LANE FLAP — a serving model self-evicting its own lane. + // `plan_serving_stable` used to early-return the FRESH plan whenever fresh chose the + // incumbent, and fresh is computed against the LIVE budget, which the incumbent's own + // weights + KV depress while it serves. So the planner kept re-deciding it could not + // afford the lane it was already running, dropped to 1, then re-added it when the KV + // freed. Measured on an idle host: 718 replans in ONE solve, lanes oscillating 1↔2, each + // flip resizing the live admission semaphore and prefill throttle under in-flight + // requests — the `no response headers for 300s` wedge that killed three benchmark runs. + // + // Same budget, same model, only difference is whether the incumbent is declared: the + // stable plan must NOT serve fewer lanes than the plan that boot would have made at rest. + #[test] + fn a_serving_model_never_sheds_its_own_lane_to_its_own_residency() { + // 9GB model on an 18GB host — the measured flap zone. At rest it plans (2 lanes, + // 16384). Once it is SERVING, its own 9GB reads as "used", and a fresh plan against + // that depressed budget returns (1 lane, 2048): its own residency costs it a lane AND + // 87% of its window, so the next tick (KV freed) plans it straight back up. That is + // the oscillation, and every flip resizes the live admission semaphore + prefill + // throttle under in-flight requests. + let models = vec![fp("big", 9, 90_000, 262_144, 3)]; + let at_rest = HostBudget { usable_bytes: 18 * GB, perf_cores: 6 }; + let boot = plan_serving(at_rest, &models, ServingDemand::new(MAX_LANES, None)).expect("servable at rest"); + + let live = HostBudget { + usable_bytes: at_rest.usable_bytes - models[0].weights_bytes, + perf_cores: 6, + }; + let fresh = plan_serving(live, &models, ServingDemand::new(MAX_LANES, None)).expect("still servable"); + // Guard the guard: if this ever stops being a flap, the test below proves nothing. + assert!( + fresh.lanes < boot.lanes, + "fixture no longer reproduces the flap (boot={} fresh={}) — re-derive the budget", + boot.lanes, + fresh.lanes + ); + + let stable = plan_serving_stable(live, &models, Some("big"), ServingDemand::new(MAX_LANES, None)) + .expect("incumbent still servable"); + assert_eq!(stable.base_model_id, "big"); + assert_eq!( + stable.lanes, boot.lanes, + "a model's OWN residency must not shrink its OWN lane count (stable={} boot={} \ + fresh={})", + stable.lanes, boot.lanes, fresh.lanes + ); + assert_eq!( + stable.served_context_window, boot.served_context_window, + "nor its own window (stable={} boot={} fresh={})", + stable.served_context_window, boot.served_context_window, fresh.served_context_window + ); + } + // what this catches: a genuine upgrade DOES happen when the better model // fits with headroom — hysteresis isn't a permanent lock-in. #[test] fn stable_upgrades_when_better_model_fits_with_headroom() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; // big 9.7 << 0.9*20=18 - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), ServingDemand::new(MAX_LANES, None)).unwrap(); assert_eq!(stable.base_model_id, "big", "more capable + ample headroom → upgrade"); } @@ -1144,7 +1340,7 @@ mod tests { fn stable_forced_down_when_incumbent_gone_from_disk() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; let only_small = vec![fp("small", 1, 4_000, 32_768, 1)]; // "big" no longer on disk - let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); + let stable = plan_serving_stable(host, &only_small, Some("big"), ServingDemand::new(MAX_LANES, None)).unwrap(); assert_eq!(stable.base_model_id, "small", "incumbent gone from disk → serve what's present"); } @@ -1163,12 +1359,12 @@ mod tests { // Plain plan at the depressed budget WOULD flap: big (9GB) no longer "fits" // 8GB, so fresh prefers the smaller model. assert_eq!( - plan_serving(dipped, &pair(), MAX_LANES).unwrap().base_model_id, + plan_serving(dipped, &pair(), ServingDemand::new(MAX_LANES, None)).unwrap().base_model_id, "small", "depressed-budget plain plan would flap to the smaller model" ); // With the incumbent credited its own weights back, the resident big stays. - let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); + let stable = plan_serving_stable(dipped, &pair(), Some("big"), ServingDemand::new(MAX_LANES, None)).unwrap(); assert_eq!(stable.base_model_id, "big", "incumbent survives its OWN load dip — no flap"); assert!(stable.lanes >= 1, "kept model still gets ≥1 lane"); }