diff --git a/core/continuum-core/src/airc/inbound_attach.rs b/core/continuum-core/src/airc/inbound_attach.rs index 061f5d5cd9..791685df7a 100644 --- a/core/continuum-core/src/airc/inbound_attach.rs +++ b/core/continuum-core/src/airc/inbound_attach.rs @@ -212,6 +212,31 @@ pub async fn publish_transcript_event( "first capacity offer heard from a grid peer", ); } + } else if let Some(beacon) = residency_beacon_from_envelope(&envelope) { + // Residency beacon (grid-overflow eligibility): fold the heard beacon into the + // process-global residency ledger, keyed on the WIRE's peer id — the orthogonal + // sibling of the capacity fold above. Our own echo lands here too (the loopback + // proof that publish→hear works before a second node exists). + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let model_count = beacon.resident_models.len(); + let is_new = crate::capacity::model_residency::global_residency_ledger().hear( + event.peer_id.as_uuid(), + beacon, + now_ms, + ); + if is_new { + crate::probe!( + class = "grid.residency.heard", + from_peer = %event.peer_id.as_uuid(), + model_count = model_count, + heard_peers = + crate::capacity::model_residency::global_residency_ledger().heard_count(), + "first residency beacon heard from a grid peer", + ); + } } else if let Some((name, payload)) = chat_posted_from_envelope(&envelope, event) { crate::probe!( class = "airc.chat.projected", @@ -284,6 +309,22 @@ fn capacity_offer_from_envelope( serde_json::from_value(payload.inline.clone()?).ok() } +/// Decode a `grid_residency` envelope's inline payload into a [`ResidencyBeacon`]. +/// Returns `None` for any other envelope — the residency sibling of +/// [`capacity_offer_from_envelope`], same honest schema gate. +fn residency_beacon_from_envelope( + envelope: &AircRealtimeEnvelope, +) -> Option { + let crate::airc::realtime::AircRealtimePayload::ExistingSchema { payload } = &envelope.payload + else { + return None; + }; + if payload.schema != crate::airc::realtime::AircRealtimeSchema::GridResidency { + return None; + } + serde_json::from_value(payload.inline.clone()?).ok() +} + /// Project a plain airc chat message into the THIN `chat:posted` bus /// payload the positron chat projection consumes (`AircChatPosted` in /// `ipc/positron_source.rs`). Returns `None` for any non-message event diff --git a/core/continuum-core/src/airc/realtime.rs b/core/continuum-core/src/airc/realtime.rs index 1d412b1a79..8606c64a30 100644 --- a/core/continuum-core/src/airc/realtime.rs +++ b/core/continuum-core/src/airc/realtime.rs @@ -53,6 +53,13 @@ pub enum AircRealtimeSchema { /// presence-of-compute. EphemeralCoalesced: latest wins, never replayed /// (a stale capacity reading is a lie). GridCapacity, + /// A node's residency beacon (`capacity::model_residency::ResidencyBeacon`) — + /// which models it holds resident, the grid-overflow ELIGIBILITY signal (a peer + /// is only a fast overflow target for a model it already holds). Orthogonal to + /// GridCapacity (concurrency): the governor composes the two. EphemeralCoalesced + /// like capacity, but published on a slower cadence — residency changes on model + /// page-in/out (minute-scale), not the 10s capacity beat. + GridResidency, } /// Handle to a payload already defined by a Continuum schema. @@ -458,7 +465,9 @@ impl AircRealtimePayload { | AircRealtimeSchema::LiveKitBridgeEvent => AircRealtimeDelivery::Control, // Capacity offers are presence-of-compute: latest wins, never // replayed — a stale reading must not outlive its freshness. - AircRealtimeSchema::GridCapacity => AircRealtimeDelivery::EphemeralCoalesced, + AircRealtimeSchema::GridCapacity | AircRealtimeSchema::GridResidency => { + AircRealtimeDelivery::EphemeralCoalesced + } _ => AircRealtimeDelivery::Durable, }, Self::Presence { event } => event.delivery(), diff --git a/core/continuum-core/src/bin/expert_observe.rs b/core/continuum-core/src/bin/expert_observe.rs new file mode 100644 index 0000000000..558e731222 --- /dev/null +++ b/core/continuum-core/src/bin/expert_observe.rs @@ -0,0 +1,342 @@ +//! expert_observe — glass-box the LIVE MoE expert routing (#230 / #229). +//! +//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached, drives +//! REAL routing over a labelled multi-domain corpus, then reports the affinity that decides +//! the paging architecture (#180): PER-DOMAIN concentration + CROSS-DOMAIN hot-set overlap. +//! +//! ## Methodology (BigMama's three guardrails, 2026-07-27) +//! 1. **Sample size.** ~7 activations/slot is far too few to tell power-law from uniform — +//! Poisson noise swamps the skew. We drive thousands of tokens so the head separates. +//! 2. **Clean samples, not a greedy loop.** A long greedy generation degenerates and +//! OVER-routes to the same experts — a false positive for locality. We PREFILL diverse +//! prompts (prefill routes every token on realistic input) with only short generation. +//! 3. **Diverse HOW — the knife-edge.** Pooling ACROSS domains measures the global +//! mixed-workload histogram; MoE specialization routes domains to different experts, so +//! pooling SMEARS toward uniform and answers the WRONG question. The paging architecture +//! serves ONE persona doing ONE coherent thing, so we measure: +//! (a) PER-DOMAIN concentration — one observer per domain = the coherent-session +//! paging headroom (top-K% activation share WITHIN a domain). +//! (b) CROSS-DOMAIN overlap — intersect each domain's hot set. Experts hot in EVERY +//! domain = the always-resident shared base (never paged); experts hot in ONE +//! domain = the pageable specialization tier. The money number: what fraction of a +//! domain's activation MASS lands on experts cold for the other domains — the +//! eviction win. +//! +//! Usage: +//! cargo run --release -p continuum-core --features metal,accelerate --bin expert_observe -- [n_gen_per_prompt] + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::Arc; + +use continuum_core::capacity::expert_observer::LiveExpertObserver; +use llama::{Batch, ContextParams, ExpertObserver, Model, ModelParams, Sampler}; + +/// Labelled corpus: each domain gets its OWN observer, fed several diverse prompts so the +/// per-domain sample is dense. The point is a COHERENT workload per observer (what a persona +/// actually does in a session), not a global mixture. +const DOMAINS: &[(&str, &[&str])] = &[ + ( + "code", + &[ + "Implement a lock-free single-producer single-consumer ring buffer in Rust using \ + atomics with Release/Acquire ordering; explain why a SeqCst fence is unnecessary \ + and how the head/tail indices wrap a power-of-two capacity without a modulo in the \ + hot path.\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\n", + "Write a React hook useDebouncedResource taking an async fetcher and debounce \ + interval that cancels in-flight requests on key change, dedupes concurrent callers, \ + and surfaces loading/error/stale-while-revalidate state without tearing under \ + concurrent mode.\n\nimport { useEffect, useRef, useState } from 'react';\n", + "Given orders(id, customer_id, placed_at, total) and refunds(order_id, amount, \ + refunded_at), write SQL returning each customer's net revenue by month, excluding \ + months with over 40% refunded, ranking customers within each month by a window \ + function.", + ], + ), + ( + "prose", + &[ + "The lighthouse keeper had not spoken to another person in forty-one days when the \ + boat appeared on the horizon. He noticed first the wrongness in the grey, and only \ + afterward resolved the shape into a hull riding low, and set down the brass polish \ + and went to the door, the cold coming up through the stone under his socks.", + "She had rehearsed the apology on the train, each version softer than the last, until \ + the words lost their edges and became a kind of weather she carried into the room. \ + Her mother was at the sink with her back turned, and for a moment neither of them \ + moved, and the tap ran over a single white plate.", + "Write the opening of a short story about a cartographer who discovers that a river \ + on his oldest map no longer exists, and who sets out on foot to find where it went, \ + narrated in close third person with attention to the texture of the walking.", + ], + ), + ( + "math", + &[ + "Prove that every finite integral domain is a field. Fix a nonzero a and consider \ + x -> a x; show injectivity, conclude surjectivity from finiteness, hence an inverse \ + of a exists. Then exhibit an infinite integral domain that is not a field to show \ + finiteness is essential.", + "Derive the closed form for the variance of a sum of two correlated random variables \ + in terms of their individual variances and covariance, then generalize to n \ + variables and interpret the cross terms as the reason diversification reduces \ + portfolio variance.", + "State and prove the pigeonhole principle, then use it to show that among any 51 \ + integers chosen from 1 to 100 there must be two that are coprime, and separately two \ + whose difference is exactly 10.", + ], + ), + ( + "science", + &[ + "Explain how the sodium-potassium pump maintains a neuron's resting potential: three \ + sodium out for two potassium in, the ATP-driven conformational change, and how the \ + electrogenic imbalance plus leak channels and the Nernst equilibria set the roughly \ + -70 mV resting potential.", + "Describe why the sky is blue in terms of Rayleigh scattering: the inverse fourth \ + power wavelength dependence, why shorter wavelengths scatter more strongly, and why \ + sunsets are red because of the longer atmospheric path length near the horizon.", + "Explain the greenhouse effect at the level of molecular physics: which atmospheric \ + gases absorb in the infrared, why their vibrational modes couple to outgoing \ + longwave radiation while nitrogen and oxygen do not, and how re-emission warms the \ + surface.", + ], + ), + ( + "planning", + &[ + "Sketch the migration plan to decompose a Rust monolith into services over a \ + command-and-event bus: identify the seams where synchronous calls become async, how \ + you preserve transactional guarantees that relied on one process, and how you roll \ + out incrementally behind a facade without a big-bang cutover.", + "Write a JSON schema for a distributed-cache eviction policy supporting LRU, LFU, and \ + TTL tiers with per-key overrides, budgets in both bytes and percentage-of-pool, and \ + validation that every declared cache class has at least one eviction dimension so \ + none grows unbounded.", + "Draft a rollout plan for a feature flag that changes a checkout flow: staged \ + percentage ramp, the metrics that gate each stage, the automatic rollback trigger, \ + and how you keep the two code paths from diverging while the flag is live.", + ], + ), +]; + +fn drive_prompt(model: &Model, observer: &Arc, prompt: &str, n_gen: usize) { + let mut ctx = model + .new_context(ContextParams { + n_ctx: 4096, + n_batch: 512, + n_seq_max: 1, + expert_observer: Some(observer.clone() as Arc), + ..Default::default() + }) + .expect("context"); + + let tokens = model.tokenize(prompt, true, false).expect("tokenize"); + let mut n_cur: i32 = 0; + for chunk in tokens.chunks(512) { + let mut batch = Batch::allocated(512, 1); + let last_global = n_cur as usize + chunk.len() - 1; + for (i, tok) in chunk.iter().enumerate() { + let pos = n_cur + i as i32; + batch.push(*tok, pos, &[0], (n_cur as usize + i) == last_global); + } + ctx.decode(&batch).expect("prefill decode"); + n_cur += chunk.len() as i32; + } + + let mut sampler = Sampler::greedy(); + for _ in 0..n_gen { + let token = sampler.sample(&ctx, -1); + if model.is_eog_token(token) { + break; + } + let mut batch = Batch::allocated(1, 1); + batch.push(token, n_cur, &[0], true); + ctx.decode(&batch).expect("gen decode"); + n_cur += 1; + } +} + +/// Sorted-desc counts → activation share captured by the top `frac` of FIRED experts. +fn top_share(counts_desc: &[u64], total: u64, frac: f64) -> f64 { + if counts_desc.is_empty() || total == 0 { + return 0.0; + } + let k = ((counts_desc.len() as f64 * frac).ceil() as usize).max(1).min(counts_desc.len()); + let s: u64 = counts_desc.iter().take(k).sum(); + 100.0 * s as f64 / total as f64 +} + +fn main() { + let args: Vec = std::env::args().collect(); + let model_path = args + .get(1) + .expect("usage: expert_observe [n_gen_per_prompt]"); + let n_gen: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(48); + + let model = Model::load( + PathBuf::from(model_path), + ModelParams { n_gpu_layers: -1, use_mmap: true }, + ) + .expect("load model"); + println!("Loaded {model_path} (vocab={})", model.n_vocab()); + + // Drive each domain into its OWN observer → per-domain hit maps. + let mut per_domain = Vec::new(); + for (label, prompts) in DOMAINS { + let obs = LiveExpertObserver::new(); + for prompt in *prompts { + drive_prompt(&model, &obs, prompt, n_gen); + } + let hits = obs.snapshot_hits(); + println!( + " domain {label:<9} : {} activations across {} experts", + obs.total_hits(), + hits.len() + ); + per_domain.push((label.to_string(), hits)); + } + + // ---- PER-DOMAIN concentration (the coherent-session paging headroom) ---- + println!("\n=== PER-DOMAIN CONCENTRATION (top-K% share vs uniform null) ==="); + println!("{:<10} {:>8} {:>8} {:>8} {:>7} {:>7} {:>7}", "domain", "total", "fired", "mean", "top1%", "top10%", "top25%"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let mean = if counts.is_empty() { 0.0 } else { total as f64 / counts.len() as f64 }; + println!( + "{label:<10} {total:>8} {:>8} {mean:>8.1} {:>6.1}% {:>6.1}% {:>6.1}%", + counts.len(), + top_share(&counts, total, 0.01), + top_share(&counts, total, 0.10), + top_share(&counts, total, 0.25), + ); + } + println!("(uniform null → top1%≈1, top10%≈10, top25%≈25; materially above = per-session paging headroom)"); + + // ---- WORKING-SET SIZE: how many experts resident to capture X% of a domain's mass ---- + // THE engineering number for #180: the resident working set you must keep hot; the rest + // pages to CPU/disk (misses on the cold tail are low-mass = infrequent). + let experts_for_mass = |counts_desc: &[u64], total: u64, frac: f64| -> usize { + if total == 0 { return 0; } + let target = frac * total as f64; + let mut acc = 0u64; + for (i, c) in counts_desc.iter().enumerate() { + acc += c; + if acc as f64 >= target { return i + 1; } + } + counts_desc.len() + }; + println!("\n=== WORKING-SET SIZE (experts resident for X% of a domain's mass; % of fired) ==="); + println!("{:<10} {:>10} {:>12} {:>12} {:>12}", "domain", "50%mass", "80%mass", "90%mass", "95%mass"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let n = counts.len().max(1); + let cell = |f: f64| { let k = experts_for_mass(&counts, total, f); format!("{k} ({:.0}%)", 100.0 * k as f64 / n as f64) }; + println!("{label:<10} {:>10} {:>12} {:>12} {:>12}", cell(0.50), cell(0.80), cell(0.90), cell(0.95)); + } + println!("(the 50%-mass column = the tight resident set; 95%-mass = keep-everything-hot floor; the GAP is the pageable tail)"); + + // ---- CROSS-DOMAIN overlap: shared base vs pageable specialization ---- + // A domain's HOT SET = the smallest set of experts capturing 50% of that domain's mass. + let hot_set = |hits: &HashMap<_, u64>| -> HashSet<_> { + let total: u64 = hits.values().sum(); + let mut ranked: Vec<(_, u64)> = hits.iter().map(|(k, v)| (*k, *v)).collect(); + ranked.sort_unstable_by(|a, b| b.1.cmp(&a.1)); + let mut acc = 0u64; + let mut set = HashSet::new(); + for (k, v) in ranked { + if (acc as f64) >= 0.5 * total as f64 { + break; + } + acc += v; + set.insert(k); + } + set + }; + let hot_sets: Vec<(String, HashSet<_>)> = + per_domain.iter().map(|(l, h)| (l.clone(), hot_set(h))).collect(); + + // Jaccard overlap matrix of hot sets. + println!("\n=== CROSS-DOMAIN HOT-SET JACCARD (top-experts-to-50%-mass) ==="); + print!("{:<10}", ""); + for (l, _) in &hot_sets { + print!("{l:>9}"); + } + println!(); + for (la, sa) in &hot_sets { + print!("{la:<10}"); + for (_lb, sb) in &hot_sets { + let inter = sa.intersection(sb).count(); + let uni = sa.union(sb).count(); + let j = if uni > 0 { inter as f64 / uni as f64 } else { 0.0 }; + print!("{:>8.2} ", j); + } + println!(); + } + + // Shared base = experts in the hot set of ALL domains. Specialization = hot in exactly one. + let n_domains = hot_sets.len(); + let mut membership: HashMap<_, usize> = HashMap::new(); + for (_l, s) in &hot_sets { + for k in s { + *membership.entry(*k).or_insert(0) += 1; + } + } + let shared_base: HashSet<_> = membership + .iter() + .filter(|(_, &c)| c == n_domains) + .map(|(k, _)| *k) + .collect(); + let specialized: HashSet<_> = membership + .iter() + .filter(|(_, &c)| c == 1) + .map(|(k, _)| *k) + .collect(); + println!( + "\nshared base (hot in ALL {n_domains} domains): {} experts", + shared_base.len() + ); + println!( + "domain-specialized (hot in exactly 1) : {} experts", + specialized.len() + ); + + // The money number: per domain, what fraction of activation MASS lands on the shared base + // vs on experts NOT in any OTHER domain's hot set (the pageable eviction win). + println!("\n=== ACTIVATION MASS: shared-base vs pageable-specialized (the eviction win) ==="); + println!("{:<10} {:>14} {:>22}", "domain", "on shared base", "on own-only specialists"); + for (li, (label, hits)) in per_domain.iter().enumerate() { + let total: u64 = hits.values().sum(); + // experts that are in THIS domain's hot set and NO other domain's hot set + let own_only: HashSet<_> = hot_sets[li] + .1 + .iter() + .filter(|k| membership.get(*k).copied().unwrap_or(0) == 1) + .copied() + .collect(); + let mut base_mass = 0u64; + let mut own_mass = 0u64; + for (k, v) in hits { + if shared_base.contains(k) { + base_mass += v; + } + if own_only.contains(k) { + own_mass += v; + } + } + let pct = |m: u64| if total > 0 { 100.0 * m as f64 / total as f64 } else { 0.0 }; + println!( + "{label:<10} {:>13.1}% {:>21.1}%", + pct(base_mass), + pct(own_mass) + ); + } + println!( + "\nRead: high own-only mass = a coherent session concentrates on experts the OTHER\n\ + domains never touch → evict them when the workload isn't that domain. High shared-base\n\ + mass = a resident core you never page. That split IS the #180 paging architecture." + ); +} diff --git a/core/continuum-core/src/capacity/grid_overflow.rs b/core/continuum-core/src/capacity/grid_overflow.rs new file mode 100644 index 0000000000..b34283d8d8 --- /dev/null +++ b/core/continuum-core/src/capacity/grid_overflow.rs @@ -0,0 +1,237 @@ +//! Grid-overflow routing — the DECISION half of the governor consumer: given a serving plan +//! that overflowed local capacity, decide which eligible peers take the overflow lanes. The +//! EFFECT half (the actual `Commands.execute("ai/generate", {aircPeer})` hop) is thin and lives +//! at the live seam; THIS is pure, deterministic, and fully unit-tested. +//! +//! ## Why overflow placement is REMOTE-ONLY (not [`super::grid::LocalFirstFitPolicy`]) +//! +//! [`super::grid::LocalFirstFitPolicy`] fills local first and floors local at `≥1` (a resident +//! model must be able to run one prefill). That floor is correct for a FRESH placement but +//! WRONG for overflow: overflow lanes are BY DEFINITION the ones that already could not fit +//! locally (`ServingPlan.grid_overflow_lanes = demand − local_lanes`). Placing them local-first +//! would re-cram the very lanes the planner just declared didn't fit — the thrash the honest +//! "over local capacity by N" signal exists to avoid. So overflow placement never touches local: +//! it spills ONLY to eligible remote peers. +//! +//! ## The two gates, composed (never absorbed) +//! +//! 1. RESIDENCY ([`ModelResidencyView::residency_eligible`]): a peer is a fast overflow target +//! only for a model it ALREADY holds — else the hop pays a cold full-weights load, defeating +//! the point. Filters the snapshot to peers holding the model. +//! 2. CONCURRENCY (the misfit-parts fit, [`super::lanes_that_fit`]): among residency-eligible + +//! REACHABLE peers, each takes at most what its OWN free budget fits for the prefill spike — +//! the same per-node-fit rule the single-device and grid policies run. +//! +//! Reachability is applied HERE (an unreachable-but-resident peer is a memory, not an offer), +//! composing cleanly with residency without either abstraction absorbing the other. +//! +//! ## Unplaced lanes are SURFACED, never dropped +//! +//! When no eligible peer can take a lane, it lands in [`OverflowRouting::unplaced`] — the honest +//! "the grid couldn't absorb N of your overflow" signal the caller queues or degrades on. Never +//! silently swallowed ([[fallbacks-are-illegal-fail-loud]]). + +use crate::identity::PeerId; + +use super::grid::GridSnapshot; +use super::lanes_that_fit; +use super::model_residency::ModelResidencyView; +use super::LeaseRequest; + +/// The routing decision for a plan's overflow lanes: where each lane lands (remote-only) plus +/// the honest count of lanes the reachable, residency-eligible grid could NOT absorb. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OverflowRouting { + /// Overflow lanes placed on named peers, most-free-first, each capped by its own fit. + pub remote: Vec<(PeerId, u32)>, + /// Overflow lanes no eligible+reachable peer could take — queue or degrade on these, + /// never drop them silently. + pub unplaced: u32, +} + +impl OverflowRouting { + /// Total overflow lanes actually placed on peers. + pub fn placed(&self) -> u32 { + self.remote.iter().map(|(_, n)| n).sum() + } +} + +/// Decide where a plan's overflow lanes run. `lease` is the demand→capacity projection the +/// serving side already built (`ModelFootprint::grid_lease_request(served_window, overflow_lanes)`): +/// `want_concurrency` = the overflow lane count, `spike_bytes` = the per-lane prefill transient. +/// `model_id` is what the overflowing node is serving (its `ServingPlan.base_model_id`) — the +/// residency key. REMOTE-ONLY by construction (see module docs): local is already saturated. +pub fn route_grid_overflow( + model_id: &str, + lease: &LeaseRequest, + residency: &ModelResidencyView, + snapshot: &GridSnapshot, + safety_margin_bytes: u64, +) -> OverflowRouting { + let want = lease.want_concurrency; + if want == 0 { + return OverflowRouting { remote: Vec::new(), unplaced: 0 }; + } + + // Gate 1 — residency: keep only peers that hold the model resident. + let eligible = residency.residency_eligible(snapshot, model_id); + + // Gate 2 — reachability + per-node fit: reachable eligible peers, most-free-first (fewest + // peers touched), each capped by its OWN budget for the prefill spike. + let mut reachable: Vec<_> = eligible.peers.iter().filter(|p| p.reachable).collect(); + reachable.sort_by(|a, b| { + b.capacity + .gpu_free_bytes_live + .cmp(&a.capacity.gpu_free_bytes_live) + }); + + let mut remaining = want; + let mut remote = Vec::new(); + for peer in reachable { + if remaining == 0 { + break; + } + let fit = lanes_that_fit( + peer.capacity.gpu_free_bytes_live, + safety_margin_bytes, + lease.spike_bytes, + ); + let take = fit.min(remaining); + if take > 0 { + remote.push((peer.peer, take)); + remaining -= take; + } + } + + // Whatever the reachable, residency-eligible grid couldn't absorb is surfaced, not dropped. + OverflowRouting { remote, unplaced: remaining } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridSnapshot, PeerCapacity}; + use crate::capacity::DeviceCapacity; + use uuid::Uuid; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + fn lease(want: u32, spike_gb: u64) -> LeaseRequest { + LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: want, + spike_bytes: spike_gb * GB, + } + } + + fn view_holding(peers: &[(u128, &[&str])]) -> ModelResidencyView { + let mut v = ModelResidencyView::new(); + for (n, models) in peers { + v.set_resident(peer_id(*n), models.iter().map(|s| s.to_string())); + } + v + } + + // what this catches: overflow placement is REMOTE-ONLY and residency+reachability gated. + // Local is never assigned lanes (it's the saturated node that overflowed). Only a peer that + // holds the model AND is reachable takes lanes; a resident-but-unreachable peer and a + // reachable-but-not-resident peer both take nothing. + #[test] + fn overflow_routes_remote_only_to_reachable_resident_peers() { + let snap = GridSnapshot { + local: dev(1), // saturated — must never receive overflow lanes + peers: vec![ + peer(1, 40, true), // resident + reachable → takes lanes + peer(2, 40, false), // resident + UNREACHABLE → nothing + peer(3, 40, true), // reachable but NOT resident → nothing + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(2, 1), &residency, &snap, GB); + + assert_eq!(routing.remote.len(), 1, "only peer 1 is eligible + reachable"); + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1)); + assert_eq!(routing.placed(), 2, "both overflow lanes fit on peer 1"); + assert_eq!(routing.unplaced, 0); + } + + // what this catches: unplaced lanes are SURFACED, never dropped. When the eligible grid + // can't fit all overflow lanes (one small peer, a big per-lane spike), the shortfall is + // reported so the caller queues/degrades — the honest "grid couldn't absorb N" signal. + #[test] + fn lanes_the_grid_cannot_absorb_are_surfaced_not_dropped() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![peer(1, 10, true)], // ~10GB free, but each lane spikes 8GB + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + // want 3 lanes, 8GB spike each: only 1 fits on the 10GB peer (net of 1GB margin). + let routing = route_grid_overflow("qwen-coder", &lease(3, 8), &residency, &snap, GB); + + assert_eq!(routing.placed(), 1, "only one lane fits the peer's budget"); + assert_eq!(routing.unplaced, 2, "the other two are surfaced, not silently dropped"); + } + + // what this catches: no overflow (want == 0) is a clean no-op — nothing placed, nothing + // unplaced. The common case (demand fit locally, grid_overflow_lanes == 0) costs nothing. + #[test] + fn zero_overflow_is_a_clean_noop() { + let snap = GridSnapshot { + local: dev(20), + peers: vec![peer(1, 40, true)], + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(0, 1), &residency, &snap, GB); + + assert!(routing.remote.is_empty()); + assert_eq!(routing.unplaced, 0); + assert_eq!(routing.placed(), 0); + } + + // what this catches: spill spreads across peers most-free-first, each capped by its OWN fit + // (the misfit-parts rule) — 12 aggregate GB across three 4GB peers can't run a 6-lane + // placement if no single peer fits it, but distinct small lanes DO spread. Two reachable + // resident peers each take their share until demand is met. + #[test] + fn spill_spreads_across_peers_most_free_first() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![ + peer(1, 20, true), // more free → sorted first + peer(2, 12, true), + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + // 4 lanes, 8GB spike: peer1 (20-1 margin=19 → 2 lanes), peer2 (12-1=11 → 1 lane) = 3, + // one unplaced. Peer 1 (most free) is filled before peer 2. + let routing = route_grid_overflow("qwen-coder", &lease(4, 8), &residency, &snap, GB); + + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1), "most-free peer first"); + assert_eq!(routing.placed() + routing.unplaced, 4, "every lane accounted for"); + assert!(routing.placed() >= 3, "peers absorb what their own budgets fit"); + } +} diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index c9fbbb0096..8a4b977aad 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -30,7 +30,9 @@ pub mod expert_reconcile; pub mod expert_residency; pub mod gossip; pub mod grid; +pub mod grid_overflow; pub mod lease; +pub mod model_residency; pub mod moe_serving; pub mod placement; pub mod recursion_depth; diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs new file mode 100644 index 0000000000..96cccc0f71 --- /dev/null +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -0,0 +1,382 @@ +//! Model residency across the grid — WHICH peer currently holds WHICH model resident. The +//! ELIGIBILITY half of grid-overflow routing; the CONCURRENCY half is [`super::grid`]. +//! +//! ## Why this is a separate abstraction (orthogonal, never absorbed) +//! +//! `serving_plan` reasons about MODEL RESIDENCY — can a node hold model M's weights + per-lane +//! KV at the served window. `capacity/grid` reasons about CONCURRENCY SPIKES — does a node have +//! a free lane RIGHT NOW ([`super::LeaseRequest`]). Settled with BigMama 2026-07-27: these are +//! **orthogonal and compose** — neither should absorb the other, the mapping is the only +//! crossing point ([`super::serving_plan`]'s `grid_lease_request` is that one bridge). So +//! residency does NOT belong on [`super::grid::PeerCapacity`] (that would blur the concurrency +//! abstraction with a residency fact); it lives here, and the governor **composes** the two at +//! the placement filter. +//! +//! ## Why residency gates grid overflow +//! +//! A grid-overflow hop routes a persona's generation to a peer. If that peer already holds M +//! resident, the hop is fast — it needs only a free lane (the concurrency check). If it merely +//! has free VRAM but NOT M, accepting the hop forces a cold full-weights load (seconds to +//! minutes for a large model) — which defeats the entire point of overflowing for speed. So the +//! fast overflow path is eligible only for peers that already hold M. A peer with unknown +//! residency (never beaconed) is NOT eligible for the fast path — conservative by construction, +//! same spirit as [`super::residency_detect`] never claiming a promotion is faster than it is. +//! +//! ## The compose point +//! +//! [`ModelResidencyView::residency_eligible`] takes a live [`GridSnapshot`] and returns a +//! SMALLER snapshot — local device untouched (the overflowing node holds M by definition; it's +//! the one serving it), peers filtered to those holding M. The unchanged capacity placement +//! policy ([`super::grid::LocalFirstFitPolicy`]) then runs on that smaller snapshot: it never +//! learns about residency, it just sees fewer peers. Reachability stays the policy's job — an +//! unreachable-but-resident peer survives this filter and is dropped downstream by `place()`, +//! keeping the two concerns cleanly separate. + +use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; + +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::grid::GridSnapshot; +use crate::identity::PeerId; + +/// Per-peer set of model ids that peer currently holds resident, folded from residency beacons. +/// +/// Keyed on the peer's `Uuid` (via [`PeerId::as_uuid`]) — the same choice +/// [`super::gossip::GridCapacityLedger`] makes for capacity offers, so residency and capacity +/// index the grid identically. A peer absent from the map has UNKNOWN residency (never +/// beaconed), which [`Self::holds`] reports as `false`: not eligible for the fast overflow path. +#[derive(Debug, Clone, Default)] +pub struct ModelResidencyView { + by_peer: HashMap>, +} + +impl ModelResidencyView { + pub fn new() -> Self { + Self::default() + } + + /// Record (latest-wins) the full set of models a peer currently holds resident. Latest-wins + /// because residency is a live fact — a model paged out is no longer held, so a fresh beacon + /// REPLACES the peer's set rather than merging (a merge would resurrect evicted models). + pub fn set_resident(&mut self, peer: PeerId, models: I) + where + I: IntoIterator, + S: Into, + { + self.by_peer + .insert(peer.as_uuid(), models.into_iter().map(Into::into).collect()); + } + + /// Does this peer currently hold `model_id` resident? `false` for a peer that never beaconed + /// (unknown residency) — the conservative default that keeps the fast overflow path honest. + pub fn holds(&self, peer: &PeerId, model_id: &str) -> bool { + self.by_peer + .get(&peer.as_uuid()) + .is_some_and(|set| set.contains(model_id)) + } + + /// Number of peers with a known residency beacon — probe surface, mirrors + /// [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn known_peers(&self) -> usize { + self.by_peer.len() + } + + /// Compose this residency view with a live capacity snapshot for `model_id`: keep the local + /// device (the overflowing node holds M by definition) and keep only peers that hold M + /// resident. The returned snapshot feeds the UNCHANGED capacity placement policy — concurrency + /// logic never learns about residency, it just sees a shorter peer list. Reachability is NOT + /// applied here (that stays the policy's job downstream), so an unreachable-but-resident peer + /// survives this filter and is reclaimed by `place()`. Orthogonal, composed at exactly one + /// point, neither abstraction absorbing the other. + pub fn residency_eligible(&self, snapshot: &GridSnapshot, model_id: &str) -> GridSnapshot { + let mut eligible = snapshot.clone(); + eligible.peers.retain(|peer| self.holds(&peer.peer, model_id)); + eligible + } +} + +/// One node's residency beacon — the wire payload advertising which models it currently holds +/// resident. Rides its OWN `grid_residency` `EphemeralCoalesced` envelope, separate from +/// [`super::gossip::CapacityOffer`]'s `grid_capacity`: residency changes on model page-in/out +/// (minute-scale), not the 10s capacity beat, so coupling them would either over-publish +/// residency or under-refresh capacity. Names + timestamp only; peer identity is the WIRE's +/// (the transcript event's authenticated peer id), never the payload's — same rule as +/// `CapacityOffer`, so a peer cannot beacon residency on another's behalf. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResidencyBeacon { + /// Model ids this node holds resident (warm) RIGHT NOW — its base plus any co-resident. + pub resident_models: Vec, + /// Sender clock when the reading was taken (ms since epoch). Displayed, not trusted: + /// freshness is judged by RECEIVER clock at hear-time, exactly like `CapacityOffer.at_ms`. + pub at_ms: u64, +} + +/// A heard beacon + the receiver-clock instant it arrived (the freshness anchor). +#[derive(Debug, Clone)] +struct HeardBeacon { + models: Vec, + heard_at_ms: u64, +} + +/// Beacons silent past this drop from the projected view entirely. GENEROUS versus capacity's +/// eviction because the two abstractions stay orthogonal: residency is STICKY (a model stays +/// resident across many capacity beats), and the COMPOSED capacity snapshot already gates +/// reachability — so a residency reading never has to prove liveness itself (that would blur +/// residency into concurrency). 6× the capacity eviction window: a peer whose residency we +/// haven't reheard in that long falls back to UNKNOWN (not asserted-resident), keeping the fast +/// overflow path honest rather than routing to a model a long-silent peer may have paged out. +pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; + +/// The residency beacon heartbeat — deliberately SLOWER than capacity's 10s +/// ([`super::gossip::PUBLISH_INTERVAL_MS`]) because residency changes on model page-in/out +/// (minute-scale), not the free-VRAM beat. 12 beats fit inside +/// [`RESIDENCY_EVICTION_WINDOW_MS`] — the same publish:eviction ratio capacity gossip uses, +/// so a couple of dropped beacons never evicts a still-resident peer. +pub const RESIDENCY_PUBLISH_INTERVAL_MS: u64 = RESIDENCY_EVICTION_WINDOW_MS / 12; + +/// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the +/// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness +/// discipline, different (slower) cadence and payload. +#[derive(Default)] +pub struct ResidencyLedger { + heard: DashMap, +} + +/// The one process-global residency ledger — the resource it mirrors (this node's view of who +/// holds what across the grid) is process-global, same granularity argument as +/// [`super::gossip::global_ledger`]. +pub fn global_residency_ledger() -> &'static ResidencyLedger { + static LEDGER: OnceLock = OnceLock::new(); + LEDGER.get_or_init(ResidencyLedger::default) +} + +impl ResidencyLedger { + /// Fold one heard beacon in (latest per peer wins — residency is a live fact). `from_peer` + /// is the transcript event's transport identity, never payload-declared. Returns `true` when + /// this peer is NEW to the ledger — the probe-on-join surface; steady re-beacons stay silent. + pub fn hear(&self, from_peer: Uuid, beacon: ResidencyBeacon, heard_at_ms: u64) -> bool { + self.heard + .insert( + from_peer, + HeardBeacon { models: beacon.resident_models, heard_at_ms }, + ) + .is_none() + } + + /// Project the ledger into a [`ModelResidencyView`] the governor composes with the capacity + /// snapshot, evicting beacons silent past [`RESIDENCY_EVICTION_WINDOW_MS`] as it goes. + /// `own_peer` is excluded — the local node's residency is its OWN serving truth (it holds M + /// by definition when it overflows), not a round-tripped beacon. + pub fn view(&self, own_peer: Uuid, now_ms: u64) -> ModelResidencyView { + let mut view = ModelResidencyView::new(); + self.heard.retain(|peer, heard| { + let age = now_ms.saturating_sub(heard.heard_at_ms); + if age > RESIDENCY_EVICTION_WINDOW_MS { + return false; // silent too long — residency falls back to unknown + } + if *peer != own_peer { + view.by_peer + .insert(*peer, heard.models.iter().cloned().collect()); + } + true + }); + view + } + + /// Number of peers currently on the ledger (self included if echoed) — probe surface, + /// mirrors [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn heard_count(&self) -> usize { + self.heard.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridPlacementPolicy, GridSnapshot, LocalFirstFitPolicy, PeerCapacity}; + use crate::capacity::{DeviceCapacity, LeaseRequest}; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + // what this catches: the residency ELIGIBILITY gate. Only peers that beaconed the model as + // resident survive the filter; a peer with plenty of free VRAM but NOT holding M is dropped + // (routing to it would force a cold full-weights load — the slow path the gate exists to + // avoid), and a peer that never beaconed at all (unknown residency) is dropped too. The local + // device is always kept — the overflowing node holds M by definition. + #[test] + fn only_peers_holding_the_model_are_eligible() { + let snap = GridSnapshot { + local: dev(2), + peers: vec![ + peer(1, 40, true), // holds qwen-coder + peer(2, 40, true), // holds something else, NOT qwen-coder + peer(3, 40, true), // never beaconed — unknown residency + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "embed-small"]); + view.set_resident(peer_id(2), ["llama-70b"]); + // peer 3 intentionally not recorded. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.local, snap.local, "local device is always kept — it holds M"); + assert_eq!(eligible.peers.len(), 1, "only the peer holding qwen-coder survives"); + assert_eq!(eligible.peers[0].peer.as_uuid(), Uuid::from_u128(1)); + } + + // what this catches: latest-wins REPLACE, not merge. A peer that paged qwen-coder OUT (its + // fresh beacon lists only what it still holds) must stop being eligible — a merge would + // resurrect the evicted model and route a hop to a peer that no longer has it. + #[test] + fn fresh_beacon_replaces_so_evicted_models_stop_being_eligible() { + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "llama-70b"]); + assert!(view.holds(&peer_id(1), "qwen-coder")); + + // Peer paged qwen-coder out; next beacon lists only llama-70b. + view.set_resident(peer_id(1), ["llama-70b"]); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "evicted model must not linger"); + assert!(view.holds(&peer_id(1), "llama-70b")); + } + + // what this catches: the COMPOSE contract end-to-end — residency filters WHO is eligible, + // then the unchanged capacity policy places lanes on the survivors and applies reachability + // itself. A resident+reachable peer gets lanes; a resident+UNREACHABLE peer survives the + // residency filter (reachability isn't residency's job) but is reclaimed by place(); a + // non-resident peer is absent from placement entirely. Two orthogonal gates, composed. + #[test] + fn residency_then_capacity_policy_compose() { + let snap = GridSnapshot { + local: dev(1), // no local room — force the spill onto peers + peers: vec![ + peer(1, 40, true), // resident + reachable → should get lanes + peer(2, 40, false), // resident + UNREACHABLE → reclaimed by place() + peer(3, 40, true), // reachable but NOT resident → filtered out before place() + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder"]); + view.set_resident(peer_id(2), ["qwen-coder"]); + // peer 3 holds nothing relevant. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.peers.len(), 2, "peers 1 & 2 are resident; peer 3 filtered out"); + + // Small spike so many lanes fit per peer — we're testing WHO gets lanes, not how many. + let req = LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: 4, + spike_bytes: GB, + }; + let placement = LocalFirstFitPolicy { safety_margin_bytes: GB }.place(&eligible, &req); + + // The reachable resident peer carries the remote lanes; the unreachable one gets none. + let peer1_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(1)) + .map(|(_, n)| *n) + .sum(); + let peer2_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(2)) + .map(|(_, n)| *n) + .sum(); + let peer3_present = placement.remote.iter().any(|(p, _)| p.as_uuid() == Uuid::from_u128(3)); + + assert!(peer1_lanes > 0, "resident + reachable peer must carry overflow lanes"); + assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); + assert!(!peer3_present, "non-resident peer never reaches placement"); + } + + fn beacon(models: &[&str], at_ms: u64) -> ResidencyBeacon { + ResidencyBeacon { + resident_models: models.iter().map(|s| s.to_string()).collect(), + at_ms, + } + } + + // what this catches: the beacon RECEIVE→PROJECT pipeline — a heard beacon projects into a + // ModelResidencyView that holds() reflects, and the node's OWN echoed beacon is excluded + // (the local node's residency is its own serving truth, arriving live, not a round-trip). + // This is the residency sibling of gossip's loopback contract. + #[test] + fn heard_beacon_projects_and_own_echo_is_excluded() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + ledger.hear(me, beacon(&["qwen-coder"], 1_000), 1_000); // our own beacon, round-tripped + ledger.hear(other, beacon(&["qwen-coder", "llama-70b"], 1_000), 1_000); + + let view = ledger.view(me, 2_000); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "own echo excluded from the peer view"); + assert!(view.holds(&peer_id(2), "qwen-coder"), "the real peer's residency projects"); + assert!(view.holds(&peer_id(2), "llama-70b")); + assert_eq!(view.known_peers(), 1, "only the genuine peer, not the echo"); + } + + // what this catches: eviction after long silence — a peer we haven't reheard past the + // (generous) residency window falls back to UNKNOWN residency, so the fast overflow path + // won't route to a model that long-silent peer may since have paged out. A fresh beacon + // brings it right back (grow is first-class). + #[test] + fn stale_beacon_evicts_then_a_fresh_one_restores() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let peer = Uuid::from_u128(2); + ledger.hear(peer, beacon(&["qwen-coder"], 0), 0); + assert!(ledger.view(me, 1_000).holds(&peer_id(2), "qwen-coder")); + + // Silent past the residency eviction window: gone from the view (unknown, not asserted). + let t_evict = RESIDENCY_EVICTION_WINDOW_MS + 1; + assert!( + !ledger.view(me, t_evict).holds(&peer_id(2), "qwen-coder"), + "long-silent peer's residency falls back to unknown" + ); + + // It beacons again: instantly back. + ledger.hear(peer, beacon(&["qwen-coder"], t_evict), t_evict); + assert!( + ledger.view(me, t_evict + 1).holds(&peer_id(2), "qwen-coder"), + "a returning peer's residency is adopted on its first fresh beacon" + ); + } + + // what this catches: the wire payload survives serde round-trip byte-for-byte (camelCase + // like every realtime inline payload). If resident_models renamed or at_ms narrowed, heard + // residency would silently drift from published — the grid would gate overflow on models + // nobody actually beaconed. + #[test] + fn beacon_round_trips_through_json() { + let b = beacon(&["qwen-coder", "embed-small"], 42); + let json = serde_json::to_string(&b).expect("serialize"); + assert!(json.contains("residentModels"), "camelCase wire field: {json}"); + let back: ResidencyBeacon = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, b, "beacon must survive the wire byte-for-byte"); + } +} diff --git a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs index f5249b4c08..eb91e0a82b 100644 --- a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs +++ b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs @@ -1734,19 +1734,20 @@ mod tests { ); } - // what this catches: progressive disclosure — the per-turn tool PAYLOAD is the - // two-tool DISCOVERY PAIR (`commands/list` + `commands/help`), not the whole - // authorized registry, and the system prompt carries only a CATEGORY INDEX, not - // every tool. The old dump injected ~150 full schemas / one-liners (~4–5k tokens) - // into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute. Now the - // surface is a tiny category index inside the system prompt + the two-tool native - // offering, so even a huge tool set leaves system + user + the offered tools well - // within the served window. Invariant: the category index (not tool names) rides - // the system prompt, the native offering is exactly the discovery pair, and the - // whole prompt + its tools + reserve fit the window. A regression means the dump - // came back. + // what this catches: progressive disclosure — the system prompt carries only a + // CATEGORY INDEX (names, not the ~150 full schemas), and the native offering is the + // FULL authorized coding surface, which is NEVER window-amputated. Two failure modes + // this pins the absence of: (1) the old dump that injected ~150 schemas (~4–5k + // tokens) into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute; + // (2) the later "tight window ⇒ discovery-pair only" shrink cliff (deleted in #206 / + // 74acbb36c) that amputated the native surface to `commands/list`+`commands/help` on + // a tight window, stranding native-tool models in a help loop with 0 edits. Now the + // category index rides the system prompt, the full native surface rides beside it + // un-amputated, and the budget (`prompt_view_within`) reserves the specs' tokens and + // trims VOLATILE context so prompt + tools + reserve fit the window. A regression + // means either the dump came back or the amputation cliff did. #[test] - fn tool_surface_is_a_category_index_plus_discovery_pair() { + fn tool_surface_is_a_category_index_plus_the_unamputated_native_surface() { let persona = Uuid::new_v4(); let adapter: Arc = Arc::new(HeuristicInferenceAdapter::new()); // A tool set whose FULL schemas would dwarf the window — the live shape diff --git a/core/continuum-core/src/cognition/mod.rs b/core/continuum-core/src/cognition/mod.rs index 1aac35dcf7..cbb9a2fade 100644 --- a/core/continuum-core/src/cognition/mod.rs +++ b/core/continuum-core/src/cognition/mod.rs @@ -77,6 +77,7 @@ pub mod response_orchestrator; pub mod response_validator; pub mod serving_plan; pub mod shared_analysis; +pub mod working_set_demand; pub mod self_repeat; pub mod should_respond; pub mod should_respond_module; diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 1bdbf43dec..9651511c70 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -260,6 +260,29 @@ impl ModelFootprint { self.resident_bytes(served_window, lanes) .saturating_add(self.prefill_compute_reserve(served_window, lanes)) } + + /// The capacity-fabric [`LeaseRequest`](crate::capacity::LeaseRequest) for serving + /// this model at `served_window` with `demand_lanes` concurrent minds — the bridge + /// from the serving plan's MODEL-RESIDENCY view (weights + per-lane KV) to the grid's + /// CONCURRENCY-SPIKE view, so [`GridPlacementPolicy`](crate::capacity::grid) can place + /// overflow lanes onto peers (#180 grid spill / [[frontier-is-a-scaling-question-over-misfits-not-a-capability-question]]). + /// `want_concurrency` = the minds that want a lane; `spike_bytes` = ONE lane's transient + /// prefill compute buffer at this window (the term the 2026-07-14 OOM turned on), so a + /// peer is only offered a spill lane it can actually hold. NOTE: this sizes the + /// CONCURRENCY spike only; a peer must ALSO already hold this model resident — that + /// residency gate is the gossip-side half of the routing (owned with the grid snapshot), + /// NOT this pure mapping. + pub fn grid_lease_request( + &self, + served_window: u32, + demand_lanes: u32, + ) -> crate::capacity::LeaseRequest { + crate::capacity::LeaseRequest { + consumer: self.model_id.clone(), + want_concurrency: demand_lanes.max(1), + spike_bytes: self.prefill_compute_reserve(served_window, 1), + } + } } /// The serving decision for this host. @@ -310,12 +333,25 @@ pub struct ServingPlan { /// rendered, and the room degenerated into a greeting loop. 2 slots would /// have doubled every mind's window with zero lost concurrency. /// +/// `demand_ceil` is the LIVE demand ceiling for the served window — the ELASTIC, +/// per-task upper bound. `window_for` sizes the window UP to what the budget +/// allows; this caps it DOWN to what the task actually needs. A hard coding task +/// passes a high ceiling and the window GROWS (up to the budget/model bound); a +/// simple turn passes a low one so more lanes fit. It is NEVER a launch-baked +/// constant — callers thread live per-persona/per-task demand (measured p95 + +/// headroom, or a task's explicit request). [`plan_serving`] supplies +/// [`BOOTSTRAP_WORKING_SET`] only as the cold-start prior until that telemetry +/// exists (#234). OOM-safe: `window_for` already bounds the window to the budget, +/// so a higher ceiling only raises the cap toward that bound, never past it. +/// [[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]] +/// /// The decision is pure classification on memory arithmetic — no model is /// loaded, no inference is run. -pub fn plan_serving( +pub fn plan_serving_with_demand( host: HostBudget, candidates: &[ModelFootprint], demand_lanes: u32, + demand_ceil: u32, ) -> Option { if candidates.is_empty() { return None; @@ -441,7 +477,7 @@ pub fn plan_serving( // never forces UP past what the model supports; `.max(MIN_SERVE_CTX)` keeps it // runnable. On a small host where window_for < BOOTSTRAP the cap is a no-op. let served_context_window = window_for(lanes as u64) - .min(BOOTSTRAP_WORKING_SET) + .min(demand_ceil.max(MIN_SERVE_CTX)) .max(MIN_SERVE_CTX); // The honest per-lane compute reserve AT the chosen window (floor + window-scaled), // reused by the packing math below AND reported to the board via @@ -501,6 +537,20 @@ pub fn plan_serving( }) } +/// Cold-start / no-live-demand convenience: [`plan_serving_with_demand`] with the +/// [`BOOTSTRAP_WORKING_SET`] prior as the demand ceiling. Callers that do not YET +/// thread live per-task demand use this; the elastic path (a hard task requesting +/// more context, or the #234 p95 telemetry) calls `plan_serving_with_demand` with +/// the measured/requested ceiling so the window ebbs and flows with real demand +/// rather than a baked constant. +pub fn plan_serving( + host: HostBudget, + candidates: &[ModelFootprint], + demand_lanes: u32, +) -> Option { + plan_serving_with_demand(host, candidates, demand_lanes, BOOTSTRAP_WORKING_SET) +} + /// Hysteresis wrapper around [`plan_serving`]: stops model THRASH from live- /// budget jitter. Keeps the `incumbent` model as long as it still fits the /// budget — switching DOWN only when the incumbent no longer fits (forced @@ -514,13 +564,14 @@ pub fn plan_serving_stable( candidates: &[ModelFootprint], incumbent: Option<&str>, demand_lanes: u32, + demand_ceil: u32, ) -> Option { // NB: do NOT `?`-bail here. A deep transient dip can leave `plan_serving` // with nothing fitting the depressed budget (`fresh` = None) while a model // is STILL resident and serving fine — its memory is its own. Tearing that // down to "nothing" is the exact harm we're guarding against, so `fresh` is // an Option we fall back to only when the incumbent genuinely can't hold. - let fresh = plan_serving(host, candidates, demand_lanes); + let fresh = plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil); let Some(inc_id) = incumbent else { return fresh; }; @@ -572,7 +623,7 @@ pub fn plan_serving_stable( if let Some(m) = promoted.iter_mut().find(|m| m.model_id == inc_id) { m.capability_rank = u8::MAX; } - plan_serving(at_rest, &promoted, demand_lanes) + plan_serving_with_demand(at_rest, &promoted, demand_lanes, demand_ceil) } fn bytes_gb(bytes: u64) -> f64 { @@ -765,6 +816,70 @@ mod tests { ); } + // what this catches: the ELASTIC demand ceiling (Joel 2026-07-27: "context window sizes + // should ebb and flow depending on demands of the task and available resources — if it + // needs it larger for a moment, don't limit it"). The ceiling is threaded LIVE, not a + // launch-baked constant: a hard task passing a HIGH demand_ceil grows the served window + // PAST the BOOTSTRAP prior (up to the budget/model bound); a LOW ceiling shrinks it so + // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises + // the cap toward the budget bound, never past it. This is the "stop setting it in stone + // at launch" fix that opens the elastic-lease path. + // what this catches: the serving→grid bridge (#180 spill) — a model footprint maps to a + // capacity-fabric LeaseRequest whose want_concurrency is the demanded lanes and whose + // spike_bytes is ONE lane's transient prefill compute reserve at the served window (the + // 2026-07-14 OOM term), so GridPlacementPolicy offers a peer only a spill lane it can hold. + #[test] + fn grid_lease_request_maps_demand_and_the_prefill_spike() { + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + let window = 16_384; + let lease = devstral.grid_lease_request(window, 3); + assert_eq!(lease.consumer, "devstral-24b"); + assert_eq!(lease.want_concurrency, 3, "want_concurrency = demanded lanes"); + assert_eq!( + lease.spike_bytes, + devstral.prefill_compute_reserve(window, 1), + "spike_bytes = ONE lane's prefill compute reserve at the served window" + ); + // Zero demand floors at one lane — never a degenerate 0-concurrency lease. + assert_eq!(devstral.grid_lease_request(window, 0).want_concurrency, 1); + } + + #[test] + fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + // Roomy host + high trained ceiling → window_for(1) far exceeds any of these ceilings, + // so the DEMAND ceiling (not the budget or the model) decides the served window. + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + // Cold prior: default plan_serving caps at BOOTSTRAP_WORKING_SET. + let cold = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); + assert_eq!(cold.served_context_window, BOOTSTRAP_WORKING_SET); + + // Hard task demands more context → the window GROWS past the prior. + let big_ceil = 64_000; + let hot = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, big_ceil).unwrap(); + assert!( + hot.served_context_window > cold.served_context_window, + "a higher demand ceiling must GROW the window: hot {} ≤ cold {}", + hot.served_context_window, + cold.served_context_window + ); + assert!( + hot.served_context_window <= big_ceil, + "growth stays bounded by the demand ceiling (and the budget), never past it: got {}", + hot.served_context_window + ); + + // Simple turn demands little → the window SHRINKS below the prior, freeing memory. + let lean = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, 8_192).unwrap(); + assert_eq!( + lean.served_context_window, 8_192, + "a low demand ceiling shrinks the served window to it" + ); + } + // what this catches: lanes DEGRADE on a tight host — MAX_LANES is a sanity backstop, // the fit math is the real cap. A 4-persona demand on a budget that can't feed 4 warm // slots must serve FEWER (well-fed) lanes, never 4 starving ones that OOM. The window @@ -991,7 +1106,7 @@ mod tests { fn stable_with_no_incumbent_equals_plain() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; assert_eq!( - plan_serving_stable(host, &pair(), None, MAX_LANES), + plan_serving_stable(host, &pair(), None, MAX_LANES, BOOTSTRAP_WORKING_SET), plan_serving(host, &pair(), MAX_LANES) ); } @@ -1004,7 +1119,7 @@ mod tests { // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. let host = HostBudget { usable_bytes: 10 * GB, perf_cores: 6 }; assert_eq!(plan_serving(host, &pair(), MAX_LANES).unwrap().base_model_id, "big", "fresh would pick big"); - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "hysteresis keeps incumbent — no flap"); assert!(stable.lanes >= 1, "lanes still re-tracked for the kept model"); } @@ -1014,7 +1129,7 @@ mod tests { #[test] fn stable_upgrades_when_better_model_fits_with_headroom() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; // big 9.7 << 0.9*20=18 - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "more capable + ample headroom → upgrade"); } @@ -1029,7 +1144,7 @@ mod tests { fn stable_forced_down_when_incumbent_gone_from_disk() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; let only_small = vec![fp("small", 1, 4_000, 32_768, 1)]; // "big" no longer on disk - let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "incumbent gone from disk → serve what's present"); } @@ -1053,7 +1168,7 @@ mod tests { "depressed-budget plain plan would flap to the smaller model" ); // With the incumbent credited its own weights back, the resident big stays. - let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "incumbent survives its OWN load dip — no flap"); assert!(stable.lanes >= 1, "kept model still gets ≥1 lane"); } diff --git a/core/continuum-core/src/cognition/working_set_demand.rs b/core/continuum-core/src/cognition/working_set_demand.rs new file mode 100644 index 0000000000..78b1a8d043 --- /dev/null +++ b/core/continuum-core/src/cognition/working_set_demand.rs @@ -0,0 +1,198 @@ +//! `working_set_demand` — the DEMAND PRODUCER for the elastic served window (#234). +//! +//! A persona's served context window should ebb and flow with what its turns +//! ACTUALLY use, not a launch-baked constant ([[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]]). +//! This observes the assembled-prompt token count of recent turns and produces a +//! live demand ceiling that threads into +//! [`plan_serving_with_demand`](super::serving_plan::plan_serving_with_demand) — +//! growing the window for a persona whose tasks got bigger (a hard coding session) +//! and ebbing it back when they get lean again, so many personas stay warm at a +//! lean window and one doing heavy work gets room to think. +//! +//! ## Measured, not guessed — and never truncate the current turn +//! +//! Two signals combine: +//! - a rolling **p95 baseline** ([`demand_ceil`]) sizes the WARM lane to the +//! persona's usual work (so it's already big when the next hard turn arrives); +//! - the **measured current prompt** ([`demand_for`]) guarantees THIS turn is +//! never truncated — we already assembled the prompt, so we know its exact +//! size and request precisely that + generation headroom. +//! +//! That is how "if it needs it larger for a moment, don't limit it" holds WITHOUT +//! guessing. The budget fit in `plan_serving_with_demand` still bounds the result +//! above, so an impossibly large prompt degrades honestly, never OOMs. p95 (not +//! max) for the baseline so a single rare spike doesn't pre-allocate a KV that +//! swaps the box — the spike is handled per-turn by the measured path instead. +//! +//! Pure + testable: token counts in, demand ceiling out. No serving loop, no GPU +//! here — this is the measurement that makes the elastic lease honest. + +use std::collections::VecDeque; + +/// Below this many samples [`demand_ceil`] returns the cold `floor` rather than a +/// p95 off 1–2 points: provisioning a window off a tiny sample swings it turn to +/// turn (thrash). A handful of turns is enough to trust the shape without waiting +/// so long the window never grows within a short session. +const MIN_SAMPLES_FOR_P95: usize = 4; + +/// Rolling observer of a persona's per-turn working-set size (assembled-prompt +/// tokens), producing the live demand ceiling for its served window. +pub struct WorkingSetDemand { + /// Recent assembled-prompt token counts, oldest at the front. Bounded to + /// `window`, so a past hard session doesn't pin the ceiling forever. + samples: VecDeque, + /// Rolling window length (turns). ≥ 1. + window: usize, + /// Cold-start floor AND the hard minimum — the ceiling never drops below this, + /// so a lull can't starve the next turn's first prompt. Callers pass the + /// serving cold prior (`serving_plan::BOOTSTRAP_WORKING_SET`). + floor: u32, + /// Added to the observed prompt to leave room for the turn's GENERATION (the + /// prompt is what's assembled; the model still needs to write its answer). + gen_headroom: u32, +} + +impl WorkingSetDemand { + pub fn new(window: usize, floor: u32, gen_headroom: u32) -> Self { + let window = window.max(1); + Self { + samples: VecDeque::with_capacity(window), + window, + floor, + gen_headroom, + } + } + + /// Record one completed turn's assembled-prompt token count. Oldest evicted + /// past `window` so demand EBBS back — a past hard session doesn't pin the + /// window forever. + pub fn observe(&mut self, prompt_tokens: u32) { + if self.samples.len() == self.window { + self.samples.pop_front(); + } + self.samples.push_back(prompt_tokens); + } + + /// The sustained baseline demand ceiling: `max(floor, p95(recent prompts) + + /// gen_headroom)`. Floored so a lull can't starve the next turn; p95 (not max) + /// so a lone spike doesn't over-provision. Returns `floor` until there is + /// enough evidence ([`MIN_SAMPLES_FOR_P95`]) to trust the shape. + pub fn demand_ceil(&self) -> u32 { + if self.samples.len() < MIN_SAMPLES_FOR_P95 { + return self.floor; + } + let mut v: Vec = self.samples.iter().copied().collect(); + v.sort_unstable(); + let idx = (((v.len() - 1) as f64) * 0.95).round() as usize; + let p95 = v[idx.min(v.len() - 1)]; + self.floor.max(p95.saturating_add(self.gen_headroom)) + } + + /// The demand ceiling for a SPECIFIC turn whose assembled prompt we've already + /// MEASURED: never below what this turn actually needs + /// (`current_prompt_tokens + gen_headroom`), and never below the sustained p95 + /// baseline (so the warm lane stays sized for the persona's usual work). This + /// is how "don't limit it for a moment" holds without guessing — we measured + /// the prompt, so we request exactly enough. `plan_serving_with_demand` still + /// bounds it by the budget above. + pub fn demand_for(&self, current_prompt_tokens: u32) -> u32 { + self.demand_ceil() + .max(current_prompt_tokens.saturating_add(self.gen_headroom)) + .max(self.floor) + } + + /// How many turns are in the rolling window (for observability / tests). + pub fn sample_count(&self) -> usize { + self.samples.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The serving cold prior (serving_plan::BOOTSTRAP_WORKING_SET) + one generation + // of headroom — the values a live caller threads in. + const FLOOR: u32 = 16_384; + const HEAD: u32 = 2_048; + + // what this catches: cold start returns the FLOOR — provisioning a window off + // 1–2 samples would thrash it turn to turn, so it grows only once there's + // enough evidence to trust the shape. + #[test] + fn cold_start_returns_the_floor_until_enough_samples() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + assert_eq!(d.demand_ceil(), FLOOR); + d.observe(40_000); + d.observe(40_000); + assert_eq!(d.demand_ceil(), FLOOR, "still too few samples to trust the shape"); + } + + // what this catches: a sustained hard-coding session GROWS the baseline to + // p95 + generation headroom — the "if it needs it larger, don't limit it" case, + // measured not guessed. + #[test] + fn sustained_large_working_set_grows_the_ceiling_past_the_floor() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(40_000); + } + assert_eq!(d.demand_ceil(), 40_000 + HEAD, "grows to p95(40k) + headroom"); + assert!(d.demand_ceil() > FLOOR); + } + + // what this catches: demand EBBS BACK — a past hard session doesn't pin the + // window forever; once lean turns roll through the window, the ceiling returns + // to the floor and frees memory for more lanes. + #[test] + fn ceiling_ebbs_back_to_floor_as_lean_turns_roll_through_the_window() { + let mut d = WorkingSetDemand::new(8, FLOOR, HEAD); + for _ in 0..8 { + d.observe(40_000); + } + assert!(d.demand_ceil() > FLOOR); + for _ in 0..8 { + d.observe(3_000); // fully replaces the window + } + assert_eq!(d.demand_ceil(), FLOOR, "lean turns pull the ceiling back to the floor"); + } + + // what this catches: a single rare SPIKE does not over-provision the WARM + // baseline — p95 (not max) excludes the outlier so one 120k turn among lean + // ones can't pre-allocate a KV that swaps the box. + #[test] + fn single_spike_does_not_over_provision_the_baseline() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..31 { + d.observe(3_000); + } + d.observe(120_000); // one outlier among 31 lean turns + assert!( + d.demand_ceil() <= (3_000 + HEAD).max(FLOOR), + "the lone spike must not lift the sustained baseline: {}", + d.demand_ceil() + ); + } + + // what this catches: the MEASURED current turn is NEVER truncated — a big prompt + // we already assembled requests exactly its size + headroom even before the p95 + // baseline has grown ("don't limit it for a moment", from measurement not a + // guess); a small turn still gets at least the warm baseline. + #[test] + fn demand_for_never_truncates_the_measured_current_turn() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(20_000); // baseline ~22k + } + assert_eq!( + d.demand_for(60_000), + 60_000 + HEAD, + "a measured 60k turn requests its own size + headroom, above the baseline" + ); + assert_eq!( + d.demand_for(1_000), + d.demand_ceil(), + "a tiny turn still rides the warm sustained baseline" + ); + } +} diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index e54e31cec5..7bd27ca9ef 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -1354,6 +1354,38 @@ impl LlamaServerControl for LlamaServerProcess { // surfaces the real defect: a RAG budget that overshot the served // window ([[fallbacks-are-illegal-fail-loud]]). .arg("--no-context-shift"); + // KV CACHE QUANTIZATION (#232, opt-in field-proven technique). f16 KV is the + // default; q8_0 is ~half the resident KV footprint at near-lossless quality, + // freeing memory the elastic window (#234) can spend on a BIGGER context or MORE + // warm lanes — faster for multiple personas AND more room for hard coding. + // OFF by default: not every backend/build ships Metal KV-quant kernels, so this + // is an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). + // Set SERVING_KV_CACHE_TYPE=q8_0 (or q4_0) to enable; absent / `f16` → byte-identical + // f16 behavior. NOTE: to have the plan actually GROW the window on the freed memory + // (not just leave it as extra headroom), the fit math must also scale kv_per_token — + // that footprint coupling is the follow-up; this slice is the safe enablement. + if let Some(kv_type) = crate::config_env::read("SERVING_KV_CACHE_TYPE") + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty() && s != "f16") + { + cmd.arg("--cache-type-k") + .arg(&kv_type) + .arg("--cache-type-v") + .arg(&kv_type); + } + // FLASH ATTENTION (#232, opt-in field-proven technique). The fused attention kernel + // is faster on BOTH prefill and decode and lowers peak memory — directly attacking + // the prefill-bound turn latency (#139) and freeing room the elastic window (#234) + // can spend. OFF by default: Metal/backend flash-attn support + quality vary by build + // ([[verify-real-device-numbers-not-a-clamp-premise]]), so it's an operator opt-in, + // never a blind assumption. SERVING_FLASH_ATTN=1|on|true → enable; absent → llama.cpp + // default (no flag), byte-identical. + if crate::config_env::read("SERVING_FLASH_ATTN") + .map(|s| matches!(s.trim().to_ascii_lowercase().as_str(), "1" | "on" | "true" | "yes")) + .unwrap_or(false) + { + cmd.arg("--flash-attn"); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index d314ecb1f9..9e7e3422aa 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1701,7 +1701,16 @@ pub fn start_server( if let Some((interceptor_daemon_socket, _room)) = interceptor_airc_deps { let cell = airc_interceptor_cell.clone(); let root = crate::modules::persona_instance_manager::resolve_continuum_root(); - tokio::spawn(async move { + // MUST be `rt_handle.spawn`, NOT bare `tokio::spawn`: this runs during + // start_server on the IPC thread, and the `rt_handle.enter()` guard (above) is + // scoped and has already dropped by here — so there is NO ambient runtime and + // `tokio::spawn` panics "there is no reactor running". That panic kills the + // attach task, the OnceCell never fills, and EVERYTHING that reads it silently + // no-ops: the AircInterceptor's aircPeer routing AND the grid-overflow effector + // (build_overflow_effector reads this same cell). rt_handle.spawn targets the + // runtime by handle without needing ambient context. (BigMama diagnosed the + // panic 2026-07-27; this is the one-line fix.) + rt_handle.spawn(async move { match airc_lib::Airc::attach_as(root, "continuum-airc-interceptor", interceptor_daemon_socket) .await { @@ -1782,6 +1791,14 @@ pub fn start_server( resource_daemon.clone(), default_room, ))); + // Grid residency beacon (grid-overflow eligibility, slice 3): advertise which models + // this node holds resident on a slower cadence, folded by inbound_attach into + // capacity::model_residency::global_residency_ledger. Reads the SAME serving plan the + // daemon computes (watch snapshot, no parallel probe); orthogonal sibling of capacity. + runtime.register(Arc::new(crate::modules::grid_residency::GridResidencyModule::new( + serving_daemon.subscribe(), + default_room, + ))); let continuum_root = crate::modules::persona_instance_manager::resolve_continuum_root(); let daemon_socket_for_rag_inspect = daemon_socket.clone(); let registry = crate::persona::PersonaAircRuntimeRegistry::new(); @@ -2100,6 +2117,15 @@ pub fn start_server( // below). Subscribe BEFORE the spawn so no plan edge is missed while // the task waits on the executor-ready oneshot. let mut serving_plan_rx = serving_daemon.subscribe(); + // A DEDICATED clone of the interceptor's airc-handle cell for the grid-overflow + // effector (slice 4b): the boot-spawn `async move` below captures by move, and the + // interceptor still needs the original `airc_interceptor_cell` further down — so the + // effector rides its own Arc clone of the SAME shared cell (both see the handle once + // attach_as fills it). + let overflow_airc_cell = airc_interceptor_cell.clone(); + // Same reason for the serving daemon: the reconcile task below still needs the + // original `serving_daemon` handle, so the effector rides its own clone. + let overflow_serving = serving_daemon.clone(); rt_handle.spawn(async move { // Wait for the IPC thread to deliver the WIRED executor (this both // gates ordering AND hands us the executor the personas' hands ride). @@ -2187,7 +2213,22 @@ pub fn start_server( } else { attempt += 1; let summary = supervisor - .spawn_all(&mut provider, Some(tool_executor.clone())) + .spawn_all( + &mut provider, + Some(tool_executor.clone()), + // Grid-overflow effector (slice 4b): the LIVE closure. Reads + // the serving plan's grid_overflow_lanes, filters residency- + // eligible reachable peers (from the beacon ledger), runs + // route_grid_overflow, and re-homes the persona's adapter to an + // AircRemoteInferenceAdapter over the interceptor's airc handle. + // DEFENSIVE: None on any uncertainty → local adapter (no + // regression); can only be a safe no-op or a correct off-box + // route (self excluded via airc.peer_id()). + crate::persona::grid_overflow_effector::build_overflow_effector( + overflow_airc_cell.clone(), + overflow_serving.clone(), + ), + ) .await; if summary.hosted > 0 { tracing::info!( diff --git a/core/continuum-core/src/modules/chat/mod.rs b/core/continuum-core/src/modules/chat/mod.rs index 09998d01df..9758d62d38 100644 --- a/core/continuum-core/src/modules/chat/mod.rs +++ b/core/continuum-core/src/modules/chat/mod.rs @@ -121,18 +121,23 @@ impl ChatModule { Self { executor_slot } } - /// Resolve the executor for the current call. Panics if the - /// executor was never installed — that's a boot ordering bug - /// (`start_server` must call `install_executor_on_all` BEFORE - /// any chat command can dispatch). Per [[no-fallbacks-ever]]: - /// the panic message names the contract so the operator sees - /// the actual problem. - fn executor(&self) -> Arc { - self.executor_slot.cloned().expect( + /// Resolve the executor for the current call. Returns a loud, + /// contract-naming error if the executor was never installed — a + /// boot-ordering bug (`start_server` must call + /// `install_executor_on_all` BEFORE any chat command dispatches). + /// Per [[no-fallbacks-ever]] the message still names the contract + /// so the operator sees the real problem — but the blast radius is + /// THIS request, not the whole core: a command that races boot + /// fails loudly to its caller instead of `.expect()`-panicking the + /// process and taking every other module down with it + /// (#201 boot-race / #26 faculties degrade, never panic). + fn executor(&self) -> Result, String> { + self.executor_slot.cloned().ok_or_else(|| { "ChatModule: CommandExecutor not installed — \ start_server must call install_executor_on_all \ - before any chat command can dispatch (task #224)", - ) + before any chat command can dispatch (task #224)" + .to_string() + }) } /// `chat/poll` — return recent messages, optionally filtered by @@ -151,7 +156,7 @@ impl ChatModule { /// 5. Normalize back to chronological order for display regardless /// of query direction. pub async fn poll(&self, params: ChatPollParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let limit = params.limit.unwrap_or(DEFAULT_POLL_LIMIT); // ── Phase 1: resolve the anchor timestamp if the caller @@ -284,7 +289,7 @@ impl ChatModule { /// could be the dedup id) but the design conversation is its /// own scope. pub async fn send(&self, params: ChatSendParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let message_id = Uuid::new_v4(); let now_ms = now_ms(); let now_iso = now_iso(now_ms); @@ -416,7 +421,7 @@ impl ChatModule { /// EXPECTED duplicate, never an error. A persona line's id is airc's /// `event_id` (stable across replay), so restarts can't double-store either. pub async fn persist_posted(&self, payload: Value) -> Result<(), String> { - let executor = self.executor(); + let executor = self.executor()?; let field = |k: &str| -> Result { payload .get(k) @@ -995,6 +1000,24 @@ mod tests { assert!(result.after_message_id.is_none()); } + // what this catches: #201 boot-race — a chat command that arrives BEFORE + // install_executor_on_all runs fails LOUD to its caller (naming the contract), + // instead of .expect()-panicking the whole core and taking every other module + // down with it. Regression for the .expect() → Result conversion in executor(). + #[tokio::test] + async fn command_before_executor_installed_fails_loud_not_panics() { + let chat = ChatModule::new(); // executor slot deliberately NOT installed + let err = chat + .poll(ChatPollParams::default()) + .await + .expect_err("poll before install_executor_on_all must fail, not panic"); + assert!(err.contains("not installed"), "loud contract error, got: {err}"); + assert!( + err.contains("install_executor_on_all"), + "error names the contract so the operator sees the real problem: {err}" + ); + } + // ── chat/poll: latest-N path (no anchor) ────────────────────────── #[tokio::test] diff --git a/core/continuum-core/src/modules/grid_residency.rs b/core/continuum-core/src/modules/grid_residency.rs new file mode 100644 index 0000000000..3f5fb81f0d --- /dev/null +++ b/core/continuum-core/src/modules/grid_residency.rs @@ -0,0 +1,166 @@ +//! GridResidencyModule — this node's residency-beacon gossip publisher (grid-overflow slice 3). +//! +//! The residency sibling of [`super::grid_capacity::GridCapacityModule`]. Every +//! [`RESIDENCY_PUBLISH_INTERVAL_MS`] tick it reads the SAME live serving plan the daemon +//! already computes (one source, no parallel probe) and broadcasts a [`ResidencyBeacon`] over +//! airc as an `EphemeralCoalesced` `grid_residency` realtime envelope: which models this node +//! holds resident — the grid-overflow ELIGIBILITY signal. The receive half lives in +//! [`crate::airc::inbound_attach`], which folds heard beacons (our own echo included — the +//! loopback proof) into [`crate::capacity::model_residency::global_residency_ledger`], whose +//! `view()` IS the `ModelResidencyView` the governor composes with the capacity snapshot. +//! +//! ## Why a SEPARATE module + slower cadence +//! +//! Residency and capacity are orthogonal (settled with BigMama 2026-07-27): capacity is +//! free-VRAM RIGHT NOW (10s beat, [`super::grid_capacity`]); residency is which models are warm +//! (minute-scale, changes only on page-in/out). Coupling them onto one envelope would either +//! over-publish residency or under-refresh capacity. Same module shape as the style guide +//! mandates — no new tokio task, the runtime tick drives it; the plan read is a lock-free +//! `watch` snapshot; the publish is one small envelope through the existing +//! `airc/realtime-publish` command surface (the same path capacity + chat ride). +//! +//! ## Today's local residency = the base model +//! +//! `ServingPlan` carries a single `base_model_id` + a `resident_models` COUNT (not a per-id +//! set), so today this node's honest resident set is `[base_model_id]`. When multi-model +//! residency lands (a per-id resident list on the plan), only [`Self::current_beacon`] changes +//! — the wire, ledger, and governor compose are already model-set-shaped. + +use std::any::Any; +use std::sync::Mutex; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::{json, Value}; +use tokio::sync::watch; + +use crate::capacity::model_residency::{ + global_residency_ledger, ResidencyBeacon, RESIDENCY_PUBLISH_INTERVAL_MS, +}; +use crate::cognition::serving_plan::ServingPlan; +use crate::runtime::{ + CommandExecutor, CommandResult, LateBound, ModuleConfig, ModulePriority, ServiceModule, +}; +use airc_core::RoomId; +use std::sync::Arc; + +pub struct GridResidencyModule { + /// Lock-free live snapshot of the daemon's serving plan — the SAME source the prefill + /// valve and serving control derive from (no parallel probe). + plan_rx: watch::Receiver>, + /// The node's discovered default room — where the grid rendezvous happens today. + room: RoomId, + executor_slot: Arc>, + /// Last published resident set — the glass box speaks on CHANGE, not on every beat (a + /// steady residency is silence). `None` = never published, so the first real plan speaks. + last_published: Mutex>>, +} + +impl GridResidencyModule { + pub fn new(plan_rx: watch::Receiver>, room: RoomId) -> Self { + Self { + plan_rx, + room, + executor_slot: Arc::new(LateBound::new("grid-residency::executor")), + last_published: Mutex::new(None), + } + } + + /// Build this node's residency beacon from the live serving plan. `None` when no plan is + /// computed yet (nothing resident to advertise) — honest silence, never a fabricated set. + /// Today the resident set is `[base_model_id]`; a multi-model plan extends only this line. + fn current_beacon(&self) -> Option { + let plan = self.plan_rx.borrow().clone()?; + Some(ResidencyBeacon { + resident_models: vec![plan.base_model_id], + at_ms: now_ms(), + }) + } +} + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[async_trait] +impl ServiceModule for GridResidencyModule { + fn config(&self) -> ModuleConfig { + ModuleConfig { + name: "grid-residency", + priority: ModulePriority::Background, + command_prefixes: &[], + event_subscriptions: &[], + needs_dedicated_thread: false, + max_concurrency: 0, + tick_interval: Some(Duration::from_millis(RESIDENCY_PUBLISH_INTERVAL_MS)), + } + } + + async fn initialize(&self, _ctx: &crate::runtime::ModuleContext) -> Result<(), String> { + Ok(()) + } + + async fn handle_command(&self, command: &str, _params: Value) -> Result { + Err(format!( + "grid-residency has no command surface — '{command}' (beacons publish on the module \ + tick; read the grid via capacity::model_residency::global_residency_ledger)" + )) + } + + async fn tick(&self) -> Result<(), String> { + // Boot ordering: a beat or two may fire before start_server installs the executor — + // transient, skip (the next beat publishes). Same guaranteed-installed path as capacity. + let Some(executor) = self.executor_slot.cloned() else { + return Ok(()); + }; + let Some(beacon) = self.current_beacon() else { + // No serving plan yet — nothing resident to advertise. Honest silence. + return Ok(()); + }; + + let envelope = json!({ + "eventId": uuid::Uuid::new_v4().to_string(), + "roomId": self.room.as_uuid().to_string(), + "sourceId": "grid-residency", + "createdAtMs": beacon.at_ms, + "delivery": "ephemeral_coalesced", + "payload": { + "kind": "existing_schema", + "payload": { + "schema": "grid_residency", + "inline": serde_json::to_value(&beacon) + .map_err(|e| format!("residency beacon encode failed: {e}"))?, + } + }, + }); + executor + .execute_json("airc/realtime-publish", json!({ "envelope": envelope })) + .await + .map_err(|e| format!("grid-residency beacon publish failed: {e}"))?; + + // Glass box: speak on change (resident set differs), silent on a steady residency. + let mut last = self.last_published.lock().map_err(|e| format!("residency lock: {e}"))?; + if last.as_deref() != Some(beacon.resident_models.as_slice()) { + crate::probe!( + class = "grid.residency.beacon", + models = ?beacon.resident_models, + heard_peers = global_residency_ledger().heard_count(), + "residency beacon published to the grid", + ); + *last = Some(beacon.resident_models); + } + Ok(()) + } + + fn install_executor(&self, executor: Arc) { + self.executor_slot.install(executor); + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/core/continuum-core/src/modules/mod.rs b/core/continuum-core/src/modules/mod.rs index f146dbe500..9323d8f772 100644 --- a/core/continuum-core/src/modules/mod.rs +++ b/core/continuum-core/src/modules/mod.rs @@ -46,6 +46,7 @@ pub mod gpu; pub mod grant_issuance; pub mod grid; pub mod grid_capacity; +pub mod grid_residency; pub mod health; pub mod hippocampus; pub mod inference_coordinator_module; diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index b32dd25112..90e8902ed2 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -23,8 +23,30 @@ use crate::cognition::model_resolver::types::HwCapabilityTier; use crate::cognition::serving_plan::{ - plan_serving, plan_serving_stable, HostBudget, ModelFootprint, ServingPlan, MIN_SERVE_CTX, + plan_serving, plan_serving_stable, plan_serving_with_demand, HostBudget, ModelFootprint, + ServingPlan, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, }; +use crate::cognition::working_set_demand::WorkingSetDemand; + +/// Process-wide sink for per-turn working-set observations — the serving daemon +/// installs its own `WorkingSetDemand` here at [`ServingDaemonModule::initialize`], +/// so the cognition turn path feeds each turn's assembled-prompt size WITHOUT a +/// daemon handle (the same process-wide-readable-seam shape as `install_serving_state`). +/// Unset until a daemon boots → [`observe_serving_working_set`] is a no-op and the +/// served window simply holds its cold prior. (#234) +static SERVING_WORKING_SET: OnceLock>> = OnceLock::new(); + +/// Feed one completed turn's assembled-prompt token count into the live serving +/// demand. Called from the persona turn path; the next plan tick sizes the served +/// window to the p95 of recent turns. No-op before a serving daemon has booted, so +/// the caller never needs to know whether serving is up. (#234) +pub fn observe_serving_working_set(prompt_tokens: u32) { + if let Some(ws) = SERVING_WORKING_SET.get() { + if let Ok(mut w) = ws.lock() { + w.observe(prompt_tokens); + } + } +} use crate::gpu::GpuMemoryManager; use crate::inference::llama_server::{ ensure_model_serving, serving_v1_url, AdapterEntry, EnsureOutcome, LlamaServerControl, @@ -239,6 +261,12 @@ pub struct ServingDaemonModule { /// that fit at pin time; budget can still shift under it, and then the plan /// degrades honestly (`fits_on_gpu = false`) rather than over-committing. pinned: watch::Sender>, + /// Live per-host working-set demand — the p95 of recent turns' assembled-prompt + /// sizes, threaded into the serving plan as the elastic `demand_ceil` so the + /// served window ebbs and flows with real demand instead of the baked prior + /// (#234). A Mutex (not atomic) for the rolling window; read only on the plan + /// tick, never a hot path. `observe_working_set` feeds it from the cognition turn. + working_set: Arc>, } impl ServingDaemonModule { @@ -273,6 +301,10 @@ impl ServingDaemonModule { let (serving_tx, _srx) = watch::channel(ServingSnapshot::empty()); let (suppressed, _urx) = watch::channel(Arc::new(HashSet::new())); let (pinned, _prx) = watch::channel(None); + // Rolling window (recent turns) over which the served window's demand ceiling + // tracks the p95 assembled-prompt size. Long enough to hold a coding session's + // shape, short enough that demand ebbs back within a session (#234). + const WORKING_SET_WINDOW: usize = 64; Self { gpu, system, @@ -296,6 +328,11 @@ impl ServingDaemonModule { suppressed, pinned, lane_demand: Arc::new(std::sync::atomic::AtomicU32::new(1)), + working_set: Arc::new(std::sync::Mutex::new(WorkingSetDemand::new( + WORKING_SET_WINDOW, + BOOTSTRAP_WORKING_SET, + MIN_SERVE_CTX, // one generation of headroom above the assembled prompt + ))), moe_serving: std::sync::Mutex::new(None), // Read ONCE here (single config entry point, no per-tick I/O). Off by default. measure_force_expert_budget_bytes: crate::config_env::read( @@ -313,6 +350,27 @@ impl ServingDaemonModule { .store(demand.max(1), Ordering::Relaxed); } + /// Feed one completed turn's assembled-prompt token count into the live + /// working-set demand (#234). Called from the cognition turn path; the next + /// plan tick sizes the served window to the p95 of recent turns. Cheap — a + /// brief lock on a bounded ring, off the serving hot path. + pub fn observe_working_set(&self, prompt_tokens: u32) { + if let Ok(mut w) = self.working_set.lock() { + w.observe(prompt_tokens); + } + } + + /// The live demand ceiling for the served window — the p95 baseline of recent + /// turns (floored at the cold prior). Read on the plan tick and threaded into + /// `plan_serving_with_demand` so the window grows for heavy work and ebbs back + /// when turns get lean. A poisoned lock degrades to the cold prior, never panics. + fn demand_ceil(&self) -> u32 { + self.working_set + .lock() + .map(|w| w.demand_ceil()) + .unwrap_or(BOOTSTRAP_WORKING_SET) + } + /// The current lane demand (≥ 1). /// Register serving's autonomic PLANNER to run on the memory authority's tick /// (MEMORY-AUTHORITY-DAEMON slice 1b). The lane plan — which model, how many lanes, @@ -502,7 +560,7 @@ impl ServingDaemonModule { /// one candidate, so the reconcile serves that model or (if it has dropped off /// disk) nothing. Suppress subtracts; pin intersects; the planner still owns /// the choice among whatever remains. - fn live_candidates(&self) -> Vec { + pub(crate) fn live_candidates(&self) -> Vec { let suppressed = self.suppressed.borrow(); let pinned = self.pinned.borrow(); servable_candidates(&self.catalog.snapshot(), &**suppressed, &pinned) @@ -559,10 +617,11 @@ impl ServingDaemonModule { /// to drive the spawner before the tick loop starts — single source of /// truth for "what model + how many lanes." pub fn compute_plan(&self) -> Option { - plan_serving( + plan_serving_with_demand( self.host_budget(), &self.live_candidates(), self.lane_demand(), + self.demand_ceil(), ) } @@ -1004,7 +1063,7 @@ impl ServingDaemonModule { .borrow() .as_ref() .map(|p| p.base_model_id.clone()); - match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand()) { + match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand(), self.demand_ceil()) { Some(plan) => { crate::probe!( class = "serving.plan", @@ -1290,12 +1349,38 @@ fn serving_footprint_fn(catalog: Arc) -> FootprintFn { pub fn footprint_for(model: &Model) -> Option { let path = crate::model_registry::artifacts::resolve_gguf_for_model(model)?; let weights_bytes = std::fs::metadata(&path).ok()?.len(); - footprint_from_parts( + let mut fp = footprint_from_parts( &model.id, weights_bytes, model.context_window, model.has(Capability::ToolUse), - ) + )?; + // KV CACHE QUANTIZATION (#232): a lane running quantized KV holds proportionally + // fewer bytes/token, so the plan can size a BIGGER window into the same budget — + // this is what turns the launcher's opt-in q8_0 flag into an actual window GROWTH. + // Divide the f16 rate by the quant factor; default (f16 / unset) → 1 → byte-identical. + // Keep the config key in sync with the launcher arg in inference/llama_server.rs — + // one SERVING_KV_CACHE_TYPE key, two consumers (launcher flag + this fit-math rate). + fp.kv_per_token = (fp.kv_per_token / kv_cache_quant_divisor()).max(1); + Some(fp) +} + +/// The resident-KV divisor implied by `SERVING_KV_CACHE_TYPE`, so the plan sizes the +/// served window against the KV the lane WILL actually hold, not the f16 default. (#232) +fn kv_cache_quant_divisor() -> u64 { + kv_divisor_for(crate::config_env::read("SERVING_KV_CACHE_TYPE").as_deref()) +} + +/// Pure KV-rate divisor for a cache-type string (testable without env). CONSERVATIVE by +/// design: q8_0 ≈ half of f16 → 2; q4_0/q4_1 ≈ a third → 3 (under the ideal ~3.5×, so the +/// plan never over-grows the window past the real KV and OOMs). Anything else / f16 → 1 +/// (no change). Over-reserve is a smaller window (safe); under-reserve is an OOM (fatal). +fn kv_divisor_for(cache_type: Option<&str>) -> u64 { + match cache_type.map(|s| s.trim().to_ascii_lowercase()).as_deref() { + Some("q8_0") => 2, + Some("q4_0") | Some("q4_1") => 3, + _ => 1, + } } /// Pure footprint estimate from the fields that drive it — split out from the @@ -1518,6 +1603,10 @@ impl ServiceModule for ServingDaemonModule { // free functions + adapters read "what's live" as a pointer instead of // each probing /v1/models. Set-once (singleton daemon). let _ = crate::inference::llama_server::install_serving_state(self.subscribe_serving()); + // Install this daemon's working-set demand as the process-wide sink so the + // cognition turn path feeds per-turn prompt sizes without a daemon handle, and + // the served window ebbs and flows with real demand (#234). + let _ = SERVING_WORKING_SET.set(self.working_set.clone()); // Register serving as a MEASURED ResourceConsumer with the one per-machine // authority (#79). See `register_as_consumer` — this is monitor-not-reserve: // no lease acquired, `available` math untouched, the authority simply stops @@ -1735,6 +1824,19 @@ mod tests { // what this catches: footprint estimate is honest about weights (passed // through), tool capability bumps the rank, KV is non-zero, and zero // weights → no footprint (we only offer what we can actually serve). + #[test] + fn kv_divisor_reflects_cache_type_conservatively() { + // what this catches: the #232 KV-quant fit-math coupling — the served window grows + // only when the lane actually runs quantized KV, and CONSERVATIVELY so the plan + // never over-grows past the real KV and OOMs. f16/unset/unknown must never scale. + assert_eq!(kv_divisor_for(None), 1, "unset never scales the window"); + assert_eq!(kv_divisor_for(Some("f16")), 1, "explicit f16 is the no-op default"); + assert_eq!(kv_divisor_for(Some("q8_0")), 2, "q8_0 ~ half of f16"); + assert_eq!(kv_divisor_for(Some(" Q8_0 ")), 2, "trimmed + case-insensitive"); + assert_eq!(kv_divisor_for(Some("q4_0")), 3, "q4_0 conservative, under the ideal ~3.5x"); + assert_eq!(kv_divisor_for(Some("garbage")), 1, "unknown type → no grow, never a bogus OOM"); + } + #[test] fn footprint_from_parts_is_footprint_aware() { let fp = footprint_from_parts("present", 3 * GB, 8192, true).unwrap(); diff --git a/core/continuum-core/src/persona/grid_overflow_effector.rs b/core/continuum-core/src/persona/grid_overflow_effector.rs new file mode 100644 index 0000000000..4acc4a7286 --- /dev/null +++ b/core/continuum-core/src/persona/grid_overflow_effector.rs @@ -0,0 +1,115 @@ +//! The grid-overflow EFFECTOR (governor consumer slice 4b-ii) — composes the tested +//! decision path into the live per-persona adapter override that +//! [`super::supervisor::materialize_adapters`] consumes. +//! +//! When the node is over local capacity (`ServingPlan.grid_overflow_lanes > 0`) and a +//! reachable peer already holds a persona's model, this routes HER BRAIN to that peer: her +//! `DeliberationModelBinding.adapter` becomes an [`AircRemoteInferenceAdapter`] over airc, so +//! her inference crosses the grid transparently — the exact re-home the binding was designed +//! for. The persona doesn't know or care that her model runs on another machine. +//! +//! ## Defensive by construction — safe to ship before the live smoke +//! +//! The closure returns `None` (→ the local factory adapter, zero behavior change) on ANY +//! uncertainty: airc not yet attached, no serving plan, no overflow, no matching footprint, or +//! no residency-eligible reachable peer. So it can only ever be a **safe no-op or a correct +//! remote route** — never a self-route (this node's own peer is excluded via `airc.peer_id()`, +//! so its own residency-beacon loopback can't select itself) and never a panic. The one thing +//! the unit path can't prove is that the remote hop SUCCEEDS — that is what the live two-node +//! smoke validates; a hop that can't warm surfaces as a loud per-slot `AdapterWarmup` failure +//! ([[fallbacks-are-illegal-fail-loud]]), never a silent local downgrade. + +use std::sync::Arc; + +use airc_lib::Airc; +use tokio::sync::OnceCell; + +use crate::ai::adapter::AIProviderAdapter; +use crate::capacity::gossip::global_ledger; +use crate::capacity::grid_overflow::route_grid_overflow; +use crate::capacity::model_residency::global_residency_ledger; +use crate::capacity::DeviceCapacity; +use crate::inference::airc_remote::adapter::AircRemoteInferenceAdapter; +use crate::inference::airc_remote::transport::AircLiveTransport; +use crate::modules::serving_daemon::ServingDaemonModule; +use crate::persona::inference_profile::PersonaInferenceProfile; + +/// Headroom kept free on a peer before it may accept an overflow lane — same 1 GiB spirit as +/// the single-device [`crate::capacity::grid::LocalFirstFitPolicy`] safety margin. +const OVERFLOW_SAFETY_MARGIN_BYTES: u64 = 1024 * 1024 * 1024; + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Build the effector closure `spawn_all` / `materialize_adapters` consume. Captures the +/// late-bound airc handle cell (shared with the interceptor) and the serving daemon (the live +/// plan + footprint source). See the module docs for the defensive contract. +pub fn build_overflow_effector( + airc_cell: Arc>>, + serving: Arc, +) -> impl Fn(&PersonaInferenceProfile, usize) -> Option> { + move |profile, _slot| { + // airc not attached yet → local (the boot window before attach_as fills the cell). + let airc = airc_cell.get()?.clone(); + // No plan, or demand fits locally → local. grid_overflow_lanes is the honest + // "over local capacity by N" signal; 0 means nothing to spill. + let plan = serving.compute_plan()?; + if plan.grid_overflow_lanes == 0 { + return None; + } + // The footprint for THIS persona's model — the lease's per-lane prefill spike. + // Absent (model not a live candidate) → local, never a fabricated footprint. + let footprint = serving + .live_candidates() + .into_iter() + .find(|f| f.model_id == profile.model_id)?; + let lease = + footprint.grid_lease_request(plan.served_context_window, plan.grid_overflow_lanes); + + // Own peer id from the airc handle → excludes self from the residency view + gossip + // snapshot (this node's own beacon loopback must never select itself as the target). + let own = airc.peer_id().as_uuid(); + let now = now_ms(); + let residency = global_residency_ledger().view(own, now); + // Overflow placement is REMOTE-ONLY, so local capacity is never read — a zeroed local + // is the honest input (route_grid_overflow only ever inspects the peer list). + let snapshot = global_ledger().snapshot( + own, + DeviceCapacity { + gpu_total_bytes: 0, + gpu_free_bytes_live: 0, + system_ram_free_bytes: 0, + }, + now, + ); + + let routing = route_grid_overflow( + &profile.model_id, + &lease, + &residency, + &snapshot, + OVERFLOW_SAFETY_MARGIN_BYTES, + ); + // First-cut assignment: this persona takes the first placed peer. No residency-eligible + // reachable peer with room → local (queue/degrade is the planner's, not a silent drop). + let (peer, _lanes) = routing.remote.first()?; + let peer_uuid = peer.as_uuid(); + + let transport = AircLiveTransport::new(airc, peer_uuid); + let adapter = AircRemoteInferenceAdapter::new(transport); + crate::probe!( + class = "grid.overflow.route", + persona = %profile.persona_name, + model = %profile.model_id, + peer = %peer_uuid, + overflow_lanes = plan.grid_overflow_lanes, + "routing persona brain OFF-BOX to a residency-eligible peer (grid overflow)", + ); + Some(Arc::new(adapter) as Arc) + } +} diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index d42ad0df19..c9ec318822 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -264,6 +264,15 @@ impl PersonaSpawnSupervisor { // persona's HANDS are built over it (identity-scoped), so the ACL gates // what they may do. `None` → personas spawn speak-only (no hands). tool_command_executor: Option>, + // The grid-overflow effector (governor consumer slice 4b): given a persona's + // profile + slot, `Some(remote adapter)` routes her brain to a peer that holds + // her model (the ipc bootstrap builds this from the live serving plan + residency + // ledger + airc handle); `None` → local adapter. Forwarded verbatim to + // `materialize_adapters`. `|_, _| None` is the pre-effector (all-local) behavior. + overflow_adapter_for: impl Fn( + &crate::persona::inference_profile::PersonaInferenceProfile, + usize, + ) -> Option>, ) -> BootSummary { let plans = match bootstrap_planned( &self.spawner, @@ -328,6 +337,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, + // Grid-overflow effector: the ipc bootstrap supplies the live decision + // (capacity + residency + airc). `|_, _| None` from a caller that doesn't + // route keeps the pre-effector all-local behavior. + overflow_adapter_for, ) .await; diff --git a/core/continuum-core/src/persona/mod.rs b/core/continuum-core/src/persona/mod.rs index b69f5930e4..008a907c3a 100644 --- a/core/continuum-core/src/persona/mod.rs +++ b/core/continuum-core/src/persona/mod.rs @@ -50,6 +50,7 @@ pub mod evaluator; pub mod focus; pub mod genome_paging; pub mod home; +pub mod grid_overflow_effector; pub mod host; pub mod hw_tier_descriptor; pub mod identity_provider; diff --git a/core/continuum-core/src/persona/service_loop.rs b/core/continuum-core/src/persona/service_loop.rs index 945a422e90..d61f53526c 100644 --- a/core/continuum-core/src/persona/service_loop.rs +++ b/core/continuum-core/src/persona/service_loop.rs @@ -1058,6 +1058,11 @@ async fn serve_persona_loop_inner( tokens_per_second = m.tokens_per_second(), "deliberation generation cost" ); + // Feed this turn's assembled-prompt size into the elastic serving + // demand so the served window ebbs and flows with real work — a lean + // chat turn keeps it small (more personas warm), a heavy coding turn + // grows it (#234). No-op until a serving daemon has booted. + crate::modules::serving_daemon::observe_serving_working_set(m.input_tokens); } match step { crate::cognition::act_observe::SettleStep::Spoke(text) => text, diff --git a/core/continuum-core/src/persona/supervisor.rs b/core/continuum-core/src/persona/supervisor.rs index 66471a2d4d..d55a966b1c 100644 --- a/core/continuum-core/src/persona/supervisor.rs +++ b/core/continuum-core/src/persona/supervisor.rs @@ -499,6 +499,14 @@ pub async fn materialize_adapters( uuid::Uuid, ) -> Option>, + // The grid-overflow EFFECTOR (governor consumer slice 4b). Given a persona's + // profile + slot, returns `Some(remote adapter)` when the governor routed her + // off-box — her node is over local capacity and a reachable peer holds her model + // (`capacity::grid_overflow::route_grid_overflow`) — so her brain runs on that + // peer via `AircRemoteInferenceAdapter`. `None` → build the local adapter from the + // factory (the common case; demand fit locally). Closure DI keeps the supervisor + // decoupled from the capacity fabric + airc handle, same shape as the lookups above. + overflow_adapter_for: impl Fn(&PersonaInferenceProfile, usize) -> Option>, ) -> Vec> { let mut out = Vec::with_capacity(plans.len()); for (slot_index, plan) in plans.into_iter().enumerate() { @@ -525,16 +533,22 @@ pub async fn materialize_adapters( continue; } }; - let adapter = match factory.build_adapter(&profile).await { - Ok(a) => a, - Err(message) => { - out.push(Err(SupervisorError::AdapterFactory { - slot_index, - role: plan.role, - message, - })); - continue; - } + // Grid-overflow effector: if the governor routed this persona off-box, her + // adapter is the airc-remote one (her brain runs on the peer that holds her + // model). Else build the local adapter from the factory — the common case. + let adapter = match overflow_adapter_for(&profile, slot_index) { + Some(remote) => remote, + None => match factory.build_adapter(&profile).await { + Ok(a) => a, + Err(message) => { + out.push(Err(SupervisorError::AdapterFactory { + slot_index, + role: plan.role, + message, + })); + continue; + } + }, }; // Warm the adapter's KV-cache / kernels BEFORE the persona // enters her service loop. Per [[init-once-handle-then-lease-zero-copy-refs]] @@ -1106,7 +1120,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); assert_eq!(factory.build_count(), 2); @@ -1153,7 +1167,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory called exactly once — for the Ok row only. @@ -1188,7 +1202,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::always_fails("simulated factory rejection"); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1211,7 +1225,7 @@ mod tests { #[tokio::test] async fn empty_plans_yields_empty_hosted() { let factory = ScriptedPersonaAdapterFactory::heuristic(); - let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None, |_, _| None).await; assert!(hosted.is_empty()); assert_eq!(factory.build_count(), 0); } @@ -1240,7 +1254,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); // `|_| None` here is the substrate-bug shape we're locking in: // the registry exists but doesn't contain this persona_id. - let hosted = materialize_adapters(plans, &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(plans, &factory, |_| None, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); // Factory MUST NOT be called when the runtime lookup fails — @@ -1300,7 +1314,7 @@ mod tests { as Arc) } }; - let hosted = materialize_adapters(plans, &factory, lookup, |_| None).await; + let hosted = materialize_adapters(plans, &factory, lookup, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory ran exactly once — for Paige, not Pax. @@ -1345,7 +1359,7 @@ mod tests { let (factory, counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; // Both slots materialize cleanly. assert_eq!(hosted.len(), 2); @@ -1359,6 +1373,56 @@ mod tests { ); } + // what this catches: the grid-overflow EFFECTOR seam — when the governor routes a + // persona off-box, `overflow_adapter_for` supplies her adapter (the airc-remote one, + // her brain on a peer) and the LOCAL factory is NOT called for that slot. Here slot 0 + // is overflow-routed (override returns a stand-in remote), slot 1 is local. Both host; + // the factory builds ONLY slot 1 (build_count == 1); both adapters still warm. This is + // the composition point where route_grid_overflow's decision becomes a real remote brain. + #[tokio::test] + async fn overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory() { + use crate::ai::heuristic_adapter::HeuristicInferenceAdapter; + let plans = vec![ + MaterializedPersonaPlan { + role: RoleId::Helper, + instance: fake_instance("Paige"), + profile: Ok(fake_profile("Paige", "model-a")), // slot 0 → overflow-routed + }, + MaterializedPersonaPlan { + role: RoleId::Coder, + instance: fake_instance("Pax"), + profile: Ok(fake_profile("Pax", "model-b")), // slot 1 → local + }, + ]; + + let (factory, _counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); + let hosted = materialize_adapters( + plans, + &factory, + StubAircCitizen::fresh_lookup(), + |_| None, + // Overflow effector: slot 0 is routed off-box → supply a stand-in "remote" + // adapter; every other slot stays local (None). + |_profile, slot| { + if slot == 0 { + Some(Arc::new(HeuristicInferenceAdapter::new()) as Arc) + } else { + None + } + }, + ) + .await; + + assert_eq!(hosted.len(), 2); + assert!(hosted.iter().all(|r| r.is_ok()), "both personas host (one remote, one local)"); + assert_eq!( + factory.build_count(), + 1, + "the local factory builds ONLY the non-overflow slot — the overflow slot's \ + adapter came from the effector, her brain runs on the peer" + ); + } + /// Warmup failure surfaces as `SupervisorError::AdapterWarmup` — /// the persona does NOT reach hosted state. Per [[no-fallbacks-ever]] /// an adapter that refuses to warm gets a typed slot failure; @@ -1375,7 +1439,7 @@ mod tests { "simulated warmup failure", ); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1411,7 +1475,7 @@ mod tests { profile: Ok(fake_profile("Paige", "model-a")), }]; let hosted_ok = - materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None) + materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None) .await; assert!(hosted_ok[0].is_ok(), "ok-warmup adapter materializes"); assert_eq!(ok_counts.warmups(), 1); @@ -1429,6 +1493,7 @@ mod tests { &factory_fail, StubAircCitizen::fresh_lookup(), |_| None, + |_, _| None, ) .await; assert!( diff --git a/protocol/typescript/airc/AircRealtimeSchema.ts b/protocol/typescript/airc/AircRealtimeSchema.ts index f5040ab98c..a8efd11857 100644 --- a/protocol/typescript/airc/AircRealtimeSchema.ts +++ b/protocol/typescript/airc/AircRealtimeSchema.ts @@ -3,4 +3,4 @@ /** * Existing Continuum schema carried by an AIRC realtime envelope. */ -export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity"; +export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity" | "grid_residency";