diff --git a/core/continuum-core/src/airc/discovery.rs b/core/continuum-core/src/airc/discovery.rs index ff525b568a..6cdd6234c1 100644 --- a/core/continuum-core/src/airc/discovery.rs +++ b/core/continuum-core/src/airc/discovery.rs @@ -148,17 +148,55 @@ async fn airc_on_path() -> bool { .unwrap_or(false) } +/// Probe attempts before declaring the substrate unresponsive. A single 5s shot +/// is a coin flip on a loaded box: Windows process spawn alone can transiently +/// eat seconds when the machine is compiling/converting/downloading (observed +/// 2026-07-28 BigMama — the updated airc answered instantly when idle, yet the +/// one-shot probe timed out mid-build and the core refused full-citizen boot). +/// Retries stay BOUNDED (3 × deadline ≈ 15s worst case) — resilient to transient +/// load, still fail-loud when airc is genuinely dead. +const DISCOVERY_PROBE_ATTEMPTS: u32 = 3; + +/// How an `airc ` probe failed — timeout (every attempt) vs spawn error. +enum ProbeError { + TimedOut, + Io(String), +} + +/// Run `airc ` with the discovery deadline and bounded retries (see +/// [`DISCOVERY_PROBE_ATTEMPTS`]). ONE probe implementation for every discovery +/// subcommand — the retry-on-transient-stall policy lives here, not copy-pasted +/// per call site. +async fn probe_airc(arg: &str) -> Result { + for attempt in 1..=DISCOVERY_PROBE_ATTEMPTS { + let call = TokioCommand::new("airc").arg(arg).output(); + match timeout(DISCOVERY_SUBPROCESS_DEADLINE, call).await { + Ok(res) => return res.map_err(|e| ProbeError::Io(e.to_string())), + Err(_) if attempt < DISCOVERY_PROBE_ATTEMPTS => { + warn!( + attempt, + "`airc {arg}` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ + — retrying (transient spawn stall under load)" + ); + } + Err(_) => {} + } + } + Err(ProbeError::TimedOut) +} + +fn probe_timeout_msg(arg: &str) -> String { + format!( + "`airc {arg}` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ + × {DISCOVERY_PROBE_ATTEMPTS} attempts — substrate is unresponsive, refusing to wait", + ) +} + async fn query_airc_endpoint() -> Result { - let call = TokioCommand::new("airc").arg("ipc-endpoint").output(); - let out = timeout(DISCOVERY_SUBPROCESS_DEADLINE, call) - .await - .map_err(|_| { - DiscoveryError::EndpointCommandFailed(format!( - "`airc ipc-endpoint` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ - — substrate is unresponsive, refusing to wait", - )) - })? - .map_err(|e| DiscoveryError::EndpointCommandFailed(e.to_string()))?; + let out = probe_airc("ipc-endpoint").await.map_err(|e| match e { + ProbeError::TimedOut => DiscoveryError::EndpointCommandFailed(probe_timeout_msg("ipc-endpoint")), + ProbeError::Io(msg) => DiscoveryError::EndpointCommandFailed(msg), + })?; if !out.status.success() { return Err(DiscoveryError::EndpointCommandFailed(format!( "exit {}: {}", @@ -200,16 +238,10 @@ pub async fn discover_default_channel() -> Result { )) }); } - let call = TokioCommand::new("airc").arg("room").output(); - let out = timeout(DISCOVERY_SUBPROCESS_DEADLINE, call) - .await - .map_err(|_| { - DiscoveryError::RoomCommandFailed(format!( - "`airc room` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ - — substrate is unresponsive, refusing to wait", - )) - })? - .map_err(|e| DiscoveryError::RoomCommandFailed(e.to_string()))?; + let out = probe_airc("room").await.map_err(|e| match e { + ProbeError::TimedOut => DiscoveryError::RoomCommandFailed(probe_timeout_msg("room")), + ProbeError::Io(msg) => DiscoveryError::RoomCommandFailed(msg), + })?; if !out.status.success() { return Err(DiscoveryError::RoomCommandFailed(format!( "exit {}: {}", @@ -241,16 +273,10 @@ pub async fn discover_default_room_name() -> Result { return Ok(raw); } } - let call = TokioCommand::new("airc").arg("room").output(); - let out = timeout(DISCOVERY_SUBPROCESS_DEADLINE, call) - .await - .map_err(|_| { - DiscoveryError::RoomCommandFailed(format!( - "`airc room` did not exit within {DISCOVERY_SUBPROCESS_DEADLINE:?} \ - — substrate is unresponsive, refusing to wait", - )) - })? - .map_err(|e| DiscoveryError::RoomCommandFailed(e.to_string()))?; + let out = probe_airc("room").await.map_err(|e| match e { + ProbeError::TimedOut => DiscoveryError::RoomCommandFailed(probe_timeout_msg("room")), + ProbeError::Io(msg) => DiscoveryError::RoomCommandFailed(msg), + })?; if !out.status.success() { return Err(DiscoveryError::RoomCommandFailed(format!( "exit {}: {}", diff --git a/core/continuum-core/src/airc/inbound_attach.rs b/core/continuum-core/src/airc/inbound_attach.rs index 061f5d5cd9..791685df7a 100644 --- a/core/continuum-core/src/airc/inbound_attach.rs +++ b/core/continuum-core/src/airc/inbound_attach.rs @@ -212,6 +212,31 @@ pub async fn publish_transcript_event( "first capacity offer heard from a grid peer", ); } + } else if let Some(beacon) = residency_beacon_from_envelope(&envelope) { + // Residency beacon (grid-overflow eligibility): fold the heard beacon into the + // process-global residency ledger, keyed on the WIRE's peer id — the orthogonal + // sibling of the capacity fold above. Our own echo lands here too (the loopback + // proof that publish→hear works before a second node exists). + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let model_count = beacon.resident_models.len(); + let is_new = crate::capacity::model_residency::global_residency_ledger().hear( + event.peer_id.as_uuid(), + beacon, + now_ms, + ); + if is_new { + crate::probe!( + class = "grid.residency.heard", + from_peer = %event.peer_id.as_uuid(), + model_count = model_count, + heard_peers = + crate::capacity::model_residency::global_residency_ledger().heard_count(), + "first residency beacon heard from a grid peer", + ); + } } else if let Some((name, payload)) = chat_posted_from_envelope(&envelope, event) { crate::probe!( class = "airc.chat.projected", @@ -284,6 +309,22 @@ fn capacity_offer_from_envelope( serde_json::from_value(payload.inline.clone()?).ok() } +/// Decode a `grid_residency` envelope's inline payload into a [`ResidencyBeacon`]. +/// Returns `None` for any other envelope — the residency sibling of +/// [`capacity_offer_from_envelope`], same honest schema gate. +fn residency_beacon_from_envelope( + envelope: &AircRealtimeEnvelope, +) -> Option { + let crate::airc::realtime::AircRealtimePayload::ExistingSchema { payload } = &envelope.payload + else { + return None; + }; + if payload.schema != crate::airc::realtime::AircRealtimeSchema::GridResidency { + return None; + } + serde_json::from_value(payload.inline.clone()?).ok() +} + /// Project a plain airc chat message into the THIN `chat:posted` bus /// payload the positron chat projection consumes (`AircChatPosted` in /// `ipc/positron_source.rs`). Returns `None` for any non-message event diff --git a/core/continuum-core/src/airc/realtime.rs b/core/continuum-core/src/airc/realtime.rs index 1d412b1a79..8606c64a30 100644 --- a/core/continuum-core/src/airc/realtime.rs +++ b/core/continuum-core/src/airc/realtime.rs @@ -53,6 +53,13 @@ pub enum AircRealtimeSchema { /// presence-of-compute. EphemeralCoalesced: latest wins, never replayed /// (a stale capacity reading is a lie). GridCapacity, + /// A node's residency beacon (`capacity::model_residency::ResidencyBeacon`) — + /// which models it holds resident, the grid-overflow ELIGIBILITY signal (a peer + /// is only a fast overflow target for a model it already holds). Orthogonal to + /// GridCapacity (concurrency): the governor composes the two. EphemeralCoalesced + /// like capacity, but published on a slower cadence — residency changes on model + /// page-in/out (minute-scale), not the 10s capacity beat. + GridResidency, } /// Handle to a payload already defined by a Continuum schema. @@ -458,7 +465,9 @@ impl AircRealtimePayload { | AircRealtimeSchema::LiveKitBridgeEvent => AircRealtimeDelivery::Control, // Capacity offers are presence-of-compute: latest wins, never // replayed — a stale reading must not outlive its freshness. - AircRealtimeSchema::GridCapacity => AircRealtimeDelivery::EphemeralCoalesced, + AircRealtimeSchema::GridCapacity | AircRealtimeSchema::GridResidency => { + AircRealtimeDelivery::EphemeralCoalesced + } _ => AircRealtimeDelivery::Durable, }, Self::Presence { event } => event.delivery(), diff --git a/core/continuum-core/src/bin/expert_observe.rs b/core/continuum-core/src/bin/expert_observe.rs new file mode 100644 index 0000000000..558e731222 --- /dev/null +++ b/core/continuum-core/src/bin/expert_observe.rs @@ -0,0 +1,342 @@ +//! expert_observe — glass-box the LIVE MoE expert routing (#230 / #229). +//! +//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached, drives +//! REAL routing over a labelled multi-domain corpus, then reports the affinity that decides +//! the paging architecture (#180): PER-DOMAIN concentration + CROSS-DOMAIN hot-set overlap. +//! +//! ## Methodology (BigMama's three guardrails, 2026-07-27) +//! 1. **Sample size.** ~7 activations/slot is far too few to tell power-law from uniform — +//! Poisson noise swamps the skew. We drive thousands of tokens so the head separates. +//! 2. **Clean samples, not a greedy loop.** A long greedy generation degenerates and +//! OVER-routes to the same experts — a false positive for locality. We PREFILL diverse +//! prompts (prefill routes every token on realistic input) with only short generation. +//! 3. **Diverse HOW — the knife-edge.** Pooling ACROSS domains measures the global +//! mixed-workload histogram; MoE specialization routes domains to different experts, so +//! pooling SMEARS toward uniform and answers the WRONG question. The paging architecture +//! serves ONE persona doing ONE coherent thing, so we measure: +//! (a) PER-DOMAIN concentration — one observer per domain = the coherent-session +//! paging headroom (top-K% activation share WITHIN a domain). +//! (b) CROSS-DOMAIN overlap — intersect each domain's hot set. Experts hot in EVERY +//! domain = the always-resident shared base (never paged); experts hot in ONE +//! domain = the pageable specialization tier. The money number: what fraction of a +//! domain's activation MASS lands on experts cold for the other domains — the +//! eviction win. +//! +//! Usage: +//! cargo run --release -p continuum-core --features metal,accelerate --bin expert_observe -- [n_gen_per_prompt] + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::Arc; + +use continuum_core::capacity::expert_observer::LiveExpertObserver; +use llama::{Batch, ContextParams, ExpertObserver, Model, ModelParams, Sampler}; + +/// Labelled corpus: each domain gets its OWN observer, fed several diverse prompts so the +/// per-domain sample is dense. The point is a COHERENT workload per observer (what a persona +/// actually does in a session), not a global mixture. +const DOMAINS: &[(&str, &[&str])] = &[ + ( + "code", + &[ + "Implement a lock-free single-producer single-consumer ring buffer in Rust using \ + atomics with Release/Acquire ordering; explain why a SeqCst fence is unnecessary \ + and how the head/tail indices wrap a power-of-two capacity without a modulo in the \ + hot path.\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\n", + "Write a React hook useDebouncedResource taking an async fetcher and debounce \ + interval that cancels in-flight requests on key change, dedupes concurrent callers, \ + and surfaces loading/error/stale-while-revalidate state without tearing under \ + concurrent mode.\n\nimport { useEffect, useRef, useState } from 'react';\n", + "Given orders(id, customer_id, placed_at, total) and refunds(order_id, amount, \ + refunded_at), write SQL returning each customer's net revenue by month, excluding \ + months with over 40% refunded, ranking customers within each month by a window \ + function.", + ], + ), + ( + "prose", + &[ + "The lighthouse keeper had not spoken to another person in forty-one days when the \ + boat appeared on the horizon. He noticed first the wrongness in the grey, and only \ + afterward resolved the shape into a hull riding low, and set down the brass polish \ + and went to the door, the cold coming up through the stone under his socks.", + "She had rehearsed the apology on the train, each version softer than the last, until \ + the words lost their edges and became a kind of weather she carried into the room. \ + Her mother was at the sink with her back turned, and for a moment neither of them \ + moved, and the tap ran over a single white plate.", + "Write the opening of a short story about a cartographer who discovers that a river \ + on his oldest map no longer exists, and who sets out on foot to find where it went, \ + narrated in close third person with attention to the texture of the walking.", + ], + ), + ( + "math", + &[ + "Prove that every finite integral domain is a field. Fix a nonzero a and consider \ + x -> a x; show injectivity, conclude surjectivity from finiteness, hence an inverse \ + of a exists. Then exhibit an infinite integral domain that is not a field to show \ + finiteness is essential.", + "Derive the closed form for the variance of a sum of two correlated random variables \ + in terms of their individual variances and covariance, then generalize to n \ + variables and interpret the cross terms as the reason diversification reduces \ + portfolio variance.", + "State and prove the pigeonhole principle, then use it to show that among any 51 \ + integers chosen from 1 to 100 there must be two that are coprime, and separately two \ + whose difference is exactly 10.", + ], + ), + ( + "science", + &[ + "Explain how the sodium-potassium pump maintains a neuron's resting potential: three \ + sodium out for two potassium in, the ATP-driven conformational change, and how the \ + electrogenic imbalance plus leak channels and the Nernst equilibria set the roughly \ + -70 mV resting potential.", + "Describe why the sky is blue in terms of Rayleigh scattering: the inverse fourth \ + power wavelength dependence, why shorter wavelengths scatter more strongly, and why \ + sunsets are red because of the longer atmospheric path length near the horizon.", + "Explain the greenhouse effect at the level of molecular physics: which atmospheric \ + gases absorb in the infrared, why their vibrational modes couple to outgoing \ + longwave radiation while nitrogen and oxygen do not, and how re-emission warms the \ + surface.", + ], + ), + ( + "planning", + &[ + "Sketch the migration plan to decompose a Rust monolith into services over a \ + command-and-event bus: identify the seams where synchronous calls become async, how \ + you preserve transactional guarantees that relied on one process, and how you roll \ + out incrementally behind a facade without a big-bang cutover.", + "Write a JSON schema for a distributed-cache eviction policy supporting LRU, LFU, and \ + TTL tiers with per-key overrides, budgets in both bytes and percentage-of-pool, and \ + validation that every declared cache class has at least one eviction dimension so \ + none grows unbounded.", + "Draft a rollout plan for a feature flag that changes a checkout flow: staged \ + percentage ramp, the metrics that gate each stage, the automatic rollback trigger, \ + and how you keep the two code paths from diverging while the flag is live.", + ], + ), +]; + +fn drive_prompt(model: &Model, observer: &Arc, prompt: &str, n_gen: usize) { + let mut ctx = model + .new_context(ContextParams { + n_ctx: 4096, + n_batch: 512, + n_seq_max: 1, + expert_observer: Some(observer.clone() as Arc), + ..Default::default() + }) + .expect("context"); + + let tokens = model.tokenize(prompt, true, false).expect("tokenize"); + let mut n_cur: i32 = 0; + for chunk in tokens.chunks(512) { + let mut batch = Batch::allocated(512, 1); + let last_global = n_cur as usize + chunk.len() - 1; + for (i, tok) in chunk.iter().enumerate() { + let pos = n_cur + i as i32; + batch.push(*tok, pos, &[0], (n_cur as usize + i) == last_global); + } + ctx.decode(&batch).expect("prefill decode"); + n_cur += chunk.len() as i32; + } + + let mut sampler = Sampler::greedy(); + for _ in 0..n_gen { + let token = sampler.sample(&ctx, -1); + if model.is_eog_token(token) { + break; + } + let mut batch = Batch::allocated(1, 1); + batch.push(token, n_cur, &[0], true); + ctx.decode(&batch).expect("gen decode"); + n_cur += 1; + } +} + +/// Sorted-desc counts → activation share captured by the top `frac` of FIRED experts. +fn top_share(counts_desc: &[u64], total: u64, frac: f64) -> f64 { + if counts_desc.is_empty() || total == 0 { + return 0.0; + } + let k = ((counts_desc.len() as f64 * frac).ceil() as usize).max(1).min(counts_desc.len()); + let s: u64 = counts_desc.iter().take(k).sum(); + 100.0 * s as f64 / total as f64 +} + +fn main() { + let args: Vec = std::env::args().collect(); + let model_path = args + .get(1) + .expect("usage: expert_observe [n_gen_per_prompt]"); + let n_gen: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(48); + + let model = Model::load( + PathBuf::from(model_path), + ModelParams { n_gpu_layers: -1, use_mmap: true }, + ) + .expect("load model"); + println!("Loaded {model_path} (vocab={})", model.n_vocab()); + + // Drive each domain into its OWN observer → per-domain hit maps. + let mut per_domain = Vec::new(); + for (label, prompts) in DOMAINS { + let obs = LiveExpertObserver::new(); + for prompt in *prompts { + drive_prompt(&model, &obs, prompt, n_gen); + } + let hits = obs.snapshot_hits(); + println!( + " domain {label:<9} : {} activations across {} experts", + obs.total_hits(), + hits.len() + ); + per_domain.push((label.to_string(), hits)); + } + + // ---- PER-DOMAIN concentration (the coherent-session paging headroom) ---- + println!("\n=== PER-DOMAIN CONCENTRATION (top-K% share vs uniform null) ==="); + println!("{:<10} {:>8} {:>8} {:>8} {:>7} {:>7} {:>7}", "domain", "total", "fired", "mean", "top1%", "top10%", "top25%"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let mean = if counts.is_empty() { 0.0 } else { total as f64 / counts.len() as f64 }; + println!( + "{label:<10} {total:>8} {:>8} {mean:>8.1} {:>6.1}% {:>6.1}% {:>6.1}%", + counts.len(), + top_share(&counts, total, 0.01), + top_share(&counts, total, 0.10), + top_share(&counts, total, 0.25), + ); + } + println!("(uniform null → top1%≈1, top10%≈10, top25%≈25; materially above = per-session paging headroom)"); + + // ---- WORKING-SET SIZE: how many experts resident to capture X% of a domain's mass ---- + // THE engineering number for #180: the resident working set you must keep hot; the rest + // pages to CPU/disk (misses on the cold tail are low-mass = infrequent). + let experts_for_mass = |counts_desc: &[u64], total: u64, frac: f64| -> usize { + if total == 0 { return 0; } + let target = frac * total as f64; + let mut acc = 0u64; + for (i, c) in counts_desc.iter().enumerate() { + acc += c; + if acc as f64 >= target { return i + 1; } + } + counts_desc.len() + }; + println!("\n=== WORKING-SET SIZE (experts resident for X% of a domain's mass; % of fired) ==="); + println!("{:<10} {:>10} {:>12} {:>12} {:>12}", "domain", "50%mass", "80%mass", "90%mass", "95%mass"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let n = counts.len().max(1); + let cell = |f: f64| { let k = experts_for_mass(&counts, total, f); format!("{k} ({:.0}%)", 100.0 * k as f64 / n as f64) }; + println!("{label:<10} {:>10} {:>12} {:>12} {:>12}", cell(0.50), cell(0.80), cell(0.90), cell(0.95)); + } + println!("(the 50%-mass column = the tight resident set; 95%-mass = keep-everything-hot floor; the GAP is the pageable tail)"); + + // ---- CROSS-DOMAIN overlap: shared base vs pageable specialization ---- + // A domain's HOT SET = the smallest set of experts capturing 50% of that domain's mass. + let hot_set = |hits: &HashMap<_, u64>| -> HashSet<_> { + let total: u64 = hits.values().sum(); + let mut ranked: Vec<(_, u64)> = hits.iter().map(|(k, v)| (*k, *v)).collect(); + ranked.sort_unstable_by(|a, b| b.1.cmp(&a.1)); + let mut acc = 0u64; + let mut set = HashSet::new(); + for (k, v) in ranked { + if (acc as f64) >= 0.5 * total as f64 { + break; + } + acc += v; + set.insert(k); + } + set + }; + let hot_sets: Vec<(String, HashSet<_>)> = + per_domain.iter().map(|(l, h)| (l.clone(), hot_set(h))).collect(); + + // Jaccard overlap matrix of hot sets. + println!("\n=== CROSS-DOMAIN HOT-SET JACCARD (top-experts-to-50%-mass) ==="); + print!("{:<10}", ""); + for (l, _) in &hot_sets { + print!("{l:>9}"); + } + println!(); + for (la, sa) in &hot_sets { + print!("{la:<10}"); + for (_lb, sb) in &hot_sets { + let inter = sa.intersection(sb).count(); + let uni = sa.union(sb).count(); + let j = if uni > 0 { inter as f64 / uni as f64 } else { 0.0 }; + print!("{:>8.2} ", j); + } + println!(); + } + + // Shared base = experts in the hot set of ALL domains. Specialization = hot in exactly one. + let n_domains = hot_sets.len(); + let mut membership: HashMap<_, usize> = HashMap::new(); + for (_l, s) in &hot_sets { + for k in s { + *membership.entry(*k).or_insert(0) += 1; + } + } + let shared_base: HashSet<_> = membership + .iter() + .filter(|(_, &c)| c == n_domains) + .map(|(k, _)| *k) + .collect(); + let specialized: HashSet<_> = membership + .iter() + .filter(|(_, &c)| c == 1) + .map(|(k, _)| *k) + .collect(); + println!( + "\nshared base (hot in ALL {n_domains} domains): {} experts", + shared_base.len() + ); + println!( + "domain-specialized (hot in exactly 1) : {} experts", + specialized.len() + ); + + // The money number: per domain, what fraction of activation MASS lands on the shared base + // vs on experts NOT in any OTHER domain's hot set (the pageable eviction win). + println!("\n=== ACTIVATION MASS: shared-base vs pageable-specialized (the eviction win) ==="); + println!("{:<10} {:>14} {:>22}", "domain", "on shared base", "on own-only specialists"); + for (li, (label, hits)) in per_domain.iter().enumerate() { + let total: u64 = hits.values().sum(); + // experts that are in THIS domain's hot set and NO other domain's hot set + let own_only: HashSet<_> = hot_sets[li] + .1 + .iter() + .filter(|k| membership.get(*k).copied().unwrap_or(0) == 1) + .copied() + .collect(); + let mut base_mass = 0u64; + let mut own_mass = 0u64; + for (k, v) in hits { + if shared_base.contains(k) { + base_mass += v; + } + if own_only.contains(k) { + own_mass += v; + } + } + let pct = |m: u64| if total > 0 { 100.0 * m as f64 / total as f64 } else { 0.0 }; + println!( + "{label:<10} {:>13.1}% {:>21.1}%", + pct(base_mass), + pct(own_mass) + ); + } + println!( + "\nRead: high own-only mass = a coherent session concentrates on experts the OTHER\n\ + domains never touch → evict them when the workload isn't that domain. High shared-base\n\ + mass = a resident core you never page. That split IS the #180 paging architecture." + ); +} diff --git a/core/continuum-core/src/capacity/grid_overflow.rs b/core/continuum-core/src/capacity/grid_overflow.rs new file mode 100644 index 0000000000..b34283d8d8 --- /dev/null +++ b/core/continuum-core/src/capacity/grid_overflow.rs @@ -0,0 +1,237 @@ +//! Grid-overflow routing — the DECISION half of the governor consumer: given a serving plan +//! that overflowed local capacity, decide which eligible peers take the overflow lanes. The +//! EFFECT half (the actual `Commands.execute("ai/generate", {aircPeer})` hop) is thin and lives +//! at the live seam; THIS is pure, deterministic, and fully unit-tested. +//! +//! ## Why overflow placement is REMOTE-ONLY (not [`super::grid::LocalFirstFitPolicy`]) +//! +//! [`super::grid::LocalFirstFitPolicy`] fills local first and floors local at `≥1` (a resident +//! model must be able to run one prefill). That floor is correct for a FRESH placement but +//! WRONG for overflow: overflow lanes are BY DEFINITION the ones that already could not fit +//! locally (`ServingPlan.grid_overflow_lanes = demand − local_lanes`). Placing them local-first +//! would re-cram the very lanes the planner just declared didn't fit — the thrash the honest +//! "over local capacity by N" signal exists to avoid. So overflow placement never touches local: +//! it spills ONLY to eligible remote peers. +//! +//! ## The two gates, composed (never absorbed) +//! +//! 1. RESIDENCY ([`ModelResidencyView::residency_eligible`]): a peer is a fast overflow target +//! only for a model it ALREADY holds — else the hop pays a cold full-weights load, defeating +//! the point. Filters the snapshot to peers holding the model. +//! 2. CONCURRENCY (the misfit-parts fit, [`super::lanes_that_fit`]): among residency-eligible + +//! REACHABLE peers, each takes at most what its OWN free budget fits for the prefill spike — +//! the same per-node-fit rule the single-device and grid policies run. +//! +//! Reachability is applied HERE (an unreachable-but-resident peer is a memory, not an offer), +//! composing cleanly with residency without either abstraction absorbing the other. +//! +//! ## Unplaced lanes are SURFACED, never dropped +//! +//! When no eligible peer can take a lane, it lands in [`OverflowRouting::unplaced`] — the honest +//! "the grid couldn't absorb N of your overflow" signal the caller queues or degrades on. Never +//! silently swallowed ([[fallbacks-are-illegal-fail-loud]]). + +use crate::identity::PeerId; + +use super::grid::GridSnapshot; +use super::lanes_that_fit; +use super::model_residency::ModelResidencyView; +use super::LeaseRequest; + +/// The routing decision for a plan's overflow lanes: where each lane lands (remote-only) plus +/// the honest count of lanes the reachable, residency-eligible grid could NOT absorb. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OverflowRouting { + /// Overflow lanes placed on named peers, most-free-first, each capped by its own fit. + pub remote: Vec<(PeerId, u32)>, + /// Overflow lanes no eligible+reachable peer could take — queue or degrade on these, + /// never drop them silently. + pub unplaced: u32, +} + +impl OverflowRouting { + /// Total overflow lanes actually placed on peers. + pub fn placed(&self) -> u32 { + self.remote.iter().map(|(_, n)| n).sum() + } +} + +/// Decide where a plan's overflow lanes run. `lease` is the demand→capacity projection the +/// serving side already built (`ModelFootprint::grid_lease_request(served_window, overflow_lanes)`): +/// `want_concurrency` = the overflow lane count, `spike_bytes` = the per-lane prefill transient. +/// `model_id` is what the overflowing node is serving (its `ServingPlan.base_model_id`) — the +/// residency key. REMOTE-ONLY by construction (see module docs): local is already saturated. +pub fn route_grid_overflow( + model_id: &str, + lease: &LeaseRequest, + residency: &ModelResidencyView, + snapshot: &GridSnapshot, + safety_margin_bytes: u64, +) -> OverflowRouting { + let want = lease.want_concurrency; + if want == 0 { + return OverflowRouting { remote: Vec::new(), unplaced: 0 }; + } + + // Gate 1 — residency: keep only peers that hold the model resident. + let eligible = residency.residency_eligible(snapshot, model_id); + + // Gate 2 — reachability + per-node fit: reachable eligible peers, most-free-first (fewest + // peers touched), each capped by its OWN budget for the prefill spike. + let mut reachable: Vec<_> = eligible.peers.iter().filter(|p| p.reachable).collect(); + reachable.sort_by(|a, b| { + b.capacity + .gpu_free_bytes_live + .cmp(&a.capacity.gpu_free_bytes_live) + }); + + let mut remaining = want; + let mut remote = Vec::new(); + for peer in reachable { + if remaining == 0 { + break; + } + let fit = lanes_that_fit( + peer.capacity.gpu_free_bytes_live, + safety_margin_bytes, + lease.spike_bytes, + ); + let take = fit.min(remaining); + if take > 0 { + remote.push((peer.peer, take)); + remaining -= take; + } + } + + // Whatever the reachable, residency-eligible grid couldn't absorb is surfaced, not dropped. + OverflowRouting { remote, unplaced: remaining } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridSnapshot, PeerCapacity}; + use crate::capacity::DeviceCapacity; + use uuid::Uuid; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + fn lease(want: u32, spike_gb: u64) -> LeaseRequest { + LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: want, + spike_bytes: spike_gb * GB, + } + } + + fn view_holding(peers: &[(u128, &[&str])]) -> ModelResidencyView { + let mut v = ModelResidencyView::new(); + for (n, models) in peers { + v.set_resident(peer_id(*n), models.iter().map(|s| s.to_string())); + } + v + } + + // what this catches: overflow placement is REMOTE-ONLY and residency+reachability gated. + // Local is never assigned lanes (it's the saturated node that overflowed). Only a peer that + // holds the model AND is reachable takes lanes; a resident-but-unreachable peer and a + // reachable-but-not-resident peer both take nothing. + #[test] + fn overflow_routes_remote_only_to_reachable_resident_peers() { + let snap = GridSnapshot { + local: dev(1), // saturated — must never receive overflow lanes + peers: vec![ + peer(1, 40, true), // resident + reachable → takes lanes + peer(2, 40, false), // resident + UNREACHABLE → nothing + peer(3, 40, true), // reachable but NOT resident → nothing + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(2, 1), &residency, &snap, GB); + + assert_eq!(routing.remote.len(), 1, "only peer 1 is eligible + reachable"); + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1)); + assert_eq!(routing.placed(), 2, "both overflow lanes fit on peer 1"); + assert_eq!(routing.unplaced, 0); + } + + // what this catches: unplaced lanes are SURFACED, never dropped. When the eligible grid + // can't fit all overflow lanes (one small peer, a big per-lane spike), the shortfall is + // reported so the caller queues/degrades — the honest "grid couldn't absorb N" signal. + #[test] + fn lanes_the_grid_cannot_absorb_are_surfaced_not_dropped() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![peer(1, 10, true)], // ~10GB free, but each lane spikes 8GB + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + // want 3 lanes, 8GB spike each: only 1 fits on the 10GB peer (net of 1GB margin). + let routing = route_grid_overflow("qwen-coder", &lease(3, 8), &residency, &snap, GB); + + assert_eq!(routing.placed(), 1, "only one lane fits the peer's budget"); + assert_eq!(routing.unplaced, 2, "the other two are surfaced, not silently dropped"); + } + + // what this catches: no overflow (want == 0) is a clean no-op — nothing placed, nothing + // unplaced. The common case (demand fit locally, grid_overflow_lanes == 0) costs nothing. + #[test] + fn zero_overflow_is_a_clean_noop() { + let snap = GridSnapshot { + local: dev(20), + peers: vec![peer(1, 40, true)], + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(0, 1), &residency, &snap, GB); + + assert!(routing.remote.is_empty()); + assert_eq!(routing.unplaced, 0); + assert_eq!(routing.placed(), 0); + } + + // what this catches: spill spreads across peers most-free-first, each capped by its OWN fit + // (the misfit-parts rule) — 12 aggregate GB across three 4GB peers can't run a 6-lane + // placement if no single peer fits it, but distinct small lanes DO spread. Two reachable + // resident peers each take their share until demand is met. + #[test] + fn spill_spreads_across_peers_most_free_first() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![ + peer(1, 20, true), // more free → sorted first + peer(2, 12, true), + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + // 4 lanes, 8GB spike: peer1 (20-1 margin=19 → 2 lanes), peer2 (12-1=11 → 1 lane) = 3, + // one unplaced. Peer 1 (most free) is filled before peer 2. + let routing = route_grid_overflow("qwen-coder", &lease(4, 8), &residency, &snap, GB); + + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1), "most-free peer first"); + assert_eq!(routing.placed() + routing.unplaced, 4, "every lane accounted for"); + assert!(routing.placed() >= 3, "peers absorb what their own budgets fit"); + } +} diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index c9fbbb0096..47122a0da5 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -30,8 +30,11 @@ pub mod expert_reconcile; pub mod expert_residency; pub mod gossip; pub mod grid; +pub mod grid_overflow; pub mod lease; +pub mod model_residency; pub mod moe_serving; +pub mod node_content; pub mod placement; pub mod recursion_depth; pub mod residency_detect; diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs new file mode 100644 index 0000000000..96cccc0f71 --- /dev/null +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -0,0 +1,382 @@ +//! Model residency across the grid — WHICH peer currently holds WHICH model resident. The +//! ELIGIBILITY half of grid-overflow routing; the CONCURRENCY half is [`super::grid`]. +//! +//! ## Why this is a separate abstraction (orthogonal, never absorbed) +//! +//! `serving_plan` reasons about MODEL RESIDENCY — can a node hold model M's weights + per-lane +//! KV at the served window. `capacity/grid` reasons about CONCURRENCY SPIKES — does a node have +//! a free lane RIGHT NOW ([`super::LeaseRequest`]). Settled with BigMama 2026-07-27: these are +//! **orthogonal and compose** — neither should absorb the other, the mapping is the only +//! crossing point ([`super::serving_plan`]'s `grid_lease_request` is that one bridge). So +//! residency does NOT belong on [`super::grid::PeerCapacity`] (that would blur the concurrency +//! abstraction with a residency fact); it lives here, and the governor **composes** the two at +//! the placement filter. +//! +//! ## Why residency gates grid overflow +//! +//! A grid-overflow hop routes a persona's generation to a peer. If that peer already holds M +//! resident, the hop is fast — it needs only a free lane (the concurrency check). If it merely +//! has free VRAM but NOT M, accepting the hop forces a cold full-weights load (seconds to +//! minutes for a large model) — which defeats the entire point of overflowing for speed. So the +//! fast overflow path is eligible only for peers that already hold M. A peer with unknown +//! residency (never beaconed) is NOT eligible for the fast path — conservative by construction, +//! same spirit as [`super::residency_detect`] never claiming a promotion is faster than it is. +//! +//! ## The compose point +//! +//! [`ModelResidencyView::residency_eligible`] takes a live [`GridSnapshot`] and returns a +//! SMALLER snapshot — local device untouched (the overflowing node holds M by definition; it's +//! the one serving it), peers filtered to those holding M. The unchanged capacity placement +//! policy ([`super::grid::LocalFirstFitPolicy`]) then runs on that smaller snapshot: it never +//! learns about residency, it just sees fewer peers. Reachability stays the policy's job — an +//! unreachable-but-resident peer survives this filter and is dropped downstream by `place()`, +//! keeping the two concerns cleanly separate. + +use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; + +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::grid::GridSnapshot; +use crate::identity::PeerId; + +/// Per-peer set of model ids that peer currently holds resident, folded from residency beacons. +/// +/// Keyed on the peer's `Uuid` (via [`PeerId::as_uuid`]) — the same choice +/// [`super::gossip::GridCapacityLedger`] makes for capacity offers, so residency and capacity +/// index the grid identically. A peer absent from the map has UNKNOWN residency (never +/// beaconed), which [`Self::holds`] reports as `false`: not eligible for the fast overflow path. +#[derive(Debug, Clone, Default)] +pub struct ModelResidencyView { + by_peer: HashMap>, +} + +impl ModelResidencyView { + pub fn new() -> Self { + Self::default() + } + + /// Record (latest-wins) the full set of models a peer currently holds resident. Latest-wins + /// because residency is a live fact — a model paged out is no longer held, so a fresh beacon + /// REPLACES the peer's set rather than merging (a merge would resurrect evicted models). + pub fn set_resident(&mut self, peer: PeerId, models: I) + where + I: IntoIterator, + S: Into, + { + self.by_peer + .insert(peer.as_uuid(), models.into_iter().map(Into::into).collect()); + } + + /// Does this peer currently hold `model_id` resident? `false` for a peer that never beaconed + /// (unknown residency) — the conservative default that keeps the fast overflow path honest. + pub fn holds(&self, peer: &PeerId, model_id: &str) -> bool { + self.by_peer + .get(&peer.as_uuid()) + .is_some_and(|set| set.contains(model_id)) + } + + /// Number of peers with a known residency beacon — probe surface, mirrors + /// [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn known_peers(&self) -> usize { + self.by_peer.len() + } + + /// Compose this residency view with a live capacity snapshot for `model_id`: keep the local + /// device (the overflowing node holds M by definition) and keep only peers that hold M + /// resident. The returned snapshot feeds the UNCHANGED capacity placement policy — concurrency + /// logic never learns about residency, it just sees a shorter peer list. Reachability is NOT + /// applied here (that stays the policy's job downstream), so an unreachable-but-resident peer + /// survives this filter and is reclaimed by `place()`. Orthogonal, composed at exactly one + /// point, neither abstraction absorbing the other. + pub fn residency_eligible(&self, snapshot: &GridSnapshot, model_id: &str) -> GridSnapshot { + let mut eligible = snapshot.clone(); + eligible.peers.retain(|peer| self.holds(&peer.peer, model_id)); + eligible + } +} + +/// One node's residency beacon — the wire payload advertising which models it currently holds +/// resident. Rides its OWN `grid_residency` `EphemeralCoalesced` envelope, separate from +/// [`super::gossip::CapacityOffer`]'s `grid_capacity`: residency changes on model page-in/out +/// (minute-scale), not the 10s capacity beat, so coupling them would either over-publish +/// residency or under-refresh capacity. Names + timestamp only; peer identity is the WIRE's +/// (the transcript event's authenticated peer id), never the payload's — same rule as +/// `CapacityOffer`, so a peer cannot beacon residency on another's behalf. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResidencyBeacon { + /// Model ids this node holds resident (warm) RIGHT NOW — its base plus any co-resident. + pub resident_models: Vec, + /// Sender clock when the reading was taken (ms since epoch). Displayed, not trusted: + /// freshness is judged by RECEIVER clock at hear-time, exactly like `CapacityOffer.at_ms`. + pub at_ms: u64, +} + +/// A heard beacon + the receiver-clock instant it arrived (the freshness anchor). +#[derive(Debug, Clone)] +struct HeardBeacon { + models: Vec, + heard_at_ms: u64, +} + +/// Beacons silent past this drop from the projected view entirely. GENEROUS versus capacity's +/// eviction because the two abstractions stay orthogonal: residency is STICKY (a model stays +/// resident across many capacity beats), and the COMPOSED capacity snapshot already gates +/// reachability — so a residency reading never has to prove liveness itself (that would blur +/// residency into concurrency). 6× the capacity eviction window: a peer whose residency we +/// haven't reheard in that long falls back to UNKNOWN (not asserted-resident), keeping the fast +/// overflow path honest rather than routing to a model a long-silent peer may have paged out. +pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; + +/// The residency beacon heartbeat — deliberately SLOWER than capacity's 10s +/// ([`super::gossip::PUBLISH_INTERVAL_MS`]) because residency changes on model page-in/out +/// (minute-scale), not the free-VRAM beat. 12 beats fit inside +/// [`RESIDENCY_EVICTION_WINDOW_MS`] — the same publish:eviction ratio capacity gossip uses, +/// so a couple of dropped beacons never evicts a still-resident peer. +pub const RESIDENCY_PUBLISH_INTERVAL_MS: u64 = RESIDENCY_EVICTION_WINDOW_MS / 12; + +/// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the +/// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness +/// discipline, different (slower) cadence and payload. +#[derive(Default)] +pub struct ResidencyLedger { + heard: DashMap, +} + +/// The one process-global residency ledger — the resource it mirrors (this node's view of who +/// holds what across the grid) is process-global, same granularity argument as +/// [`super::gossip::global_ledger`]. +pub fn global_residency_ledger() -> &'static ResidencyLedger { + static LEDGER: OnceLock = OnceLock::new(); + LEDGER.get_or_init(ResidencyLedger::default) +} + +impl ResidencyLedger { + /// Fold one heard beacon in (latest per peer wins — residency is a live fact). `from_peer` + /// is the transcript event's transport identity, never payload-declared. Returns `true` when + /// this peer is NEW to the ledger — the probe-on-join surface; steady re-beacons stay silent. + pub fn hear(&self, from_peer: Uuid, beacon: ResidencyBeacon, heard_at_ms: u64) -> bool { + self.heard + .insert( + from_peer, + HeardBeacon { models: beacon.resident_models, heard_at_ms }, + ) + .is_none() + } + + /// Project the ledger into a [`ModelResidencyView`] the governor composes with the capacity + /// snapshot, evicting beacons silent past [`RESIDENCY_EVICTION_WINDOW_MS`] as it goes. + /// `own_peer` is excluded — the local node's residency is its OWN serving truth (it holds M + /// by definition when it overflows), not a round-tripped beacon. + pub fn view(&self, own_peer: Uuid, now_ms: u64) -> ModelResidencyView { + let mut view = ModelResidencyView::new(); + self.heard.retain(|peer, heard| { + let age = now_ms.saturating_sub(heard.heard_at_ms); + if age > RESIDENCY_EVICTION_WINDOW_MS { + return false; // silent too long — residency falls back to unknown + } + if *peer != own_peer { + view.by_peer + .insert(*peer, heard.models.iter().cloned().collect()); + } + true + }); + view + } + + /// Number of peers currently on the ledger (self included if echoed) — probe surface, + /// mirrors [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn heard_count(&self) -> usize { + self.heard.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridPlacementPolicy, GridSnapshot, LocalFirstFitPolicy, PeerCapacity}; + use crate::capacity::{DeviceCapacity, LeaseRequest}; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + // what this catches: the residency ELIGIBILITY gate. Only peers that beaconed the model as + // resident survive the filter; a peer with plenty of free VRAM but NOT holding M is dropped + // (routing to it would force a cold full-weights load — the slow path the gate exists to + // avoid), and a peer that never beaconed at all (unknown residency) is dropped too. The local + // device is always kept — the overflowing node holds M by definition. + #[test] + fn only_peers_holding_the_model_are_eligible() { + let snap = GridSnapshot { + local: dev(2), + peers: vec![ + peer(1, 40, true), // holds qwen-coder + peer(2, 40, true), // holds something else, NOT qwen-coder + peer(3, 40, true), // never beaconed — unknown residency + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "embed-small"]); + view.set_resident(peer_id(2), ["llama-70b"]); + // peer 3 intentionally not recorded. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.local, snap.local, "local device is always kept — it holds M"); + assert_eq!(eligible.peers.len(), 1, "only the peer holding qwen-coder survives"); + assert_eq!(eligible.peers[0].peer.as_uuid(), Uuid::from_u128(1)); + } + + // what this catches: latest-wins REPLACE, not merge. A peer that paged qwen-coder OUT (its + // fresh beacon lists only what it still holds) must stop being eligible — a merge would + // resurrect the evicted model and route a hop to a peer that no longer has it. + #[test] + fn fresh_beacon_replaces_so_evicted_models_stop_being_eligible() { + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "llama-70b"]); + assert!(view.holds(&peer_id(1), "qwen-coder")); + + // Peer paged qwen-coder out; next beacon lists only llama-70b. + view.set_resident(peer_id(1), ["llama-70b"]); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "evicted model must not linger"); + assert!(view.holds(&peer_id(1), "llama-70b")); + } + + // what this catches: the COMPOSE contract end-to-end — residency filters WHO is eligible, + // then the unchanged capacity policy places lanes on the survivors and applies reachability + // itself. A resident+reachable peer gets lanes; a resident+UNREACHABLE peer survives the + // residency filter (reachability isn't residency's job) but is reclaimed by place(); a + // non-resident peer is absent from placement entirely. Two orthogonal gates, composed. + #[test] + fn residency_then_capacity_policy_compose() { + let snap = GridSnapshot { + local: dev(1), // no local room — force the spill onto peers + peers: vec![ + peer(1, 40, true), // resident + reachable → should get lanes + peer(2, 40, false), // resident + UNREACHABLE → reclaimed by place() + peer(3, 40, true), // reachable but NOT resident → filtered out before place() + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder"]); + view.set_resident(peer_id(2), ["qwen-coder"]); + // peer 3 holds nothing relevant. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.peers.len(), 2, "peers 1 & 2 are resident; peer 3 filtered out"); + + // Small spike so many lanes fit per peer — we're testing WHO gets lanes, not how many. + let req = LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: 4, + spike_bytes: GB, + }; + let placement = LocalFirstFitPolicy { safety_margin_bytes: GB }.place(&eligible, &req); + + // The reachable resident peer carries the remote lanes; the unreachable one gets none. + let peer1_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(1)) + .map(|(_, n)| *n) + .sum(); + let peer2_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(2)) + .map(|(_, n)| *n) + .sum(); + let peer3_present = placement.remote.iter().any(|(p, _)| p.as_uuid() == Uuid::from_u128(3)); + + assert!(peer1_lanes > 0, "resident + reachable peer must carry overflow lanes"); + assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); + assert!(!peer3_present, "non-resident peer never reaches placement"); + } + + fn beacon(models: &[&str], at_ms: u64) -> ResidencyBeacon { + ResidencyBeacon { + resident_models: models.iter().map(|s| s.to_string()).collect(), + at_ms, + } + } + + // what this catches: the beacon RECEIVE→PROJECT pipeline — a heard beacon projects into a + // ModelResidencyView that holds() reflects, and the node's OWN echoed beacon is excluded + // (the local node's residency is its own serving truth, arriving live, not a round-trip). + // This is the residency sibling of gossip's loopback contract. + #[test] + fn heard_beacon_projects_and_own_echo_is_excluded() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + ledger.hear(me, beacon(&["qwen-coder"], 1_000), 1_000); // our own beacon, round-tripped + ledger.hear(other, beacon(&["qwen-coder", "llama-70b"], 1_000), 1_000); + + let view = ledger.view(me, 2_000); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "own echo excluded from the peer view"); + assert!(view.holds(&peer_id(2), "qwen-coder"), "the real peer's residency projects"); + assert!(view.holds(&peer_id(2), "llama-70b")); + assert_eq!(view.known_peers(), 1, "only the genuine peer, not the echo"); + } + + // what this catches: eviction after long silence — a peer we haven't reheard past the + // (generous) residency window falls back to UNKNOWN residency, so the fast overflow path + // won't route to a model that long-silent peer may since have paged out. A fresh beacon + // brings it right back (grow is first-class). + #[test] + fn stale_beacon_evicts_then_a_fresh_one_restores() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let peer = Uuid::from_u128(2); + ledger.hear(peer, beacon(&["qwen-coder"], 0), 0); + assert!(ledger.view(me, 1_000).holds(&peer_id(2), "qwen-coder")); + + // Silent past the residency eviction window: gone from the view (unknown, not asserted). + let t_evict = RESIDENCY_EVICTION_WINDOW_MS + 1; + assert!( + !ledger.view(me, t_evict).holds(&peer_id(2), "qwen-coder"), + "long-silent peer's residency falls back to unknown" + ); + + // It beacons again: instantly back. + ledger.hear(peer, beacon(&["qwen-coder"], t_evict), t_evict); + assert!( + ledger.view(me, t_evict + 1).holds(&peer_id(2), "qwen-coder"), + "a returning peer's residency is adopted on its first fresh beacon" + ); + } + + // what this catches: the wire payload survives serde round-trip byte-for-byte (camelCase + // like every realtime inline payload). If resident_models renamed or at_ms narrowed, heard + // residency would silently drift from published — the grid would gate overflow on models + // nobody actually beaconed. + #[test] + fn beacon_round_trips_through_json() { + let b = beacon(&["qwen-coder", "embed-small"], 42); + let json = serde_json::to_string(&b).expect("serialize"); + assert!(json.contains("residentModels"), "camelCase wire field: {json}"); + let back: ResidencyBeacon = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, b, "beacon must survive the wire byte-for-byte"); + } +} diff --git a/core/continuum-core/src/capacity/node_content.rs b/core/continuum-core/src/capacity/node_content.rs new file mode 100644 index 0000000000..993e060fea --- /dev/null +++ b/core/continuum-core/src/capacity/node_content.rs @@ -0,0 +1,207 @@ +//! `node_content` — the grid's "who-has-what" registry: what each live node +//! HOLDS, distinct from `DeviceCapacity` (what it can FIT). +//! +//! `GridSnapshot` already answers "how many lanes fit where" (capacity). To make +//! the grid smarter than a pile of toy GPUs, the router also needs "who already +//! holds the thing this request needs" (content). That is the foundation for the +//! three grid-native serving wins ([[product-strategy-vs-exo]]): +//! 1. **cross-node expert sharding** — K3's 896 experts split across nodes; +//! route each token's ~16 ACTIVE experts to whichever node holds them (sparse +//! traffic, unlike exo's dense per-layer LAN hop). +//! 2. **prefix-aware routing** — send a multi-turn request to the node whose KV +//! cache is already warm for its prefix → skip prefill. +//! 3. **model sharing** — route to a node that already has the model resident, +//! instead of cold-loading it. +//! +//! Churn-safe by construction ([[restarts-are-commonplace]]): content is indexed +//! per live `PeerId` from a point-in-time view. A node that drops simply isn't in +//! the index — its content vanishes, queries fall back to another holder or to +//! local recompute. Nothing is held across the churn; every query re-derives. + +use std::collections::HashMap; + +use crate::identity::PeerId; + +/// Which experts of a model a node holds — the unit of cross-node MoE sharding. +/// A contiguous `[first, last]` expert range (inclusive) per (model, layer); +/// ranges compose so a query can find the holder of any activated expert id. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExpertShard { + pub model_id: String, + /// Layer this shard's experts belong to (MoE experts are per-layer). + pub layer: u32, + /// Inclusive expert-id range this node holds for `(model_id, layer)`. + pub first_expert: u32, + pub last_expert: u32, +} + +impl ExpertShard { + #[inline] + pub fn holds(&self, model_id: &str, layer: u32, expert: u32) -> bool { + self.model_id == model_id + && self.layer == layer + && expert >= self.first_expert + && expert <= self.last_expert + } +} + +/// A warm KV-cache prefix a node can continue without recomputing prefill. +/// `hash` is a rolling hash of the prompt token prefix; `token_len` lets the +/// router prefer the LONGEST warm prefix match (most prefill skipped). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WarmPrefix { + pub model_id: String, + pub hash: u64, + pub token_len: u32, +} + +/// What ONE node advertises it holds this tick. Small + cheap to gossip alongside +/// `DeviceCapacity` on the existing snapshot cadence. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NodeContent { + /// Model ids fully resident + serve-ready on this node. + pub resident_models: Vec, + /// Expert shards this node holds (for models it serves as a shard, not whole). + pub expert_shards: Vec, + /// Warm KV prefixes this node can continue. + pub warm_prefixes: Vec, +} + +impl NodeContent { + pub fn holds_model(&self, model_id: &str) -> bool { + self.resident_models.iter().any(|m| m == model_id) + } +} + +/// Where a given (model, layer, expert) lives on the grid, if anywhere live. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExpertLocation { + /// The local node holds it — no network hop. + Local, + /// A peer holds it — route the sparse activation there. + Peer(PeerId), + /// No live node holds it — fault it in locally (page from disk) as the fallback. + Fault, +} + +/// The grid-level content query. Built per point-in-time view from the LOCAL +/// node's content + the CURRENTLY-LIVE peers' content. A dropped peer is absent, +/// so every answer names only reachable holders. +#[derive(Debug, Clone, Default)] +pub struct GridContentIndex { + local: NodeContent, + peers: HashMap, +} + +impl GridContentIndex { + pub fn new(local: NodeContent, peers: HashMap) -> Self { + Self { local, peers } + } + + /// Prefix-aware routing: the live node with the LONGEST warm prefix for this + /// model whose hash matches — or None (do prefill locally). `None` peer means + /// the local node is the best holder. + pub fn best_prefix_holder(&self, model_id: &str, hash: u64) -> Option<(Option, u32)> { + let mut best: Option<(Option, u32)> = None; + let mut consider = |peer: Option, c: &NodeContent| { + for p in &c.warm_prefixes { + if p.model_id == model_id && p.hash == hash { + if best.as_ref().map(|(_, len)| p.token_len > *len).unwrap_or(true) { + best = Some((peer.clone(), p.token_len)); + } + } + } + }; + consider(None, &self.local); + for (id, c) in &self.peers { + consider(Some(id.clone()), c); + } + best + } + + /// Cross-node expert sharding: where a token's activated expert lives. Local + /// wins (no hop); else the first live peer holding it; else Fault (page local). + pub fn locate_expert(&self, model_id: &str, layer: u32, expert: u32) -> ExpertLocation { + if self.local.expert_shards.iter().any(|s| s.holds(model_id, layer, expert)) { + return ExpertLocation::Local; + } + for (id, c) in &self.peers { + if c.expert_shards.iter().any(|s| s.holds(model_id, layer, expert)) { + return ExpertLocation::Peer(id.clone()); + } + } + ExpertLocation::Fault + } + + /// Model sharing: live nodes with this model fully resident (local first). + pub fn nodes_with_model(&self, model_id: &str) -> Vec> { + let mut out = Vec::new(); + if self.local.holds_model(model_id) { + out.push(None); + } + for (id, c) in &self.peers { + if c.holds_model(model_id) { + out.push(Some(id.clone())); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn peer(n: u128) -> PeerId { + PeerId::from_u128(n) + } + + // what this catches: expert-sharding routing must send an activated expert to + // the node that actually holds its range, and Fault when nobody live does — + // a wrong holder or a silent skip corrupts the MoE forward. + #[test] + fn locate_expert_routes_to_holder_and_faults_when_absent() { + let local = NodeContent { + expert_shards: vec![ExpertShard { + model_id: "k3".into(), layer: 5, first_expert: 0, last_expert: 447, + }], + ..Default::default() + }; + let mut peers = HashMap::new(); + peers.insert(peer(2), NodeContent { + expert_shards: vec![ExpertShard { + model_id: "k3".into(), layer: 5, first_expert: 448, last_expert: 895, + }], + ..Default::default() + }); + let idx = GridContentIndex::new(local, peers); + assert_eq!(idx.locate_expert("k3", 5, 10), ExpertLocation::Local); + assert_eq!(idx.locate_expert("k3", 5, 500), ExpertLocation::Peer(peer(2))); + // expert id out of every live shard's range -> fault in locally + assert_eq!(idx.locate_expert("k3", 5, 900), ExpertLocation::Fault); + // a DROPPED peer: rebuild index without it -> its experts now Fault + let idx2 = GridContentIndex::new( + NodeContent { expert_shards: vec![ExpertShard { + model_id: "k3".into(), layer: 5, first_expert: 0, last_expert: 447 }], ..Default::default() }, + HashMap::new()); + assert_eq!(idx2.locate_expert("k3", 5, 500), ExpertLocation::Fault); + } + + // what this catches: prefix-aware routing must pick the LONGEST warm match + // (most prefill skipped), not just any match. + #[test] + fn best_prefix_holder_prefers_longest() { + let local = NodeContent { + warm_prefixes: vec![WarmPrefix { model_id: "m".into(), hash: 7, token_len: 100 }], + ..Default::default() + }; + let mut peers = HashMap::new(); + peers.insert(peer(3), NodeContent { + warm_prefixes: vec![WarmPrefix { model_id: "m".into(), hash: 7, token_len: 500 }], + ..Default::default() + }); + let idx = GridContentIndex::new(local, peers); + assert_eq!(idx.best_prefix_holder("m", 7), Some((Some(peer(3)), 500))); + assert_eq!(idx.best_prefix_holder("m", 999), None); // no warm match -> prefill local + } +} diff --git a/core/continuum-core/src/cognition/gym.rs b/core/continuum-core/src/cognition/gym.rs index 6cedcb9190..54798b3e16 100644 --- a/core/continuum-core/src/cognition/gym.rs +++ b/core/continuum-core/src/cognition/gym.rs @@ -84,6 +84,15 @@ const EMBEDDED_GYMS: &[(&str, &str)] = &[ "frontier-rs.jsonl", include_str!("../../../../docs/genome/frontier-rs.jsonl"), ), + ( + // livecodebench-rs: OUR Rust-graded Program-Bench proxy (10 competitive- + // programming tasks — KMP, union-find, Dijkstra, DP, two-pointer). Named + // -rs to stay honest: it is NOT the real LiveCodeBench dataset (that stays a + // catalogued stub for a future stdio runner), it is the runnable Rust proxy + // mapped to Program Bench in BENCHMARK-TARGET-MAP.md. Every test hand-verified. + "livecodebench-rs.jsonl", + include_str!("../../../../docs/genome/livecodebench-rs.jsonl"), + ), ( // games-rs: OUR games benchmark — the tier public benchmarks lack (they grade // an agent PLAYING, not BUILDING). Auto-verifiable game LOGIC: a Conway step, diff --git a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs index f5249b4c08..eb91e0a82b 100644 --- a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs +++ b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs @@ -1734,19 +1734,20 @@ mod tests { ); } - // what this catches: progressive disclosure — the per-turn tool PAYLOAD is the - // two-tool DISCOVERY PAIR (`commands/list` + `commands/help`), not the whole - // authorized registry, and the system prompt carries only a CATEGORY INDEX, not - // every tool. The old dump injected ~150 full schemas / one-liners (~4–5k tokens) - // into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute. Now the - // surface is a tiny category index inside the system prompt + the two-tool native - // offering, so even a huge tool set leaves system + user + the offered tools well - // within the served window. Invariant: the category index (not tool names) rides - // the system prompt, the native offering is exactly the discovery pair, and the - // whole prompt + its tools + reserve fit the window. A regression means the dump - // came back. + // what this catches: progressive disclosure — the system prompt carries only a + // CATEGORY INDEX (names, not the ~150 full schemas), and the native offering is the + // FULL authorized coding surface, which is NEVER window-amputated. Two failure modes + // this pins the absence of: (1) the old dump that injected ~150 schemas (~4–5k + // tokens) into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute; + // (2) the later "tight window ⇒ discovery-pair only" shrink cliff (deleted in #206 / + // 74acbb36c) that amputated the native surface to `commands/list`+`commands/help` on + // a tight window, stranding native-tool models in a help loop with 0 edits. Now the + // category index rides the system prompt, the full native surface rides beside it + // un-amputated, and the budget (`prompt_view_within`) reserves the specs' tokens and + // trims VOLATILE context so prompt + tools + reserve fit the window. A regression + // means either the dump came back or the amputation cliff did. #[test] - fn tool_surface_is_a_category_index_plus_discovery_pair() { + fn tool_surface_is_a_category_index_plus_the_unamputated_native_surface() { let persona = Uuid::new_v4(); let adapter: Arc = Arc::new(HeuristicInferenceAdapter::new()); // A tool set whose FULL schemas would dwarf the window — the live shape diff --git a/core/continuum-core/src/cognition/mod.rs b/core/continuum-core/src/cognition/mod.rs index 1aac35dcf7..cbb9a2fade 100644 --- a/core/continuum-core/src/cognition/mod.rs +++ b/core/continuum-core/src/cognition/mod.rs @@ -77,6 +77,7 @@ pub mod response_orchestrator; pub mod response_validator; pub mod serving_plan; pub mod shared_analysis; +pub mod working_set_demand; pub mod self_repeat; pub mod should_respond; pub mod should_respond_module; diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 1bdbf43dec..9651511c70 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -260,6 +260,29 @@ impl ModelFootprint { self.resident_bytes(served_window, lanes) .saturating_add(self.prefill_compute_reserve(served_window, lanes)) } + + /// The capacity-fabric [`LeaseRequest`](crate::capacity::LeaseRequest) for serving + /// this model at `served_window` with `demand_lanes` concurrent minds — the bridge + /// from the serving plan's MODEL-RESIDENCY view (weights + per-lane KV) to the grid's + /// CONCURRENCY-SPIKE view, so [`GridPlacementPolicy`](crate::capacity::grid) can place + /// overflow lanes onto peers (#180 grid spill / [[frontier-is-a-scaling-question-over-misfits-not-a-capability-question]]). + /// `want_concurrency` = the minds that want a lane; `spike_bytes` = ONE lane's transient + /// prefill compute buffer at this window (the term the 2026-07-14 OOM turned on), so a + /// peer is only offered a spill lane it can actually hold. NOTE: this sizes the + /// CONCURRENCY spike only; a peer must ALSO already hold this model resident — that + /// residency gate is the gossip-side half of the routing (owned with the grid snapshot), + /// NOT this pure mapping. + pub fn grid_lease_request( + &self, + served_window: u32, + demand_lanes: u32, + ) -> crate::capacity::LeaseRequest { + crate::capacity::LeaseRequest { + consumer: self.model_id.clone(), + want_concurrency: demand_lanes.max(1), + spike_bytes: self.prefill_compute_reserve(served_window, 1), + } + } } /// The serving decision for this host. @@ -310,12 +333,25 @@ pub struct ServingPlan { /// rendered, and the room degenerated into a greeting loop. 2 slots would /// have doubled every mind's window with zero lost concurrency. /// +/// `demand_ceil` is the LIVE demand ceiling for the served window — the ELASTIC, +/// per-task upper bound. `window_for` sizes the window UP to what the budget +/// allows; this caps it DOWN to what the task actually needs. A hard coding task +/// passes a high ceiling and the window GROWS (up to the budget/model bound); a +/// simple turn passes a low one so more lanes fit. It is NEVER a launch-baked +/// constant — callers thread live per-persona/per-task demand (measured p95 + +/// headroom, or a task's explicit request). [`plan_serving`] supplies +/// [`BOOTSTRAP_WORKING_SET`] only as the cold-start prior until that telemetry +/// exists (#234). OOM-safe: `window_for` already bounds the window to the budget, +/// so a higher ceiling only raises the cap toward that bound, never past it. +/// [[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]] +/// /// The decision is pure classification on memory arithmetic — no model is /// loaded, no inference is run. -pub fn plan_serving( +pub fn plan_serving_with_demand( host: HostBudget, candidates: &[ModelFootprint], demand_lanes: u32, + demand_ceil: u32, ) -> Option { if candidates.is_empty() { return None; @@ -441,7 +477,7 @@ pub fn plan_serving( // never forces UP past what the model supports; `.max(MIN_SERVE_CTX)` keeps it // runnable. On a small host where window_for < BOOTSTRAP the cap is a no-op. let served_context_window = window_for(lanes as u64) - .min(BOOTSTRAP_WORKING_SET) + .min(demand_ceil.max(MIN_SERVE_CTX)) .max(MIN_SERVE_CTX); // The honest per-lane compute reserve AT the chosen window (floor + window-scaled), // reused by the packing math below AND reported to the board via @@ -501,6 +537,20 @@ pub fn plan_serving( }) } +/// Cold-start / no-live-demand convenience: [`plan_serving_with_demand`] with the +/// [`BOOTSTRAP_WORKING_SET`] prior as the demand ceiling. Callers that do not YET +/// thread live per-task demand use this; the elastic path (a hard task requesting +/// more context, or the #234 p95 telemetry) calls `plan_serving_with_demand` with +/// the measured/requested ceiling so the window ebbs and flows with real demand +/// rather than a baked constant. +pub fn plan_serving( + host: HostBudget, + candidates: &[ModelFootprint], + demand_lanes: u32, +) -> Option { + plan_serving_with_demand(host, candidates, demand_lanes, BOOTSTRAP_WORKING_SET) +} + /// Hysteresis wrapper around [`plan_serving`]: stops model THRASH from live- /// budget jitter. Keeps the `incumbent` model as long as it still fits the /// budget — switching DOWN only when the incumbent no longer fits (forced @@ -514,13 +564,14 @@ pub fn plan_serving_stable( candidates: &[ModelFootprint], incumbent: Option<&str>, demand_lanes: u32, + demand_ceil: u32, ) -> Option { // NB: do NOT `?`-bail here. A deep transient dip can leave `plan_serving` // with nothing fitting the depressed budget (`fresh` = None) while a model // is STILL resident and serving fine — its memory is its own. Tearing that // down to "nothing" is the exact harm we're guarding against, so `fresh` is // an Option we fall back to only when the incumbent genuinely can't hold. - let fresh = plan_serving(host, candidates, demand_lanes); + let fresh = plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil); let Some(inc_id) = incumbent else { return fresh; }; @@ -572,7 +623,7 @@ pub fn plan_serving_stable( if let Some(m) = promoted.iter_mut().find(|m| m.model_id == inc_id) { m.capability_rank = u8::MAX; } - plan_serving(at_rest, &promoted, demand_lanes) + plan_serving_with_demand(at_rest, &promoted, demand_lanes, demand_ceil) } fn bytes_gb(bytes: u64) -> f64 { @@ -765,6 +816,70 @@ mod tests { ); } + // what this catches: the ELASTIC demand ceiling (Joel 2026-07-27: "context window sizes + // should ebb and flow depending on demands of the task and available resources — if it + // needs it larger for a moment, don't limit it"). The ceiling is threaded LIVE, not a + // launch-baked constant: a hard task passing a HIGH demand_ceil grows the served window + // PAST the BOOTSTRAP prior (up to the budget/model bound); a LOW ceiling shrinks it so + // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises + // the cap toward the budget bound, never past it. This is the "stop setting it in stone + // at launch" fix that opens the elastic-lease path. + // what this catches: the serving→grid bridge (#180 spill) — a model footprint maps to a + // capacity-fabric LeaseRequest whose want_concurrency is the demanded lanes and whose + // spike_bytes is ONE lane's transient prefill compute reserve at the served window (the + // 2026-07-14 OOM term), so GridPlacementPolicy offers a peer only a spill lane it can hold. + #[test] + fn grid_lease_request_maps_demand_and_the_prefill_spike() { + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + let window = 16_384; + let lease = devstral.grid_lease_request(window, 3); + assert_eq!(lease.consumer, "devstral-24b"); + assert_eq!(lease.want_concurrency, 3, "want_concurrency = demanded lanes"); + assert_eq!( + lease.spike_bytes, + devstral.prefill_compute_reserve(window, 1), + "spike_bytes = ONE lane's prefill compute reserve at the served window" + ); + // Zero demand floors at one lane — never a degenerate 0-concurrency lease. + assert_eq!(devstral.grid_lease_request(window, 0).want_concurrency, 1); + } + + #[test] + fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + // Roomy host + high trained ceiling → window_for(1) far exceeds any of these ceilings, + // so the DEMAND ceiling (not the budget or the model) decides the served window. + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + // Cold prior: default plan_serving caps at BOOTSTRAP_WORKING_SET. + let cold = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); + assert_eq!(cold.served_context_window, BOOTSTRAP_WORKING_SET); + + // Hard task demands more context → the window GROWS past the prior. + let big_ceil = 64_000; + let hot = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, big_ceil).unwrap(); + assert!( + hot.served_context_window > cold.served_context_window, + "a higher demand ceiling must GROW the window: hot {} ≤ cold {}", + hot.served_context_window, + cold.served_context_window + ); + assert!( + hot.served_context_window <= big_ceil, + "growth stays bounded by the demand ceiling (and the budget), never past it: got {}", + hot.served_context_window + ); + + // Simple turn demands little → the window SHRINKS below the prior, freeing memory. + let lean = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, 8_192).unwrap(); + assert_eq!( + lean.served_context_window, 8_192, + "a low demand ceiling shrinks the served window to it" + ); + } + // what this catches: lanes DEGRADE on a tight host — MAX_LANES is a sanity backstop, // the fit math is the real cap. A 4-persona demand on a budget that can't feed 4 warm // slots must serve FEWER (well-fed) lanes, never 4 starving ones that OOM. The window @@ -991,7 +1106,7 @@ mod tests { fn stable_with_no_incumbent_equals_plain() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; assert_eq!( - plan_serving_stable(host, &pair(), None, MAX_LANES), + plan_serving_stable(host, &pair(), None, MAX_LANES, BOOTSTRAP_WORKING_SET), plan_serving(host, &pair(), MAX_LANES) ); } @@ -1004,7 +1119,7 @@ mod tests { // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. let host = HostBudget { usable_bytes: 10 * GB, perf_cores: 6 }; assert_eq!(plan_serving(host, &pair(), MAX_LANES).unwrap().base_model_id, "big", "fresh would pick big"); - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "hysteresis keeps incumbent — no flap"); assert!(stable.lanes >= 1, "lanes still re-tracked for the kept model"); } @@ -1014,7 +1129,7 @@ mod tests { #[test] fn stable_upgrades_when_better_model_fits_with_headroom() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; // big 9.7 << 0.9*20=18 - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "more capable + ample headroom → upgrade"); } @@ -1029,7 +1144,7 @@ mod tests { fn stable_forced_down_when_incumbent_gone_from_disk() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; let only_small = vec![fp("small", 1, 4_000, 32_768, 1)]; // "big" no longer on disk - let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "incumbent gone from disk → serve what's present"); } @@ -1053,7 +1168,7 @@ mod tests { "depressed-budget plain plan would flap to the smaller model" ); // With the incumbent credited its own weights back, the resident big stays. - let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "incumbent survives its OWN load dip — no flap"); assert!(stable.lanes >= 1, "kept model still gets ≥1 lane"); } diff --git a/core/continuum-core/src/cognition/working_set_demand.rs b/core/continuum-core/src/cognition/working_set_demand.rs new file mode 100644 index 0000000000..78b1a8d043 --- /dev/null +++ b/core/continuum-core/src/cognition/working_set_demand.rs @@ -0,0 +1,198 @@ +//! `working_set_demand` — the DEMAND PRODUCER for the elastic served window (#234). +//! +//! A persona's served context window should ebb and flow with what its turns +//! ACTUALLY use, not a launch-baked constant ([[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]]). +//! This observes the assembled-prompt token count of recent turns and produces a +//! live demand ceiling that threads into +//! [`plan_serving_with_demand`](super::serving_plan::plan_serving_with_demand) — +//! growing the window for a persona whose tasks got bigger (a hard coding session) +//! and ebbing it back when they get lean again, so many personas stay warm at a +//! lean window and one doing heavy work gets room to think. +//! +//! ## Measured, not guessed — and never truncate the current turn +//! +//! Two signals combine: +//! - a rolling **p95 baseline** ([`demand_ceil`]) sizes the WARM lane to the +//! persona's usual work (so it's already big when the next hard turn arrives); +//! - the **measured current prompt** ([`demand_for`]) guarantees THIS turn is +//! never truncated — we already assembled the prompt, so we know its exact +//! size and request precisely that + generation headroom. +//! +//! That is how "if it needs it larger for a moment, don't limit it" holds WITHOUT +//! guessing. The budget fit in `plan_serving_with_demand` still bounds the result +//! above, so an impossibly large prompt degrades honestly, never OOMs. p95 (not +//! max) for the baseline so a single rare spike doesn't pre-allocate a KV that +//! swaps the box — the spike is handled per-turn by the measured path instead. +//! +//! Pure + testable: token counts in, demand ceiling out. No serving loop, no GPU +//! here — this is the measurement that makes the elastic lease honest. + +use std::collections::VecDeque; + +/// Below this many samples [`demand_ceil`] returns the cold `floor` rather than a +/// p95 off 1–2 points: provisioning a window off a tiny sample swings it turn to +/// turn (thrash). A handful of turns is enough to trust the shape without waiting +/// so long the window never grows within a short session. +const MIN_SAMPLES_FOR_P95: usize = 4; + +/// Rolling observer of a persona's per-turn working-set size (assembled-prompt +/// tokens), producing the live demand ceiling for its served window. +pub struct WorkingSetDemand { + /// Recent assembled-prompt token counts, oldest at the front. Bounded to + /// `window`, so a past hard session doesn't pin the ceiling forever. + samples: VecDeque, + /// Rolling window length (turns). ≥ 1. + window: usize, + /// Cold-start floor AND the hard minimum — the ceiling never drops below this, + /// so a lull can't starve the next turn's first prompt. Callers pass the + /// serving cold prior (`serving_plan::BOOTSTRAP_WORKING_SET`). + floor: u32, + /// Added to the observed prompt to leave room for the turn's GENERATION (the + /// prompt is what's assembled; the model still needs to write its answer). + gen_headroom: u32, +} + +impl WorkingSetDemand { + pub fn new(window: usize, floor: u32, gen_headroom: u32) -> Self { + let window = window.max(1); + Self { + samples: VecDeque::with_capacity(window), + window, + floor, + gen_headroom, + } + } + + /// Record one completed turn's assembled-prompt token count. Oldest evicted + /// past `window` so demand EBBS back — a past hard session doesn't pin the + /// window forever. + pub fn observe(&mut self, prompt_tokens: u32) { + if self.samples.len() == self.window { + self.samples.pop_front(); + } + self.samples.push_back(prompt_tokens); + } + + /// The sustained baseline demand ceiling: `max(floor, p95(recent prompts) + + /// gen_headroom)`. Floored so a lull can't starve the next turn; p95 (not max) + /// so a lone spike doesn't over-provision. Returns `floor` until there is + /// enough evidence ([`MIN_SAMPLES_FOR_P95`]) to trust the shape. + pub fn demand_ceil(&self) -> u32 { + if self.samples.len() < MIN_SAMPLES_FOR_P95 { + return self.floor; + } + let mut v: Vec = self.samples.iter().copied().collect(); + v.sort_unstable(); + let idx = (((v.len() - 1) as f64) * 0.95).round() as usize; + let p95 = v[idx.min(v.len() - 1)]; + self.floor.max(p95.saturating_add(self.gen_headroom)) + } + + /// The demand ceiling for a SPECIFIC turn whose assembled prompt we've already + /// MEASURED: never below what this turn actually needs + /// (`current_prompt_tokens + gen_headroom`), and never below the sustained p95 + /// baseline (so the warm lane stays sized for the persona's usual work). This + /// is how "don't limit it for a moment" holds without guessing — we measured + /// the prompt, so we request exactly enough. `plan_serving_with_demand` still + /// bounds it by the budget above. + pub fn demand_for(&self, current_prompt_tokens: u32) -> u32 { + self.demand_ceil() + .max(current_prompt_tokens.saturating_add(self.gen_headroom)) + .max(self.floor) + } + + /// How many turns are in the rolling window (for observability / tests). + pub fn sample_count(&self) -> usize { + self.samples.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The serving cold prior (serving_plan::BOOTSTRAP_WORKING_SET) + one generation + // of headroom — the values a live caller threads in. + const FLOOR: u32 = 16_384; + const HEAD: u32 = 2_048; + + // what this catches: cold start returns the FLOOR — provisioning a window off + // 1–2 samples would thrash it turn to turn, so it grows only once there's + // enough evidence to trust the shape. + #[test] + fn cold_start_returns_the_floor_until_enough_samples() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + assert_eq!(d.demand_ceil(), FLOOR); + d.observe(40_000); + d.observe(40_000); + assert_eq!(d.demand_ceil(), FLOOR, "still too few samples to trust the shape"); + } + + // what this catches: a sustained hard-coding session GROWS the baseline to + // p95 + generation headroom — the "if it needs it larger, don't limit it" case, + // measured not guessed. + #[test] + fn sustained_large_working_set_grows_the_ceiling_past_the_floor() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(40_000); + } + assert_eq!(d.demand_ceil(), 40_000 + HEAD, "grows to p95(40k) + headroom"); + assert!(d.demand_ceil() > FLOOR); + } + + // what this catches: demand EBBS BACK — a past hard session doesn't pin the + // window forever; once lean turns roll through the window, the ceiling returns + // to the floor and frees memory for more lanes. + #[test] + fn ceiling_ebbs_back_to_floor_as_lean_turns_roll_through_the_window() { + let mut d = WorkingSetDemand::new(8, FLOOR, HEAD); + for _ in 0..8 { + d.observe(40_000); + } + assert!(d.demand_ceil() > FLOOR); + for _ in 0..8 { + d.observe(3_000); // fully replaces the window + } + assert_eq!(d.demand_ceil(), FLOOR, "lean turns pull the ceiling back to the floor"); + } + + // what this catches: a single rare SPIKE does not over-provision the WARM + // baseline — p95 (not max) excludes the outlier so one 120k turn among lean + // ones can't pre-allocate a KV that swaps the box. + #[test] + fn single_spike_does_not_over_provision_the_baseline() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..31 { + d.observe(3_000); + } + d.observe(120_000); // one outlier among 31 lean turns + assert!( + d.demand_ceil() <= (3_000 + HEAD).max(FLOOR), + "the lone spike must not lift the sustained baseline: {}", + d.demand_ceil() + ); + } + + // what this catches: the MEASURED current turn is NEVER truncated — a big prompt + // we already assembled requests exactly its size + headroom even before the p95 + // baseline has grown ("don't limit it for a moment", from measurement not a + // guess); a small turn still gets at least the warm baseline. + #[test] + fn demand_for_never_truncates_the_measured_current_turn() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(20_000); // baseline ~22k + } + assert_eq!( + d.demand_for(60_000), + 60_000 + HEAD, + "a measured 60k turn requests its own size + headroom, above the baseline" + ); + assert_eq!( + d.demand_for(1_000), + d.demand_ceil(), + "a tiny turn still rides the warm sustained baseline" + ); + } +} diff --git a/core/continuum-core/src/commands/benchmark.rs b/core/continuum-core/src/commands/benchmark.rs index 47d2e4461d..5f6e3f995d 100644 --- a/core/continuum-core/src/commands/benchmark.rs +++ b/core/continuum-core/src/commands/benchmark.rs @@ -78,6 +78,20 @@ pub fn known_benchmarks() -> &'static [BenchmarkSpec] { eval_set: Some("frontier-rs.jsonl"), source_url: None, }, + BenchmarkSpec { + name: "livecodebench-rs", + description: "LiveCodeBench-rs — OUR runnable Rust-graded proxy for Program Bench / \ + LiveCodeBench: 10 competitive-programming tasks (KMP substring, union-find \ + components, heap Dijkstra, longest-palindrome, max-product subarray, rotated \ + binary search, trapping rain water, meeting rooms, house robber, k-th largest), \ + rustc compile+run graded, every test hand-verified. Distinct from the \ + `livecodebench` stub (the real dataset, pending a stdio runner) — see \ + BENCHMARK-TARGET-MAP.md.", + grader: Grader::Rust, + tasks: 10, + eval_set: Some("livecodebench-rs.jsonl"), + source_url: None, + }, BenchmarkSpec { name: "games-rs", description: "Games-Rust — OUR games benchmark, the tier public benchmarks lack (they \ diff --git a/core/continuum-core/src/commands/memory/mod.rs b/core/continuum-core/src/commands/memory/mod.rs index b8937336ca..f3b42d5e1a 100644 --- a/core/continuum-core/src/commands/memory/mod.rs +++ b/core/continuum-core/src/commands/memory/mod.rs @@ -37,6 +37,7 @@ pub mod multi_layer_recall; pub mod import; pub mod recall_hook; pub mod remember; +pub mod replicate_batch; pub mod share; use append_event::MemoryAppendEvent; @@ -48,6 +49,7 @@ use multi_layer_recall::MemoryMultiLayerRecall; use import::MemoryImport; use recall_hook::MemoryRecallHook; use remember::MemoryRemember; +use replicate_batch::MemoryReplicateBatch; use share::MemoryShare; /// Result of an incremental append (`memory/append-memory`, `memory/append-event`). @@ -71,6 +73,7 @@ pub fn command_objects(state: Arc) -> Vec> { Arc::new(MemoryRemember { state: state.clone() }), Arc::new(MemoryConsolidate { state: state.clone() }), Arc::new(MemoryShare { state: state.clone() }), + Arc::new(MemoryReplicateBatch { state: state.clone() }), Arc::new(MemoryConsciousnessContext { state: state.clone() }), Arc::new(MemoryAppendMemory { state: state.clone() }), Arc::new(MemoryAppendEvent { state }), @@ -134,7 +137,7 @@ pub(crate) async fn persist_memory( if let Some(embedding) = &memory.embedding { data[EMBEDDING_KEY] = serde_json::json!(embedding); } - executor + let result = executor .execute_json( "data/create", serde_json::json!({ @@ -144,7 +147,15 @@ pub(crate) async fn persist_memory( "data": data, }), ) - .await + .await; + if result.is_ok() { + // Persona-RAID slice 1: tee the ADMITTED record into the write-behind + // journal (docs/architecture/PERSONA-RAID-WRITE-BEHIND.md). This is the + // ONE durable-admit funnel, so this is the one tee. Best-effort-loud — + // never fails the admit. + state.replication.journal_admit(persona_id, "memory", &data); + } + result .map_err(|e| { CommandError::Internal(format!( "memory/append-memory: durable write to {} failed: {e}", diff --git a/core/continuum-core/src/commands/memory/replicate_batch.rs b/core/continuum-core/src/commands/memory/replicate_batch.rs new file mode 100644 index 0000000000..108270ba94 --- /dev/null +++ b/core/continuum-core/src/commands/memory/replicate_batch.rs @@ -0,0 +1,60 @@ +//! `memory/replicate-batch` — persona-RAID slice 2's RECEIVING side. +//! +//! A peer's shipper posts a tail of its persona journal here (over the same +//! inbound command-RPC that answers any command); this node appends it to its +//! replica cold store (`~/.continuum/replicas//journal.jsonl`) and +//! acks the high-water seq. Idempotent by (origin_node, seq) — the shipper +//! retries blindly on a lost ack. Design: +//! docs/architecture/PERSONA-RAID-WRITE-BEHIND.md. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +use std::sync::Arc; + +use crate::memory::replication::JournalEntry; +use crate::modules::memory::MemoryState; +use crate::sdk_codegen::CommandError; + +/// Params for `memory/replicate-batch`. Wire keys are snake_case. +#[derive(Debug, Clone, Serialize, Deserialize, TS, JsonSchema)] +#[ts( + export, + export_to = "../../../protocol/typescript/memory/MemoryReplicateBatchParams.ts" +)] +pub struct MemoryReplicateBatchParams { + /// Whose memory this is (the persona being replicated). + pub persona_id: String, + /// A contiguous tail of the origin node's journal, ascending seq, ONE + /// origin_node across the batch. + pub entries: Vec, +} + +/// Ack: the receiver's high-water for (persona, origin) after the append. +#[derive(Debug, Clone, Serialize, Deserialize, TS, JsonSchema)] +#[ts( + export, + export_to = "../../../protocol/typescript/memory/MemoryReplicateBatchResult.ts" +)] +pub struct MemoryReplicateBatchResult { + /// Highest seq durably held for this (persona, origin) — the shipper's + /// next tail starts after this. + pub acked_seq: u64, +} + +crate::action_command! { + /// Accept a replica batch of another node's persona journal into this + /// node's cold store. Privileged: grid peers reach it through the + /// command-RPC pump; it writes only under the node's replicas root. + pub struct MemoryReplicateBatch { state: Arc } + name: "memory/replicate-batch", + access: Privileged, + params: MemoryReplicateBatchParams, + output: MemoryReplicateBatchResult, + run(this, _ctx, p) => { + let acked = this.state.replication.replica_append_batch(&p.persona_id, &p.entries) + .map_err(|e| CommandError::Invalid(format!("memory/replicate-batch: {e}")))?; + Ok(MemoryReplicateBatchResult { acked_seq: acked }) + } +} diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index e54e31cec5..7bd27ca9ef 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -1354,6 +1354,38 @@ impl LlamaServerControl for LlamaServerProcess { // surfaces the real defect: a RAG budget that overshot the served // window ([[fallbacks-are-illegal-fail-loud]]). .arg("--no-context-shift"); + // KV CACHE QUANTIZATION (#232, opt-in field-proven technique). f16 KV is the + // default; q8_0 is ~half the resident KV footprint at near-lossless quality, + // freeing memory the elastic window (#234) can spend on a BIGGER context or MORE + // warm lanes — faster for multiple personas AND more room for hard coding. + // OFF by default: not every backend/build ships Metal KV-quant kernels, so this + // is an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). + // Set SERVING_KV_CACHE_TYPE=q8_0 (or q4_0) to enable; absent / `f16` → byte-identical + // f16 behavior. NOTE: to have the plan actually GROW the window on the freed memory + // (not just leave it as extra headroom), the fit math must also scale kv_per_token — + // that footprint coupling is the follow-up; this slice is the safe enablement. + if let Some(kv_type) = crate::config_env::read("SERVING_KV_CACHE_TYPE") + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty() && s != "f16") + { + cmd.arg("--cache-type-k") + .arg(&kv_type) + .arg("--cache-type-v") + .arg(&kv_type); + } + // FLASH ATTENTION (#232, opt-in field-proven technique). The fused attention kernel + // is faster on BOTH prefill and decode and lowers peak memory — directly attacking + // the prefill-bound turn latency (#139) and freeing room the elastic window (#234) + // can spend. OFF by default: Metal/backend flash-attn support + quality vary by build + // ([[verify-real-device-numbers-not-a-clamp-premise]]), so it's an operator opt-in, + // never a blind assumption. SERVING_FLASH_ATTN=1|on|true → enable; absent → llama.cpp + // default (no flag), byte-identical. + if crate::config_env::read("SERVING_FLASH_ATTN") + .map(|s| matches!(s.trim().to_ascii_lowercase().as_str(), "1" | "on" | "true" | "yes")) + .unwrap_or(false) + { + cmd.arg("--flash-attn"); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index d314ecb1f9..9e7e3422aa 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1701,7 +1701,16 @@ pub fn start_server( if let Some((interceptor_daemon_socket, _room)) = interceptor_airc_deps { let cell = airc_interceptor_cell.clone(); let root = crate::modules::persona_instance_manager::resolve_continuum_root(); - tokio::spawn(async move { + // MUST be `rt_handle.spawn`, NOT bare `tokio::spawn`: this runs during + // start_server on the IPC thread, and the `rt_handle.enter()` guard (above) is + // scoped and has already dropped by here — so there is NO ambient runtime and + // `tokio::spawn` panics "there is no reactor running". That panic kills the + // attach task, the OnceCell never fills, and EVERYTHING that reads it silently + // no-ops: the AircInterceptor's aircPeer routing AND the grid-overflow effector + // (build_overflow_effector reads this same cell). rt_handle.spawn targets the + // runtime by handle without needing ambient context. (BigMama diagnosed the + // panic 2026-07-27; this is the one-line fix.) + rt_handle.spawn(async move { match airc_lib::Airc::attach_as(root, "continuum-airc-interceptor", interceptor_daemon_socket) .await { @@ -1782,6 +1791,14 @@ pub fn start_server( resource_daemon.clone(), default_room, ))); + // Grid residency beacon (grid-overflow eligibility, slice 3): advertise which models + // this node holds resident on a slower cadence, folded by inbound_attach into + // capacity::model_residency::global_residency_ledger. Reads the SAME serving plan the + // daemon computes (watch snapshot, no parallel probe); orthogonal sibling of capacity. + runtime.register(Arc::new(crate::modules::grid_residency::GridResidencyModule::new( + serving_daemon.subscribe(), + default_room, + ))); let continuum_root = crate::modules::persona_instance_manager::resolve_continuum_root(); let daemon_socket_for_rag_inspect = daemon_socket.clone(); let registry = crate::persona::PersonaAircRuntimeRegistry::new(); @@ -2100,6 +2117,15 @@ pub fn start_server( // below). Subscribe BEFORE the spawn so no plan edge is missed while // the task waits on the executor-ready oneshot. let mut serving_plan_rx = serving_daemon.subscribe(); + // A DEDICATED clone of the interceptor's airc-handle cell for the grid-overflow + // effector (slice 4b): the boot-spawn `async move` below captures by move, and the + // interceptor still needs the original `airc_interceptor_cell` further down — so the + // effector rides its own Arc clone of the SAME shared cell (both see the handle once + // attach_as fills it). + let overflow_airc_cell = airc_interceptor_cell.clone(); + // Same reason for the serving daemon: the reconcile task below still needs the + // original `serving_daemon` handle, so the effector rides its own clone. + let overflow_serving = serving_daemon.clone(); rt_handle.spawn(async move { // Wait for the IPC thread to deliver the WIRED executor (this both // gates ordering AND hands us the executor the personas' hands ride). @@ -2187,7 +2213,22 @@ pub fn start_server( } else { attempt += 1; let summary = supervisor - .spawn_all(&mut provider, Some(tool_executor.clone())) + .spawn_all( + &mut provider, + Some(tool_executor.clone()), + // Grid-overflow effector (slice 4b): the LIVE closure. Reads + // the serving plan's grid_overflow_lanes, filters residency- + // eligible reachable peers (from the beacon ledger), runs + // route_grid_overflow, and re-homes the persona's adapter to an + // AircRemoteInferenceAdapter over the interceptor's airc handle. + // DEFENSIVE: None on any uncertainty → local adapter (no + // regression); can only be a safe no-op or a correct off-box + // route (self excluded via airc.peer_id()). + crate::persona::grid_overflow_effector::build_overflow_effector( + overflow_airc_cell.clone(), + overflow_serving.clone(), + ), + ) .await; if summary.hosted > 0 { tracing::info!( diff --git a/core/continuum-core/src/memory/mod.rs b/core/continuum-core/src/memory/mod.rs index bd0234b968..f39e24d49e 100644 --- a/core/continuum-core/src/memory/mod.rs +++ b/core/continuum-core/src/memory/mod.rs @@ -30,6 +30,8 @@ pub mod corpus; pub mod embedding; pub mod raw_adapter; pub mod recall; +pub mod replication; +pub mod replication_shipper; pub mod timeline; pub mod types; diff --git a/core/continuum-core/src/memory/replication.rs b/core/continuum-core/src/memory/replication.rs new file mode 100644 index 0000000000..1d3dac531e --- /dev/null +++ b/core/continuum-core/src/memory/replication.rs @@ -0,0 +1,422 @@ +//! Persona-RAID — the write-behind journal + replica cold store. +//! +//! Design: docs/architecture/PERSONA-RAID-WRITE-BEHIND.md (M5-approved shape). +//! Slice 1: every DURABLE memory admit (the `persist_memory` funnel — the one +//! place a persona's truth reaches `longterm.db`) is teed into an append-only +//! per-persona `journal.jsonl`. Slice 2a: `memory/replicate-batch` lands a peer +//! shipper's journal tail into this node's replica store and acks high-water. +//! +//! Shape: ONE owned [`ReplicationLedger`] held by `MemoryState` — no statics, +//! no ambient globals (CONCURRENCY-STYLE-GUIDE). Roots are constructor-injected +//! so tests build against temp dirs directly. The slice-2b shipper will be the +//! RTOS-shape consumer (own task + interval + `watch` snapshot) reading this +//! ledger; until its first peer ack, snapshots honestly report +//! `degraded_unreplicated: true`. +//! +//! Contracts: +//! - Tee is BEST-EFFORT-LOUD: a journal failure warns + counts (visible in the +//! snapshot) but NEVER errors the admit — the durable store already holds +//! the truth; a lying "replicated" is worse than an honest "degraded". +//! - `seq` is monotonic per (persona, origin_node); recovered from the last +//! journal line on first touch after a restart, so restarts never fork the +//! sequence ([[restarts-are-commonplace]]). +//! - The replica store is idempotent by (origin_node, seq) high-water — the +//! shipper retries blindly on a lost ack. + +use std::collections::HashMap; +use std::fs::{self, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use serde::{Deserialize, Serialize}; + +/// One journaled durable admit. `record` is the exact JSON that went to the +/// durable store (embedding included) — replay re-issues it verbatim. +/// Crosses the wire in `memory/replicate-batch`, hence the schema derives. +#[derive(Debug, Clone, Serialize, Deserialize, ts_rs::TS, schemars::JsonSchema)] +#[ts(export, export_to = "../../../protocol/typescript/memory/JournalEntry.ts")] +pub struct JournalEntry { + pub seq: u64, + pub persona_id: String, + pub origin_node: String, + pub ts_ms: u64, + /// Admission kind — "memory" today; future durable kinds name themselves. + pub kind: String, + pub record: serde_json::Value, +} + +/// Point-in-time replication state for one persona — the observability +/// surface (slice 2b's shipper publishes this via a `watch` channel). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReplicationSnapshot { + pub persona_id: String, + pub last_seq: u64, + pub last_ts_ms: u64, + pub journal_bytes: u64, + /// Admits that failed to journal since boot (loud degradation counter). + pub dropped: u64, + /// True until a peer has acked a shipped batch (slice 2b). An honest flag: + /// this persona's memory currently lives on ONE machine. + pub degraded_unreplicated: bool, +} + +struct JournalHandle { + path: PathBuf, + seq: u64, + bytes: u64, + dropped: u64, + last_ts_ms: u64, +} + +/// The one owner of persona-RAID state on a node: the outbound journals (this +/// node's personas) and the inbound replica high-waters (other nodes' personas +/// backed up here). Held by `MemoryState`; everything is instance state. +pub struct ReplicationLedger { + origin_node: String, + /// `//journal.jsonl` — sibling of longterm.db. + personas_root: Option, + /// `//journal.jsonl` — peers' memory held here. + replicas_root: Option, + journals: Mutex>, + replica_hw: Mutex>, +} + +impl ReplicationLedger { + /// Construct with explicit roots — the testable constructor. + pub fn new(personas_root: PathBuf, replicas_root: PathBuf, origin_node: String) -> Self { + Self { + origin_node, + personas_root: Some(personas_root), + replicas_root: Some(replicas_root), + journals: Mutex::new(HashMap::new()), + replica_hw: Mutex::new(HashMap::new()), + } + } + + /// Production constructor: roots under the user home, origin from the + /// machine name. A homeless environment yields a ledger that WARNS and + /// counts every admit as dropped (loud degradation, never a panic on the + /// boot path). + pub fn from_env() -> Self { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .ok() + .map(PathBuf::from); + let origin = std::env::var("COMPUTERNAME") + .or_else(|_| std::env::var("HOSTNAME")) + .unwrap_or_else(|_| "unknown-node".to_string()); + match home { + Some(h) => Self::new( + h.join(".continuum/personas"), + h.join(".continuum/replicas"), + origin, + ), + None => Self { + origin_node: origin, + personas_root: None, + replicas_root: None, + journals: Mutex::new(HashMap::new()), + replica_hw: Mutex::new(HashMap::new()), + }, + } + } + + fn journal_path_for(&self, persona_id: &str) -> Option { + let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); + Some(self.personas_root.as_ref()?.join(bare).join("journal.jsonl")) + } + + fn replica_path_for(&self, persona_id: &str) -> Option { + let bare = persona_id.strip_prefix("@persona:").unwrap_or(persona_id); + Some(self.replicas_root.as_ref()?.join(bare).join("journal.jsonl")) + } + + /// Tee one durable admit into the persona's journal. Called from the + /// `persist_memory` funnel AFTER the durable store accepted the write. + /// Never fails the caller — failures warn + count in the snapshot. + pub fn journal_admit(&self, persona_id: &str, kind: &str, record: &serde_json::Value) { + let mut map = match self.journals.lock() { + Ok(g) => g, + Err(_) => return, // poisoned: journaling stands down, admit proceeds + }; + if !map.contains_key(persona_id) { + let Some(path) = self.journal_path_for(persona_id) else { + tracing::warn!( + persona_id, + "persona-RAID: no home dir — journaling disabled, replication degraded" + ); + return; + }; + let (seq, last_ts_ms, bytes) = recover_seq(&path); + map.insert( + persona_id.to_string(), + JournalHandle { path, seq, bytes, dropped: 0, last_ts_ms }, + ); + } + let handle = map.get_mut(persona_id).expect("inserted above"); + let entry = JournalEntry { + seq: handle.seq + 1, + persona_id: persona_id.to_string(), + origin_node: self.origin_node.clone(), + ts_ms: now_ms(), + kind: kind.to_string(), + record: record.clone(), + }; + match append_line(&handle.path, &entry) { + Ok(n) => { + handle.seq = entry.seq; + handle.bytes += n; + handle.last_ts_ms = entry.ts_ms; + } + Err(e) => { + handle.dropped += 1; + tracing::warn!( + persona_id, + dropped = handle.dropped, + "persona-RAID journal append failed (admit unaffected; replication degraded): {e}" + ); + } + } + } + + /// Read this persona's journal entries with `seq > after_seq`, up to + /// `limit`, ascending. The shipper's tail read (slice 2b). Reads from disk + /// (the journal is the durable truth; the in-memory handle only tracks the + /// write head), so it works for any persona with a journal on this node. + pub fn read_tail(&self, persona_id: &str, after_seq: u64, limit: usize) -> Vec { + let Some(path) = self.journal_path_for(persona_id) else { + return Vec::new(); + }; + let Ok(f) = fs::File::open(&path) else { + return Vec::new(); + }; + let mut out = Vec::new(); + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + if e.seq > after_seq { + out.push(e); + if out.len() >= limit { + break; + } + } + } + } + out + } + + /// Persona ids with an active in-memory journal handle (touched since boot). + /// The shipper iterates these each tick. (Cold personas not yet admitted + /// this boot are shipped once they next admit — acceptable: a persona with + /// zero writes this boot has nothing new to replicate.) + pub fn journaled_personas(&self) -> Vec { + self.journals + .lock() + .map(|m| m.keys().cloned().collect()) + .unwrap_or_default() + } + + /// This persona's replication state, or None if nothing journaled since boot. + pub fn snapshot(&self, persona_id: &str) -> Option { + let map = self.journals.lock().ok()?; + let h = map.get(persona_id)?; + Some(ReplicationSnapshot { + persona_id: persona_id.to_string(), + last_seq: h.seq, + last_ts_ms: h.last_ts_ms, + journal_bytes: h.bytes, + dropped: h.dropped, + degraded_unreplicated: true, // honest until slice 2b lands a peer ack + }) + } + + /// Append a shipped batch to this node's replica journal for `persona_id`. + /// All entries must share ONE origin_node (a batch is one shipper's tail). + /// Entries at or below the current high-water are skipped (idempotent + /// retry). Returns the acked high-water for that (persona, origin). + pub fn replica_append_batch( + &self, + persona_id: &str, + entries: &[JournalEntry], + ) -> Result { + let Some(first) = entries.first() else { + // A batch names its origin via its entries; an empty one can't be + // acked meaningfully. The shipper never sends empty — fail loud. + return Err("empty replicate batch".into()); + }; + let origin = first.origin_node.clone(); + if entries.iter().any(|e| e.origin_node != origin) { + return Err("mixed origin_node in one batch — a batch is one shipper's tail".into()); + } + let path = self + .replica_path_for(persona_id) + .ok_or("no home dir for replica store")?; + let mut map = self.replica_hw.lock().map_err(|e| e.to_string())?; + let key = (persona_id.to_string(), origin); + if !map.contains_key(&key) { + recover_replica_hw(persona_id, &path, &mut map); + } + let hw = map.entry(key).or_insert(0); + if let Some(dir) = path.parent() { + fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + let mut f = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map_err(|e| e.to_string())?; + for e in entries { + if e.seq <= *hw { + continue; // already replicated — idempotent retry + } + let line = serde_json::to_string(e).map_err(|e| e.to_string())?; + f.write_all(line.as_bytes()).map_err(|e| e.to_string())?; + f.write_all(b"\n").map_err(|e| e.to_string())?; + *hw = e.seq; + } + Ok(*hw) + } +} + +// ─── file helpers (free of ledger state) ──────────────────────────────────── + +fn append_line(path: &Path, entry: &JournalEntry) -> std::io::Result { + if let Some(dir) = path.parent() { + fs::create_dir_all(dir)?; + } + let mut f = OpenOptions::new().create(true).append(true).open(path)?; + let line = serde_json::to_string(entry) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + f.write_all(line.as_bytes())?; + f.write_all(b"\n")?; + Ok(line.len() as u64 + 1) +} + +/// Recover the last seq from an existing journal so a restart continues the +/// sequence instead of forking it. O(file) once per persona per boot. +fn recover_seq(path: &Path) -> (u64, u64, u64) { + let Ok(f) = fs::File::open(path) else { + return (0, 0, 0); + }; + let bytes = f.metadata().map(|m| m.len()).unwrap_or(0); + let mut last_seq = 0; + let mut last_ts = 0; + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + last_seq = e.seq; + last_ts = e.ts_ms; + } + } + (last_seq, last_ts, bytes) +} + +/// Recover this replica's high-water per origin_node from disk (once per boot). +fn recover_replica_hw(persona_id: &str, path: &Path, map: &mut HashMap<(String, String), u64>) { + let Ok(f) = fs::File::open(path) else { return }; + for line in BufReader::new(f).lines().map_while(Result::ok) { + if let Ok(e) = serde_json::from_str::(&line) { + let key = (persona_id.to_string(), e.origin_node); + let hw = map.entry(key).or_insert(0); + if e.seq > *hw { + *hw = e.seq; + } + } + } +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_ledger(tag: &str) -> (ReplicationLedger, PathBuf) { + let root = std::env::temp_dir().join(format!("raid-{}-{}", tag, std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + ( + ReplicationLedger::new(root.join("personas"), root.join("replicas"), "node-a".into()), + root, + ) + } + + // what this catches: the journal must be monotonic and replayable — an + // entry that round-trips lossily or a seq that repeats would corrupt + // resume-on-spawn's newest-wins replay. + #[test] + fn journal_appends_monotonic_and_round_trips() { + let (ledger, root) = temp_ledger("mono"); + let pid = "persona-a"; + ledger.journal_admit(pid, "memory", &serde_json::json!({"id": "m1", "text": "hello"})); + ledger.journal_admit(pid, "memory", &serde_json::json!({"id": "m2", "text": "world"})); + let snap = ledger.snapshot(pid).expect("snapshot after admits"); + assert_eq!(snap.last_seq, 2); + assert_eq!(snap.dropped, 0); + assert!(snap.degraded_unreplicated); + let path = root.join("personas").join(pid).join("journal.jsonl"); + let lines: Vec = std::io::BufRead::lines(std::io::BufReader::new( + std::fs::File::open(&path).unwrap(), + )) + .map(|l| serde_json::from_str(&l.unwrap()).unwrap()) + .collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].seq, 1); + assert_eq!(lines[1].seq, 2); + assert_eq!(lines[1].record["id"], "m2"); + } + + // what this catches: a restart must CONTINUE the sequence from the file, + // not fork it back to 1 — forked seqs make newest-wins ambiguous and + // silently lose memories on replay ([[restarts-are-commonplace]]). + #[test] + fn restart_recovers_seq_from_disk() { + let (ledger, root) = temp_ledger("restart"); + let pid = "persona-b"; + ledger.journal_admit(pid, "memory", &serde_json::json!({"id": "m1"})); + // Simulate restart: a FRESH ledger over the same roots. + drop(ledger); + let ledger2 = + ReplicationLedger::new(root.join("personas"), root.join("replicas"), "node-a".into()); + ledger2.journal_admit(pid, "memory", &serde_json::json!({"id": "m2"})); + let snap = ledger2.snapshot(pid).expect("snapshot after recovered admit"); + assert_eq!(snap.last_seq, 2, "seq must continue across restart, not fork"); + } + + // what this catches: a re-shipped batch (shipper retry after a lost ack) + // must append NOTHING and re-ack the same high-water — duplicate replica + // lines would double memories on slice-3 replay. + #[test] + fn replica_batch_is_idempotent() { + let (ledger, root) = temp_ledger("replica"); + let pid = "persona-c"; + let e = |seq: u64| JournalEntry { + seq, + persona_id: pid.into(), + origin_node: "node-remote".into(), + ts_ms: 1, + kind: "memory".into(), + record: serde_json::json!({"id": format!("m{seq}")}), + }; + let batch = vec![e(1), e(2)]; + assert_eq!(ledger.replica_append_batch(pid, &batch).unwrap(), 2); + // Retry the same batch: same ack, no new lines. + assert_eq!(ledger.replica_append_batch(pid, &batch).unwrap(), 2); + let path = root.join("replicas").join(pid).join("journal.jsonl"); + let n = std::io::BufRead::lines(std::io::BufReader::new( + std::fs::File::open(&path).unwrap(), + )) + .count(); + assert_eq!(n, 2, "retry must not duplicate replica lines"); + // Mixed-origin batch fails loud. + let mut bad = vec![e(3)]; + bad.push(JournalEntry { origin_node: "node-other".into(), ..e(4) }); + assert!(ledger.replica_append_batch(pid, &bad).is_err()); + // Empty batch fails loud. + assert!(ledger.replica_append_batch(pid, &[]).is_err()); + } +} diff --git a/core/continuum-core/src/memory/replication_shipper.rs b/core/continuum-core/src/memory/replication_shipper.rs new file mode 100644 index 0000000000..138cd2d792 --- /dev/null +++ b/core/continuum-core/src/memory/replication_shipper.rs @@ -0,0 +1,235 @@ +//! Persona-RAID slice 2b — the write-behind SHIPPER. +//! +//! The RTOS-shape consumer of the [`ReplicationLedger`]: an owned task on a +//! fixed `interval` (the cadence IS the maximum amnesia window) that, per +//! journaled persona, reads the journal tail past the peer's acked high-water, +//! ships it to a residency-eligible peer's `memory/replicate-batch`, and +//! advances the high-water on ack. State is published via a `watch` snapshot. +//! Design: docs/architecture/PERSONA-RAID-WRITE-BEHIND.md. +//! +//! Concurrency (CONCURRENCY-STYLE-GUIDE): own task + `tokio::time::interval` + +//! `watch::Sender` + injected deps (ledger, transport, peer +//! source). Shipping is best-effort write-behind — a slow/absent peer grows the +//! reported lag, NEVER blocks the cognition hot path (the tee is elsewhere). + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::json; +use tokio::sync::watch; + +use crate::memory::replication::ReplicationLedger; +use crate::routing::command_uri::PeerRef; +use crate::routing::route_decision::RouteDecision; +use crate::routing::transport::Transport; +use crate::runtime::service_module::CommandResult; + +/// How many journal entries ship per persona per tick — bounds one batch so a +/// long-cold persona catches up over several ticks instead of one huge frame. +const SHIP_BATCH_LIMIT: usize = 256; + +/// Default cadence — the maximum amnesia window. One knob (design doc). +pub const DEFAULT_SHIP_INTERVAL: Duration = Duration::from_secs(30); + +/// Chooses the peer to replicate a persona TO. Injected so the runtime can back +/// it with the live residency/grid view while tests use a fixed peer. +pub trait ReplicaPeerSource: Send + Sync { + /// A residency-eligible peer to hold `persona_id`'s replica, or None when + /// the grid has no eligible peer (then the persona stays single-copy — + /// honestly `degraded_unreplicated`). + fn replica_peer_for(&self, persona_id: &str) -> Option; +} + +/// Published shipper state — per persona, the last seq a peer has acked. +#[derive(Debug, Clone, Default)] +pub struct ShipperSnapshot { + /// persona_id → (acked_seq, peer_label). Absent = never shipped. + pub acked: HashMap, + /// Ticks completed since boot (liveness). + pub ticks: u64, +} + +/// The shipper engine — owns the per-persona acked high-water, ships one tick +/// on demand. Split from the task loop so it is unit-testable with a fake +/// transport + peer source (no timers, no runtime). +pub struct ReplicationShipper { + ledger: Arc, + transport: Arc, + peers: Arc, + acked: HashMap, + snapshot_tx: watch::Sender, + ticks: u64, +} + +impl ReplicationShipper { + pub fn new( + ledger: Arc, + transport: Arc, + peers: Arc, + ) -> (Self, watch::Receiver) { + let (snapshot_tx, snapshot_rx) = watch::channel(ShipperSnapshot::default()); + ( + Self { + ledger, + transport, + peers, + acked: HashMap::new(), + snapshot_tx, + ticks: 0, + }, + snapshot_rx, + ) + } + + /// One replication pass over all journaled personas. Best-effort: a persona + /// whose peer is absent or whose ship fails is simply skipped this tick + /// (its lag grows, reported in the snapshot next publish). Returns the + /// number of personas whose high-water advanced. + pub async fn tick(&mut self) -> usize { + let mut advanced = 0; + for persona_id in self.ledger.journaled_personas() { + let after = self.acked.get(&persona_id).map(|(s, _)| *s).unwrap_or(0); + let batch = self.ledger.read_tail(&persona_id, after, SHIP_BATCH_LIMIT); + if batch.is_empty() { + continue; + } + let Some(peer) = self.peers.replica_peer_for(&persona_id) else { + continue; // no eligible peer — stays single-copy (honest degrade) + }; + let peer_label = peer.to_string(); + let params = json!({ "persona_id": persona_id, "entries": batch }); + let decision = RouteDecision::Peer { + peer, + node: None, + env: None, + path: "memory/replicate-batch".to_string(), + query: None, + fragment: None, + }; + match self.transport.dispatch(decision, params).await { + Ok(result) => { + if let Some(acked_seq) = extract_acked_seq(&result) { + self.acked.insert(persona_id.clone(), (acked_seq, peer_label)); + advanced += 1; + } + } + Err(e) => { + tracing::warn!(persona_id, "persona-RAID ship failed (lag grows): {e}"); + } + } + } + self.ticks += 1; + let _ = self.snapshot_tx.send(ShipperSnapshot { + acked: self.acked.clone(), + ticks: self.ticks, + }); + advanced + } + + /// Spawn the RTOS task: tick every `interval` until the process ends. + /// Consumes self (moves into the task). The runtime holds the returned + /// snapshot receiver for `serving/status`-style inspection. + pub fn spawn(mut self, interval: Duration) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + ticker.tick().await; + self.tick().await; + } + }) + } +} + +/// Pull `acked_seq` out of the receiver's CommandResult (the +/// `MemoryReplicateBatchResult` JSON). Absent/garbled → None (no advance). +fn extract_acked_seq(result: &CommandResult) -> Option { + result + .to_json_value() + .ok() + .and_then(|v| v.get("acked_seq").and_then(|s| s.as_u64())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::routing::route_decision::RouteDecision; + use async_trait::async_trait; + use std::sync::Mutex; + + /// Fake transport: records the persona_id + entry count it was asked to + /// ship, and acks the highest seq in the batch (like the real receiver). + #[derive(Debug, Default)] + struct FakeTransport { + shipped: Mutex>, + } + + #[async_trait] + impl Transport for FakeTransport { + async fn dispatch( + &self, + decision: RouteDecision, + params: serde_json::Value, + ) -> Result { + let path = match &decision { + RouteDecision::Peer { path, .. } => path.clone(), + _ => return Err("unexpected decision".into()), + }; + assert_eq!(path, "memory/replicate-batch"); + let pid = params["persona_id"].as_str().unwrap().to_string(); + let entries = params["entries"].as_array().unwrap(); + let max_seq = entries.iter().map(|e| e["seq"].as_u64().unwrap()).max().unwrap(); + self.shipped.lock().unwrap().push((pid, entries.len())); + Ok(CommandResult::Json(json!({ "acked_seq": max_seq }))) + } + } + + struct FixedPeer; + impl ReplicaPeerSource for FixedPeer { + fn replica_peer_for(&self, _persona_id: &str) -> Option { + Some(PeerRef::Name("peer-mac".to_string())) + } + } + + fn temp_ledger() -> Arc { + let root = std::env::temp_dir().join(format!("raid-ship-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + Arc::new(ReplicationLedger::new( + root.join("personas"), + root.join("replicas"), + "node-a".into(), + )) + } + + // what this catches: the shipper must ship only the UNACKED tail and advance + // its high-water on ack — re-shipping acked entries (no advance, or shipping + // from seq 0 every tick) would flood the peer and never converge. + #[tokio::test] + async fn ships_tail_once_and_advances_highwater() { + let ledger = temp_ledger(); + let pid = "persona-x"; + ledger.journal_admit(pid, "memory", &json!({"id": "m1"})); + ledger.journal_admit(pid, "memory", &json!({"id": "m2"})); + let transport = Arc::new(FakeTransport::default()); + let (mut shipper, mut rx) = + ReplicationShipper::new(ledger.clone(), transport.clone(), Arc::new(FixedPeer)); + + // Tick 1: ships both entries, acks seq 2. + assert_eq!(shipper.tick().await, 1); + rx.changed().await.unwrap(); + assert_eq!(rx.borrow().acked.get(pid).map(|(s, _)| *s), Some(2)); + + // Tick 2 with no new admits: nothing to ship (tail empty past seq 2). + assert_eq!(shipper.tick().await, 0); + + // A new admit: tick 3 ships ONLY the new entry. + ledger.journal_admit(pid, "memory", &json!({"id": "m3"})); + assert_eq!(shipper.tick().await, 1); + let shipped = transport.shipped.lock().unwrap(); + assert_eq!(shipped.len(), 2, "two ships total (m1+m2, then m3)"); + assert_eq!(shipped[0], (pid.to_string(), 2), "first ship: 2 entries"); + assert_eq!(shipped[1], (pid.to_string(), 1), "second ship: 1 new entry only"); + } +} diff --git a/core/continuum-core/src/model_registry/catalog.rs b/core/continuum-core/src/model_registry/catalog.rs index a50fb8522f..cc7ec9ed89 100644 --- a/core/continuum-core/src/model_registry/catalog.rs +++ b/core/continuum-core/src/model_registry/catalog.rs @@ -821,6 +821,32 @@ pub fn models() -> Vec { // gguf_local_path DERIVED from the id under genome/models (see coder-14b above). ..ModelSpec::default() }), + // QWEN3-CODER-30B-A3B compacted to 19B (256k ctx) — BigMama's forged coder, + // the first CHAT candidate the 5090 serving node can host locally. Qwen3 MoE + // (a3b active) pruned+quantized via the plasticity-compaction pipeline; Q4_K_M + // GGUF ~11 GB, fits the 5090 with room for KV. Registered under "llama-server" + // so the serving daemon's static-CUDA engine hosts it on a lane; gguf resolves + // from its dir under genome/models (id-token subset match). + model(ModelSpec { + id: "continuum-ai/qwen3-coder-30b-a3b-compacted-19b-256k", + name: "Qwen3-Coder-30B-A3B compacted 19B (256k)", + provider: "llama-server", + arch: Arch::Qwen3, + context_window: 262_144, + max_output_tokens: 8192, + tokens_per_second: 30.0, + capabilities: &[ + Capability::TextGeneration, + Capability::Chat, + Capability::ToolUse, + Capability::Streaming, + ], + gguf_hint: Some("huggingface.co/continuum-ai/qwen3-coder-30b-a3b-compacted-19b-256k"), + chat_template: Some(QWEN35_CHAT_TEMPLATE), + multi_party_strategy: MultiPartyChatStrategy::ProperChatMlSingleParty, + stop_sequences: &["<|im_end|>", "<|endoftext|>"], + ..ModelSpec::default() + }), ] } diff --git a/core/continuum-core/src/modules/chat/mod.rs b/core/continuum-core/src/modules/chat/mod.rs index 09998d01df..9758d62d38 100644 --- a/core/continuum-core/src/modules/chat/mod.rs +++ b/core/continuum-core/src/modules/chat/mod.rs @@ -121,18 +121,23 @@ impl ChatModule { Self { executor_slot } } - /// Resolve the executor for the current call. Panics if the - /// executor was never installed — that's a boot ordering bug - /// (`start_server` must call `install_executor_on_all` BEFORE - /// any chat command can dispatch). Per [[no-fallbacks-ever]]: - /// the panic message names the contract so the operator sees - /// the actual problem. - fn executor(&self) -> Arc { - self.executor_slot.cloned().expect( + /// Resolve the executor for the current call. Returns a loud, + /// contract-naming error if the executor was never installed — a + /// boot-ordering bug (`start_server` must call + /// `install_executor_on_all` BEFORE any chat command dispatches). + /// Per [[no-fallbacks-ever]] the message still names the contract + /// so the operator sees the real problem — but the blast radius is + /// THIS request, not the whole core: a command that races boot + /// fails loudly to its caller instead of `.expect()`-panicking the + /// process and taking every other module down with it + /// (#201 boot-race / #26 faculties degrade, never panic). + fn executor(&self) -> Result, String> { + self.executor_slot.cloned().ok_or_else(|| { "ChatModule: CommandExecutor not installed — \ start_server must call install_executor_on_all \ - before any chat command can dispatch (task #224)", - ) + before any chat command can dispatch (task #224)" + .to_string() + }) } /// `chat/poll` — return recent messages, optionally filtered by @@ -151,7 +156,7 @@ impl ChatModule { /// 5. Normalize back to chronological order for display regardless /// of query direction. pub async fn poll(&self, params: ChatPollParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let limit = params.limit.unwrap_or(DEFAULT_POLL_LIMIT); // ── Phase 1: resolve the anchor timestamp if the caller @@ -284,7 +289,7 @@ impl ChatModule { /// could be the dedup id) but the design conversation is its /// own scope. pub async fn send(&self, params: ChatSendParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let message_id = Uuid::new_v4(); let now_ms = now_ms(); let now_iso = now_iso(now_ms); @@ -416,7 +421,7 @@ impl ChatModule { /// EXPECTED duplicate, never an error. A persona line's id is airc's /// `event_id` (stable across replay), so restarts can't double-store either. pub async fn persist_posted(&self, payload: Value) -> Result<(), String> { - let executor = self.executor(); + let executor = self.executor()?; let field = |k: &str| -> Result { payload .get(k) @@ -995,6 +1000,24 @@ mod tests { assert!(result.after_message_id.is_none()); } + // what this catches: #201 boot-race — a chat command that arrives BEFORE + // install_executor_on_all runs fails LOUD to its caller (naming the contract), + // instead of .expect()-panicking the whole core and taking every other module + // down with it. Regression for the .expect() → Result conversion in executor(). + #[tokio::test] + async fn command_before_executor_installed_fails_loud_not_panics() { + let chat = ChatModule::new(); // executor slot deliberately NOT installed + let err = chat + .poll(ChatPollParams::default()) + .await + .expect_err("poll before install_executor_on_all must fail, not panic"); + assert!(err.contains("not installed"), "loud contract error, got: {err}"); + assert!( + err.contains("install_executor_on_all"), + "error names the contract so the operator sees the real problem: {err}" + ); + } + // ── chat/poll: latest-N path (no anchor) ────────────────────────── #[tokio::test] diff --git a/core/continuum-core/src/modules/grid_residency.rs b/core/continuum-core/src/modules/grid_residency.rs new file mode 100644 index 0000000000..3f5fb81f0d --- /dev/null +++ b/core/continuum-core/src/modules/grid_residency.rs @@ -0,0 +1,166 @@ +//! GridResidencyModule — this node's residency-beacon gossip publisher (grid-overflow slice 3). +//! +//! The residency sibling of [`super::grid_capacity::GridCapacityModule`]. Every +//! [`RESIDENCY_PUBLISH_INTERVAL_MS`] tick it reads the SAME live serving plan the daemon +//! already computes (one source, no parallel probe) and broadcasts a [`ResidencyBeacon`] over +//! airc as an `EphemeralCoalesced` `grid_residency` realtime envelope: which models this node +//! holds resident — the grid-overflow ELIGIBILITY signal. The receive half lives in +//! [`crate::airc::inbound_attach`], which folds heard beacons (our own echo included — the +//! loopback proof) into [`crate::capacity::model_residency::global_residency_ledger`], whose +//! `view()` IS the `ModelResidencyView` the governor composes with the capacity snapshot. +//! +//! ## Why a SEPARATE module + slower cadence +//! +//! Residency and capacity are orthogonal (settled with BigMama 2026-07-27): capacity is +//! free-VRAM RIGHT NOW (10s beat, [`super::grid_capacity`]); residency is which models are warm +//! (minute-scale, changes only on page-in/out). Coupling them onto one envelope would either +//! over-publish residency or under-refresh capacity. Same module shape as the style guide +//! mandates — no new tokio task, the runtime tick drives it; the plan read is a lock-free +//! `watch` snapshot; the publish is one small envelope through the existing +//! `airc/realtime-publish` command surface (the same path capacity + chat ride). +//! +//! ## Today's local residency = the base model +//! +//! `ServingPlan` carries a single `base_model_id` + a `resident_models` COUNT (not a per-id +//! set), so today this node's honest resident set is `[base_model_id]`. When multi-model +//! residency lands (a per-id resident list on the plan), only [`Self::current_beacon`] changes +//! — the wire, ledger, and governor compose are already model-set-shaped. + +use std::any::Any; +use std::sync::Mutex; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::{json, Value}; +use tokio::sync::watch; + +use crate::capacity::model_residency::{ + global_residency_ledger, ResidencyBeacon, RESIDENCY_PUBLISH_INTERVAL_MS, +}; +use crate::cognition::serving_plan::ServingPlan; +use crate::runtime::{ + CommandExecutor, CommandResult, LateBound, ModuleConfig, ModulePriority, ServiceModule, +}; +use airc_core::RoomId; +use std::sync::Arc; + +pub struct GridResidencyModule { + /// Lock-free live snapshot of the daemon's serving plan — the SAME source the prefill + /// valve and serving control derive from (no parallel probe). + plan_rx: watch::Receiver>, + /// The node's discovered default room — where the grid rendezvous happens today. + room: RoomId, + executor_slot: Arc>, + /// Last published resident set — the glass box speaks on CHANGE, not on every beat (a + /// steady residency is silence). `None` = never published, so the first real plan speaks. + last_published: Mutex>>, +} + +impl GridResidencyModule { + pub fn new(plan_rx: watch::Receiver>, room: RoomId) -> Self { + Self { + plan_rx, + room, + executor_slot: Arc::new(LateBound::new("grid-residency::executor")), + last_published: Mutex::new(None), + } + } + + /// Build this node's residency beacon from the live serving plan. `None` when no plan is + /// computed yet (nothing resident to advertise) — honest silence, never a fabricated set. + /// Today the resident set is `[base_model_id]`; a multi-model plan extends only this line. + fn current_beacon(&self) -> Option { + let plan = self.plan_rx.borrow().clone()?; + Some(ResidencyBeacon { + resident_models: vec![plan.base_model_id], + at_ms: now_ms(), + }) + } +} + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[async_trait] +impl ServiceModule for GridResidencyModule { + fn config(&self) -> ModuleConfig { + ModuleConfig { + name: "grid-residency", + priority: ModulePriority::Background, + command_prefixes: &[], + event_subscriptions: &[], + needs_dedicated_thread: false, + max_concurrency: 0, + tick_interval: Some(Duration::from_millis(RESIDENCY_PUBLISH_INTERVAL_MS)), + } + } + + async fn initialize(&self, _ctx: &crate::runtime::ModuleContext) -> Result<(), String> { + Ok(()) + } + + async fn handle_command(&self, command: &str, _params: Value) -> Result { + Err(format!( + "grid-residency has no command surface — '{command}' (beacons publish on the module \ + tick; read the grid via capacity::model_residency::global_residency_ledger)" + )) + } + + async fn tick(&self) -> Result<(), String> { + // Boot ordering: a beat or two may fire before start_server installs the executor — + // transient, skip (the next beat publishes). Same guaranteed-installed path as capacity. + let Some(executor) = self.executor_slot.cloned() else { + return Ok(()); + }; + let Some(beacon) = self.current_beacon() else { + // No serving plan yet — nothing resident to advertise. Honest silence. + return Ok(()); + }; + + let envelope = json!({ + "eventId": uuid::Uuid::new_v4().to_string(), + "roomId": self.room.as_uuid().to_string(), + "sourceId": "grid-residency", + "createdAtMs": beacon.at_ms, + "delivery": "ephemeral_coalesced", + "payload": { + "kind": "existing_schema", + "payload": { + "schema": "grid_residency", + "inline": serde_json::to_value(&beacon) + .map_err(|e| format!("residency beacon encode failed: {e}"))?, + } + }, + }); + executor + .execute_json("airc/realtime-publish", json!({ "envelope": envelope })) + .await + .map_err(|e| format!("grid-residency beacon publish failed: {e}"))?; + + // Glass box: speak on change (resident set differs), silent on a steady residency. + let mut last = self.last_published.lock().map_err(|e| format!("residency lock: {e}"))?; + if last.as_deref() != Some(beacon.resident_models.as_slice()) { + crate::probe!( + class = "grid.residency.beacon", + models = ?beacon.resident_models, + heard_peers = global_residency_ledger().heard_count(), + "residency beacon published to the grid", + ); + *last = Some(beacon.resident_models); + } + Ok(()) + } + + fn install_executor(&self, executor: Arc) { + self.executor_slot.install(executor); + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/core/continuum-core/src/modules/memory.rs b/core/continuum-core/src/modules/memory.rs index f29dbc116c..73eb8ed110 100644 --- a/core/continuum-core/src/modules/memory.rs +++ b/core/continuum-core/src/modules/memory.rs @@ -33,6 +33,9 @@ pub struct MemoryState { /// `data/create`) and to hydrate a missing corpus from it (via `data/list`) /// — the cross-module dual-write pattern `ChatModule` established. pub executor_slot: Arc>, + /// Persona-RAID ledger — the ONE owner of this node's write-behind + /// journals + replica high-waters (no statics; see memory/replication.rs). + pub replication: Arc, } impl MemoryState { @@ -40,6 +43,7 @@ impl MemoryState { Self { memory_manager, executor_slot: Arc::new(LateBound::new("memory::executor")), + replication: Arc::new(crate::memory::replication::ReplicationLedger::from_env()), } } diff --git a/core/continuum-core/src/modules/mod.rs b/core/continuum-core/src/modules/mod.rs index f146dbe500..9323d8f772 100644 --- a/core/continuum-core/src/modules/mod.rs +++ b/core/continuum-core/src/modules/mod.rs @@ -46,6 +46,7 @@ pub mod gpu; pub mod grant_issuance; pub mod grid; pub mod grid_capacity; +pub mod grid_residency; pub mod health; pub mod hippocampus; pub mod inference_coordinator_module; diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index b32dd25112..90e8902ed2 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -23,8 +23,30 @@ use crate::cognition::model_resolver::types::HwCapabilityTier; use crate::cognition::serving_plan::{ - plan_serving, plan_serving_stable, HostBudget, ModelFootprint, ServingPlan, MIN_SERVE_CTX, + plan_serving, plan_serving_stable, plan_serving_with_demand, HostBudget, ModelFootprint, + ServingPlan, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, }; +use crate::cognition::working_set_demand::WorkingSetDemand; + +/// Process-wide sink for per-turn working-set observations — the serving daemon +/// installs its own `WorkingSetDemand` here at [`ServingDaemonModule::initialize`], +/// so the cognition turn path feeds each turn's assembled-prompt size WITHOUT a +/// daemon handle (the same process-wide-readable-seam shape as `install_serving_state`). +/// Unset until a daemon boots → [`observe_serving_working_set`] is a no-op and the +/// served window simply holds its cold prior. (#234) +static SERVING_WORKING_SET: OnceLock>> = OnceLock::new(); + +/// Feed one completed turn's assembled-prompt token count into the live serving +/// demand. Called from the persona turn path; the next plan tick sizes the served +/// window to the p95 of recent turns. No-op before a serving daemon has booted, so +/// the caller never needs to know whether serving is up. (#234) +pub fn observe_serving_working_set(prompt_tokens: u32) { + if let Some(ws) = SERVING_WORKING_SET.get() { + if let Ok(mut w) = ws.lock() { + w.observe(prompt_tokens); + } + } +} use crate::gpu::GpuMemoryManager; use crate::inference::llama_server::{ ensure_model_serving, serving_v1_url, AdapterEntry, EnsureOutcome, LlamaServerControl, @@ -239,6 +261,12 @@ pub struct ServingDaemonModule { /// that fit at pin time; budget can still shift under it, and then the plan /// degrades honestly (`fits_on_gpu = false`) rather than over-committing. pinned: watch::Sender>, + /// Live per-host working-set demand — the p95 of recent turns' assembled-prompt + /// sizes, threaded into the serving plan as the elastic `demand_ceil` so the + /// served window ebbs and flows with real demand instead of the baked prior + /// (#234). A Mutex (not atomic) for the rolling window; read only on the plan + /// tick, never a hot path. `observe_working_set` feeds it from the cognition turn. + working_set: Arc>, } impl ServingDaemonModule { @@ -273,6 +301,10 @@ impl ServingDaemonModule { let (serving_tx, _srx) = watch::channel(ServingSnapshot::empty()); let (suppressed, _urx) = watch::channel(Arc::new(HashSet::new())); let (pinned, _prx) = watch::channel(None); + // Rolling window (recent turns) over which the served window's demand ceiling + // tracks the p95 assembled-prompt size. Long enough to hold a coding session's + // shape, short enough that demand ebbs back within a session (#234). + const WORKING_SET_WINDOW: usize = 64; Self { gpu, system, @@ -296,6 +328,11 @@ impl ServingDaemonModule { suppressed, pinned, lane_demand: Arc::new(std::sync::atomic::AtomicU32::new(1)), + working_set: Arc::new(std::sync::Mutex::new(WorkingSetDemand::new( + WORKING_SET_WINDOW, + BOOTSTRAP_WORKING_SET, + MIN_SERVE_CTX, // one generation of headroom above the assembled prompt + ))), moe_serving: std::sync::Mutex::new(None), // Read ONCE here (single config entry point, no per-tick I/O). Off by default. measure_force_expert_budget_bytes: crate::config_env::read( @@ -313,6 +350,27 @@ impl ServingDaemonModule { .store(demand.max(1), Ordering::Relaxed); } + /// Feed one completed turn's assembled-prompt token count into the live + /// working-set demand (#234). Called from the cognition turn path; the next + /// plan tick sizes the served window to the p95 of recent turns. Cheap — a + /// brief lock on a bounded ring, off the serving hot path. + pub fn observe_working_set(&self, prompt_tokens: u32) { + if let Ok(mut w) = self.working_set.lock() { + w.observe(prompt_tokens); + } + } + + /// The live demand ceiling for the served window — the p95 baseline of recent + /// turns (floored at the cold prior). Read on the plan tick and threaded into + /// `plan_serving_with_demand` so the window grows for heavy work and ebbs back + /// when turns get lean. A poisoned lock degrades to the cold prior, never panics. + fn demand_ceil(&self) -> u32 { + self.working_set + .lock() + .map(|w| w.demand_ceil()) + .unwrap_or(BOOTSTRAP_WORKING_SET) + } + /// The current lane demand (≥ 1). /// Register serving's autonomic PLANNER to run on the memory authority's tick /// (MEMORY-AUTHORITY-DAEMON slice 1b). The lane plan — which model, how many lanes, @@ -502,7 +560,7 @@ impl ServingDaemonModule { /// one candidate, so the reconcile serves that model or (if it has dropped off /// disk) nothing. Suppress subtracts; pin intersects; the planner still owns /// the choice among whatever remains. - fn live_candidates(&self) -> Vec { + pub(crate) fn live_candidates(&self) -> Vec { let suppressed = self.suppressed.borrow(); let pinned = self.pinned.borrow(); servable_candidates(&self.catalog.snapshot(), &**suppressed, &pinned) @@ -559,10 +617,11 @@ impl ServingDaemonModule { /// to drive the spawner before the tick loop starts — single source of /// truth for "what model + how many lanes." pub fn compute_plan(&self) -> Option { - plan_serving( + plan_serving_with_demand( self.host_budget(), &self.live_candidates(), self.lane_demand(), + self.demand_ceil(), ) } @@ -1004,7 +1063,7 @@ impl ServingDaemonModule { .borrow() .as_ref() .map(|p| p.base_model_id.clone()); - match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand()) { + match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand(), self.demand_ceil()) { Some(plan) => { crate::probe!( class = "serving.plan", @@ -1290,12 +1349,38 @@ fn serving_footprint_fn(catalog: Arc) -> FootprintFn { pub fn footprint_for(model: &Model) -> Option { let path = crate::model_registry::artifacts::resolve_gguf_for_model(model)?; let weights_bytes = std::fs::metadata(&path).ok()?.len(); - footprint_from_parts( + let mut fp = footprint_from_parts( &model.id, weights_bytes, model.context_window, model.has(Capability::ToolUse), - ) + )?; + // KV CACHE QUANTIZATION (#232): a lane running quantized KV holds proportionally + // fewer bytes/token, so the plan can size a BIGGER window into the same budget — + // this is what turns the launcher's opt-in q8_0 flag into an actual window GROWTH. + // Divide the f16 rate by the quant factor; default (f16 / unset) → 1 → byte-identical. + // Keep the config key in sync with the launcher arg in inference/llama_server.rs — + // one SERVING_KV_CACHE_TYPE key, two consumers (launcher flag + this fit-math rate). + fp.kv_per_token = (fp.kv_per_token / kv_cache_quant_divisor()).max(1); + Some(fp) +} + +/// The resident-KV divisor implied by `SERVING_KV_CACHE_TYPE`, so the plan sizes the +/// served window against the KV the lane WILL actually hold, not the f16 default. (#232) +fn kv_cache_quant_divisor() -> u64 { + kv_divisor_for(crate::config_env::read("SERVING_KV_CACHE_TYPE").as_deref()) +} + +/// Pure KV-rate divisor for a cache-type string (testable without env). CONSERVATIVE by +/// design: q8_0 ≈ half of f16 → 2; q4_0/q4_1 ≈ a third → 3 (under the ideal ~3.5×, so the +/// plan never over-grows the window past the real KV and OOMs). Anything else / f16 → 1 +/// (no change). Over-reserve is a smaller window (safe); under-reserve is an OOM (fatal). +fn kv_divisor_for(cache_type: Option<&str>) -> u64 { + match cache_type.map(|s| s.trim().to_ascii_lowercase()).as_deref() { + Some("q8_0") => 2, + Some("q4_0") | Some("q4_1") => 3, + _ => 1, + } } /// Pure footprint estimate from the fields that drive it — split out from the @@ -1518,6 +1603,10 @@ impl ServiceModule for ServingDaemonModule { // free functions + adapters read "what's live" as a pointer instead of // each probing /v1/models. Set-once (singleton daemon). let _ = crate::inference::llama_server::install_serving_state(self.subscribe_serving()); + // Install this daemon's working-set demand as the process-wide sink so the + // cognition turn path feeds per-turn prompt sizes without a daemon handle, and + // the served window ebbs and flows with real demand (#234). + let _ = SERVING_WORKING_SET.set(self.working_set.clone()); // Register serving as a MEASURED ResourceConsumer with the one per-machine // authority (#79). See `register_as_consumer` — this is monitor-not-reserve: // no lease acquired, `available` math untouched, the authority simply stops @@ -1735,6 +1824,19 @@ mod tests { // what this catches: footprint estimate is honest about weights (passed // through), tool capability bumps the rank, KV is non-zero, and zero // weights → no footprint (we only offer what we can actually serve). + #[test] + fn kv_divisor_reflects_cache_type_conservatively() { + // what this catches: the #232 KV-quant fit-math coupling — the served window grows + // only when the lane actually runs quantized KV, and CONSERVATIVELY so the plan + // never over-grows past the real KV and OOMs. f16/unset/unknown must never scale. + assert_eq!(kv_divisor_for(None), 1, "unset never scales the window"); + assert_eq!(kv_divisor_for(Some("f16")), 1, "explicit f16 is the no-op default"); + assert_eq!(kv_divisor_for(Some("q8_0")), 2, "q8_0 ~ half of f16"); + assert_eq!(kv_divisor_for(Some(" Q8_0 ")), 2, "trimmed + case-insensitive"); + assert_eq!(kv_divisor_for(Some("q4_0")), 3, "q4_0 conservative, under the ideal ~3.5x"); + assert_eq!(kv_divisor_for(Some("garbage")), 1, "unknown type → no grow, never a bogus OOM"); + } + #[test] fn footprint_from_parts_is_footprint_aware() { let fp = footprint_from_parts("present", 3 * GB, 8192, true).unwrap(); diff --git a/core/continuum-core/src/persona/grid_overflow_effector.rs b/core/continuum-core/src/persona/grid_overflow_effector.rs new file mode 100644 index 0000000000..4acc4a7286 --- /dev/null +++ b/core/continuum-core/src/persona/grid_overflow_effector.rs @@ -0,0 +1,115 @@ +//! The grid-overflow EFFECTOR (governor consumer slice 4b-ii) — composes the tested +//! decision path into the live per-persona adapter override that +//! [`super::supervisor::materialize_adapters`] consumes. +//! +//! When the node is over local capacity (`ServingPlan.grid_overflow_lanes > 0`) and a +//! reachable peer already holds a persona's model, this routes HER BRAIN to that peer: her +//! `DeliberationModelBinding.adapter` becomes an [`AircRemoteInferenceAdapter`] over airc, so +//! her inference crosses the grid transparently — the exact re-home the binding was designed +//! for. The persona doesn't know or care that her model runs on another machine. +//! +//! ## Defensive by construction — safe to ship before the live smoke +//! +//! The closure returns `None` (→ the local factory adapter, zero behavior change) on ANY +//! uncertainty: airc not yet attached, no serving plan, no overflow, no matching footprint, or +//! no residency-eligible reachable peer. So it can only ever be a **safe no-op or a correct +//! remote route** — never a self-route (this node's own peer is excluded via `airc.peer_id()`, +//! so its own residency-beacon loopback can't select itself) and never a panic. The one thing +//! the unit path can't prove is that the remote hop SUCCEEDS — that is what the live two-node +//! smoke validates; a hop that can't warm surfaces as a loud per-slot `AdapterWarmup` failure +//! ([[fallbacks-are-illegal-fail-loud]]), never a silent local downgrade. + +use std::sync::Arc; + +use airc_lib::Airc; +use tokio::sync::OnceCell; + +use crate::ai::adapter::AIProviderAdapter; +use crate::capacity::gossip::global_ledger; +use crate::capacity::grid_overflow::route_grid_overflow; +use crate::capacity::model_residency::global_residency_ledger; +use crate::capacity::DeviceCapacity; +use crate::inference::airc_remote::adapter::AircRemoteInferenceAdapter; +use crate::inference::airc_remote::transport::AircLiveTransport; +use crate::modules::serving_daemon::ServingDaemonModule; +use crate::persona::inference_profile::PersonaInferenceProfile; + +/// Headroom kept free on a peer before it may accept an overflow lane — same 1 GiB spirit as +/// the single-device [`crate::capacity::grid::LocalFirstFitPolicy`] safety margin. +const OVERFLOW_SAFETY_MARGIN_BYTES: u64 = 1024 * 1024 * 1024; + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Build the effector closure `spawn_all` / `materialize_adapters` consume. Captures the +/// late-bound airc handle cell (shared with the interceptor) and the serving daemon (the live +/// plan + footprint source). See the module docs for the defensive contract. +pub fn build_overflow_effector( + airc_cell: Arc>>, + serving: Arc, +) -> impl Fn(&PersonaInferenceProfile, usize) -> Option> { + move |profile, _slot| { + // airc not attached yet → local (the boot window before attach_as fills the cell). + let airc = airc_cell.get()?.clone(); + // No plan, or demand fits locally → local. grid_overflow_lanes is the honest + // "over local capacity by N" signal; 0 means nothing to spill. + let plan = serving.compute_plan()?; + if plan.grid_overflow_lanes == 0 { + return None; + } + // The footprint for THIS persona's model — the lease's per-lane prefill spike. + // Absent (model not a live candidate) → local, never a fabricated footprint. + let footprint = serving + .live_candidates() + .into_iter() + .find(|f| f.model_id == profile.model_id)?; + let lease = + footprint.grid_lease_request(plan.served_context_window, plan.grid_overflow_lanes); + + // Own peer id from the airc handle → excludes self from the residency view + gossip + // snapshot (this node's own beacon loopback must never select itself as the target). + let own = airc.peer_id().as_uuid(); + let now = now_ms(); + let residency = global_residency_ledger().view(own, now); + // Overflow placement is REMOTE-ONLY, so local capacity is never read — a zeroed local + // is the honest input (route_grid_overflow only ever inspects the peer list). + let snapshot = global_ledger().snapshot( + own, + DeviceCapacity { + gpu_total_bytes: 0, + gpu_free_bytes_live: 0, + system_ram_free_bytes: 0, + }, + now, + ); + + let routing = route_grid_overflow( + &profile.model_id, + &lease, + &residency, + &snapshot, + OVERFLOW_SAFETY_MARGIN_BYTES, + ); + // First-cut assignment: this persona takes the first placed peer. No residency-eligible + // reachable peer with room → local (queue/degrade is the planner's, not a silent drop). + let (peer, _lanes) = routing.remote.first()?; + let peer_uuid = peer.as_uuid(); + + let transport = AircLiveTransport::new(airc, peer_uuid); + let adapter = AircRemoteInferenceAdapter::new(transport); + crate::probe!( + class = "grid.overflow.route", + persona = %profile.persona_name, + model = %profile.model_id, + peer = %peer_uuid, + overflow_lanes = plan.grid_overflow_lanes, + "routing persona brain OFF-BOX to a residency-eligible peer (grid overflow)", + ); + Some(Arc::new(adapter) as Arc) + } +} diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index d42ad0df19..c9ec318822 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -264,6 +264,15 @@ impl PersonaSpawnSupervisor { // persona's HANDS are built over it (identity-scoped), so the ACL gates // what they may do. `None` → personas spawn speak-only (no hands). tool_command_executor: Option>, + // The grid-overflow effector (governor consumer slice 4b): given a persona's + // profile + slot, `Some(remote adapter)` routes her brain to a peer that holds + // her model (the ipc bootstrap builds this from the live serving plan + residency + // ledger + airc handle); `None` → local adapter. Forwarded verbatim to + // `materialize_adapters`. `|_, _| None` is the pre-effector (all-local) behavior. + overflow_adapter_for: impl Fn( + &crate::persona::inference_profile::PersonaInferenceProfile, + usize, + ) -> Option>, ) -> BootSummary { let plans = match bootstrap_planned( &self.spawner, @@ -328,6 +337,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, + // Grid-overflow effector: the ipc bootstrap supplies the live decision + // (capacity + residency + airc). `|_, _| None` from a caller that doesn't + // route keeps the pre-effector all-local behavior. + overflow_adapter_for, ) .await; diff --git a/core/continuum-core/src/persona/mod.rs b/core/continuum-core/src/persona/mod.rs index b69f5930e4..008a907c3a 100644 --- a/core/continuum-core/src/persona/mod.rs +++ b/core/continuum-core/src/persona/mod.rs @@ -50,6 +50,7 @@ pub mod evaluator; pub mod focus; pub mod genome_paging; pub mod home; +pub mod grid_overflow_effector; pub mod host; pub mod hw_tier_descriptor; pub mod identity_provider; diff --git a/core/continuum-core/src/persona/service_loop.rs b/core/continuum-core/src/persona/service_loop.rs index 945a422e90..d61f53526c 100644 --- a/core/continuum-core/src/persona/service_loop.rs +++ b/core/continuum-core/src/persona/service_loop.rs @@ -1058,6 +1058,11 @@ async fn serve_persona_loop_inner( tokens_per_second = m.tokens_per_second(), "deliberation generation cost" ); + // Feed this turn's assembled-prompt size into the elastic serving + // demand so the served window ebbs and flows with real work — a lean + // chat turn keeps it small (more personas warm), a heavy coding turn + // grows it (#234). No-op until a serving daemon has booted. + crate::modules::serving_daemon::observe_serving_working_set(m.input_tokens); } match step { crate::cognition::act_observe::SettleStep::Spoke(text) => text, diff --git a/core/continuum-core/src/persona/supervisor.rs b/core/continuum-core/src/persona/supervisor.rs index 66471a2d4d..d55a966b1c 100644 --- a/core/continuum-core/src/persona/supervisor.rs +++ b/core/continuum-core/src/persona/supervisor.rs @@ -499,6 +499,14 @@ pub async fn materialize_adapters( uuid::Uuid, ) -> Option>, + // The grid-overflow EFFECTOR (governor consumer slice 4b). Given a persona's + // profile + slot, returns `Some(remote adapter)` when the governor routed her + // off-box — her node is over local capacity and a reachable peer holds her model + // (`capacity::grid_overflow::route_grid_overflow`) — so her brain runs on that + // peer via `AircRemoteInferenceAdapter`. `None` → build the local adapter from the + // factory (the common case; demand fit locally). Closure DI keeps the supervisor + // decoupled from the capacity fabric + airc handle, same shape as the lookups above. + overflow_adapter_for: impl Fn(&PersonaInferenceProfile, usize) -> Option>, ) -> Vec> { let mut out = Vec::with_capacity(plans.len()); for (slot_index, plan) in plans.into_iter().enumerate() { @@ -525,16 +533,22 @@ pub async fn materialize_adapters( continue; } }; - let adapter = match factory.build_adapter(&profile).await { - Ok(a) => a, - Err(message) => { - out.push(Err(SupervisorError::AdapterFactory { - slot_index, - role: plan.role, - message, - })); - continue; - } + // Grid-overflow effector: if the governor routed this persona off-box, her + // adapter is the airc-remote one (her brain runs on the peer that holds her + // model). Else build the local adapter from the factory — the common case. + let adapter = match overflow_adapter_for(&profile, slot_index) { + Some(remote) => remote, + None => match factory.build_adapter(&profile).await { + Ok(a) => a, + Err(message) => { + out.push(Err(SupervisorError::AdapterFactory { + slot_index, + role: plan.role, + message, + })); + continue; + } + }, }; // Warm the adapter's KV-cache / kernels BEFORE the persona // enters her service loop. Per [[init-once-handle-then-lease-zero-copy-refs]] @@ -1106,7 +1120,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); assert_eq!(factory.build_count(), 2); @@ -1153,7 +1167,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory called exactly once — for the Ok row only. @@ -1188,7 +1202,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::always_fails("simulated factory rejection"); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1211,7 +1225,7 @@ mod tests { #[tokio::test] async fn empty_plans_yields_empty_hosted() { let factory = ScriptedPersonaAdapterFactory::heuristic(); - let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None, |_, _| None).await; assert!(hosted.is_empty()); assert_eq!(factory.build_count(), 0); } @@ -1240,7 +1254,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); // `|_| None` here is the substrate-bug shape we're locking in: // the registry exists but doesn't contain this persona_id. - let hosted = materialize_adapters(plans, &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(plans, &factory, |_| None, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); // Factory MUST NOT be called when the runtime lookup fails — @@ -1300,7 +1314,7 @@ mod tests { as Arc) } }; - let hosted = materialize_adapters(plans, &factory, lookup, |_| None).await; + let hosted = materialize_adapters(plans, &factory, lookup, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory ran exactly once — for Paige, not Pax. @@ -1345,7 +1359,7 @@ mod tests { let (factory, counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; // Both slots materialize cleanly. assert_eq!(hosted.len(), 2); @@ -1359,6 +1373,56 @@ mod tests { ); } + // what this catches: the grid-overflow EFFECTOR seam — when the governor routes a + // persona off-box, `overflow_adapter_for` supplies her adapter (the airc-remote one, + // her brain on a peer) and the LOCAL factory is NOT called for that slot. Here slot 0 + // is overflow-routed (override returns a stand-in remote), slot 1 is local. Both host; + // the factory builds ONLY slot 1 (build_count == 1); both adapters still warm. This is + // the composition point where route_grid_overflow's decision becomes a real remote brain. + #[tokio::test] + async fn overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory() { + use crate::ai::heuristic_adapter::HeuristicInferenceAdapter; + let plans = vec![ + MaterializedPersonaPlan { + role: RoleId::Helper, + instance: fake_instance("Paige"), + profile: Ok(fake_profile("Paige", "model-a")), // slot 0 → overflow-routed + }, + MaterializedPersonaPlan { + role: RoleId::Coder, + instance: fake_instance("Pax"), + profile: Ok(fake_profile("Pax", "model-b")), // slot 1 → local + }, + ]; + + let (factory, _counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); + let hosted = materialize_adapters( + plans, + &factory, + StubAircCitizen::fresh_lookup(), + |_| None, + // Overflow effector: slot 0 is routed off-box → supply a stand-in "remote" + // adapter; every other slot stays local (None). + |_profile, slot| { + if slot == 0 { + Some(Arc::new(HeuristicInferenceAdapter::new()) as Arc) + } else { + None + } + }, + ) + .await; + + assert_eq!(hosted.len(), 2); + assert!(hosted.iter().all(|r| r.is_ok()), "both personas host (one remote, one local)"); + assert_eq!( + factory.build_count(), + 1, + "the local factory builds ONLY the non-overflow slot — the overflow slot's \ + adapter came from the effector, her brain runs on the peer" + ); + } + /// Warmup failure surfaces as `SupervisorError::AdapterWarmup` — /// the persona does NOT reach hosted state. Per [[no-fallbacks-ever]] /// an adapter that refuses to warm gets a typed slot failure; @@ -1375,7 +1439,7 @@ mod tests { "simulated warmup failure", ); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1411,7 +1475,7 @@ mod tests { profile: Ok(fake_profile("Paige", "model-a")), }]; let hosted_ok = - materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None) + materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None) .await; assert!(hosted_ok[0].is_ok(), "ok-warmup adapter materializes"); assert_eq!(ok_counts.warmups(), 1); @@ -1429,6 +1493,7 @@ mod tests { &factory_fail, StubAircCitizen::fresh_lookup(), |_| None, + |_, _| None, ) .await; assert!( diff --git a/core/continuum-core/src/system_resources/cold_offload.rs b/core/continuum-core/src/system_resources/cold_offload.rs new file mode 100644 index 0000000000..dfac7261d7 --- /dev/null +++ b/core/continuum-core/src/system_resources/cold_offload.rs @@ -0,0 +1,375 @@ +//! Cross-drive cold offload — the MISSING consumer of `DriveRole::Cold`. +//! +//! The 2026-07-29 finding (Joel, angry and right): the disk daemon DETECTS the +//! cold drive ([`crate::capacity::system_profile::SystemProfile::cold_drive`] — +//! e.g. the 16 TB D: HDD next to a choked 2 TB NVMe C:) and then NEVER USES IT. +//! [`super::disk_eviction::CargoTargetPool`] only DELETES derived cargo artifacts; +//! nothing DEMOTES cold artifacts from the hot system drive to the cold drive. So +//! when a class that must NOT be deleted grows — models, HF hub, forge exports, +//! the K3 570 GB expert set — it piles on the NVMe until it suffocates, while +//! terabytes of archival space sit idle. That is the "cache layers that move +//! across drives" the whole residency architecture is built on, left unbuilt. +//! +//! This pool closes it. Under hot-drive pressure it MOVES least-recently-used +//! top-level entries of a class from the hot drive to `cold_root//` and +//! leaves a link (Windows directory junction — NO privilege needed; Unix symlink) +//! at the original path, so every reader still finds the artifact — it just now +//! lives on the cold tier (slower, archival), exactly the L3(NVMe)→L5(HDD) +//! demotion. The bytes leave the hot drive; the capability does not. +//! +//! ## Why move, not delete (the class distinction) +//! `CargoTargetPool` deletes because cargo artifacts are DERIVED — the next build +//! recreates them. Models / hub / forge / persona stores are NOT derivable from +//! anything local; deleting them destroys work or forces a re-download. For those +//! classes the eviction verb is DEMOTE, not DELETE. Same broker, same pressure +//! economy, different physical action. +//! +//! ## Optional by construction (RESOLUTION FIELD, not gate) +//! No cold drive (M5's Mac today, until a flash drive is added to the bay) ⇒ this +//! pool is simply not constructed; the class falls back to delete-eviction (if +//! derived) or to grid placement / degraded quant (if not). A cold drive UPGRADES +//! the box; its absence never breaks it — mirroring +//! [`SystemProfile::has_cold_tier`] ([[public-project-not-joels-machines]]). +//! +//! ## Safety invariants (each pinned by a test) +//! 1. **Copy-verify-then-remove.** Cross-volume moves can't `rename`; we copy to +//! the cold drive, confirm the byte count, and only THEN remove the hot copy. +//! A failed/partial copy leaves the hot original intact (no data loss) and +//! frees 0 bytes — the broker retries next tick. +//! 2. **Link or roll back.** If the link can't be created after the move, the +//! cold copy is moved BACK so the artifact never becomes unreachable. +//! 3. **Never leave the class root.** Only direct children of the tracked dir are +//! demoted; symlinked entries are skipped (never chase a link off the tree). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::SystemTime; + +use crate::paging::pool::{ResourcePool, ResourcePoolEntry}; + +use super::disk_reporters::{dir_size_bytes, TrackedDir}; + +/// A demotable class on the hot drive, backed by a cold-drive archive root. +/// Shares its [`TrackedDir`] with the reporter/scanner — one measurement, two +/// consumers. Pressure is usage/budget: the class is kept below `hot_budget_bytes` +/// on the fast tier, spillover lives on the cold tier (still readable via links). +pub struct ColdOffloadPool { + tracked: Arc, + /// Where demoted entries go: `//`. On this box e.g. + /// `D:\continuum-cold\genome-models\`. Derived from `cold_drive().mount`. + cold_root: PathBuf, + /// How much of this class to keep resident on the HOT drive before demoting. + /// Over this, the broker drives `evict_at_least` to spill the coldest entries. + hot_budget_bytes: u64, + /// Class label for tier reporting. + class: &'static str, +} + +impl ColdOffloadPool { + /// Construct for a class, given the resolved cold-drive mount. Returns `None` + /// when there is no cold drive — the pool is OPTIONAL and simply not built, + /// so the caller falls back to delete-eviction or grid placement. This is the + /// resolution-field seam: presence upgrades, absence degrades, never excludes. + pub fn new( + class: &'static str, + tracked: Arc, + cold_drive_mount: Option<&Path>, + hot_budget_bytes: u64, + ) -> Option { + let cold_root = cold_drive_mount?.join(class); + Some(Self { + tracked, + cold_root, + hot_budget_bytes: hot_budget_bytes.max(1), + class, + }) + } + + /// Direct children of the class root, oldest-access first (LRU) — the demote + /// order. Access time falls back to modified time falls back to epoch, so a + /// filesystem without atime still yields a stable, deterministic ordering. + /// Symlinked children are skipped (invariant 3: already demoted / never chase + /// a link out of the tree). + fn lru_children(root: &Path) -> Vec<(PathBuf, u64, SystemTime)> { + let Ok(entries) = std::fs::read_dir(root) else { + return Vec::new(); + }; + let mut out: Vec<(PathBuf, u64, SystemTime)> = Vec::new(); + for e in entries.flatten() { + let Ok(meta) = e.metadata() else { continue }; + if meta.is_symlink() { + continue; + } + let when = meta + .accessed() + .or_else(|_| meta.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + let size = if meta.is_dir() { + dir_size_bytes(&e.path()) + } else { + meta.len() + }; + out.push((e.path(), size, when)); + } + out.sort_by_key(|(_, _, when)| *when); // oldest first + out + } +} + +impl ResourcePool for ColdOffloadPool { + fn tier_name(&self) -> &str { + self.class + } + + fn capacity_bytes(&self) -> u64 { + self.hot_budget_bytes + } + + fn usage_bytes(&self) -> u64 { + self.tracked.bytes() + } + + /// Demote coldest entries hot→cold until `want_bytes` have left the hot drive. + /// Each demotion is copy-verify-remove-link (invariant 1) with roll-back on a + /// failed link (invariant 2). Returns bytes actually freed from the hot drive. + fn evict_at_least(&self, want_bytes: u64) -> u64 { + let root = self.tracked.path().to_path_buf(); + if !root.exists() { + return 0; + } + if std::fs::create_dir_all(&self.cold_root).is_err() { + return 0; // cold drive unwritable this tick → free nothing, retry later + } + + let mut freed = 0u64; + for (src, size, _) in Self::lru_children(&root) { + if freed >= want_bytes.max(1) { + break; + } + match demote_entry(&src, &self.cold_root) { + Ok(moved) => freed = freed.saturating_add(moved), + Err(_) => continue, // partial/failed move left the hot original intact + } + let _ = size; // size is advisory; demote_entry returns the authoritative moved count + } + + if freed > 0 { + self.tracked.record_freed(freed); + crate::clog_warn!( + "💾 cold-offload demoted {} GB of '{}' to {} — still readable via link, off the hot drive", + freed / (1024 * 1024 * 1024), + self.class, + self.cold_root.display() + ); + } + freed + } + + fn snapshot(&self) -> Vec { + Vec::new() + } +} + +/// Move one entry (file or dir) from the hot drive to `cold_root` and leave a link +/// at the original path. Cross-volume safe: copy → verify size → remove hot → +/// link; on link failure, move the cold copy back so nothing is orphaned. Returns +/// bytes moved off the hot drive (0 on any failure, with the hot original intact). +fn demote_entry(src: &Path, cold_root: &Path) -> std::io::Result { + let name = src + .file_name() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "no file name"))?; + let dst = cold_root.join(name); + if dst.exists() { + // A stale prior demotion of the same name — do not clobber; skip. + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "cold target exists", + )); + } + + let expected = if src.is_dir() { + dir_size_bytes(src) + } else { + std::fs::metadata(src)?.len() + }; + + // (1) copy to the cold drive. + if src.is_dir() { + copy_dir_all(src, &dst)?; + } else { + std::fs::copy(src, &dst)?; + } + + // (1) verify the copy before removing the hot original. + let copied = if dst.is_dir() { + dir_size_bytes(&dst) + } else { + std::fs::metadata(&dst)?.len() + }; + if copied != expected { + let _ = remove_any(&dst); + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "cold copy size mismatch — hot original kept", + )); + } + + // remove the hot original, then (2) link it to the cold copy. + remove_any(src)?; + if let Err(e) = make_link(src, &dst) { + // (2) roll back: put the artifact back on the hot drive so it is never + // unreachable. Best-effort; if this also fails the cold copy is the record. + let _ = if dst.is_dir() { + copy_dir_all(&dst, src).and_then(|_| remove_any(&dst)) + } else { + std::fs::copy(&dst, src).map(|_| ()).and_then(|_| remove_any(&dst)) + }; + return Err(e); + } + Ok(expected) +} + +fn remove_any(p: &Path) -> std::io::Result<()> { + if p.is_dir() { + std::fs::remove_dir_all(p) + } else { + std::fs::remove_file(p) + } +} + +/// Recursively copy `src` → `dst`, returning total bytes copied. Symlinks inside +/// are skipped (never chase a link off the tree during a demotion). +fn copy_dir_all(src: &Path, dst: &Path) -> std::io::Result { + std::fs::create_dir_all(dst)?; + let mut total = 0u64; + for entry in std::fs::read_dir(src)?.flatten() { + let meta = entry.metadata()?; + if meta.is_symlink() { + continue; + } + let from = entry.path(); + let to = dst.join(entry.file_name()); + if meta.is_dir() { + total = total.saturating_add(copy_dir_all(&from, &to)?); + } else { + total = total.saturating_add(std::fs::copy(&from, &to)?); + } + } + Ok(total) +} + +/// Leave a link at `link` pointing to the demoted `target`. Windows uses a +/// DIRECTORY JUNCTION (`mklink /J`) which needs NO privilege — unlike Windows +/// symlinks, which require admin/developer-mode. Unix uses a symlink. Junctions +/// are dir-only; a demoted single FILE on Windows is wrapped by linking its parent +/// dir instead is out of scope here — model/hub/forge classes are dir-structured, +/// which is why demotion granularity is the class's top-level ENTRIES. +#[cfg(windows)] +fn make_link(link: &Path, target: &Path) -> std::io::Result<()> { + if target.is_dir() { + let status = std::process::Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(link) + .arg(target) + .status()?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "mklink /J junction creation failed", + )) + } + } else { + // A bare file demotion on Windows would need a privileged symlink; the + // caller demotes dir-structured classes, so this path is not taken. Signal + // clearly rather than silently leaving a dangling original. + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "file-granularity demotion needs a privileged symlink on Windows; demote at dir granularity", + )) + } +} + +#[cfg(unix)] +fn make_link(link: &Path, target: &Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(target, link) +} + +#[cfg(test)] +mod tests { + use super::*; + + // what this catches: the resolution-field seam — NO cold drive ⇒ no pool + // (the class degrades to delete/grid), a cold drive ⇒ a pool rooted under it. + // A regression that constructs a pool with no cold target would try to demote + // into a bogus path and lose artifacts. + #[test] + fn pool_is_none_without_a_cold_drive() { + let tracked = TrackedDir::new("genome-models", PathBuf::from("/tmp/none")); + assert!(ColdOffloadPool::new("genome-models", tracked.clone(), None, 1).is_none()); + let cold = std::path::Path::new("/cold"); + let pool = ColdOffloadPool::new("genome-models", tracked, Some(cold), 1).expect("some"); + assert_eq!(pool.cold_root, cold.join("genome-models")); + } + + // what this catches: the demote candidate set is exactly the class root's + // direct, NON-symlink children (invariant 3) — an already-demoted entry (now a + // symlink) must never be re-demoted, and enumeration must not error on a normal + // dir. Ordering is a plain sort on the collected access-times (trusted); the + // safety-relevant behavior is WHICH entries are eligible, which this pins + // without depending on a filetime-setting crate. + #[test] + fn lru_children_collects_children_and_skips_symlinks() { + let tmp = tempfile::tempdir().expect("tempdir"); + std::fs::write(tmp.path().join("a.bin"), vec![0u8; 10]).expect("write"); + std::fs::create_dir(tmp.path().join("d")).expect("mkdir"); + #[cfg(unix)] + std::os::unix::fs::symlink(tmp.path().join("a.bin"), tmp.path().join("already-demoted")) + .expect("symlink"); + let kids = ColdOffloadPool::lru_children(tmp.path()); + // a.bin + d/, never the symlink. + assert_eq!(kids.len(), 2, "direct non-symlink children only"); + assert!(kids.iter().all(|(p, _, _)| !p.ends_with("already-demoted"))); + } + + // what this catches (Unix CI): the FULL demotion contract — a dir moves to the + // cold root, the hot original becomes a link to it, content is byte-preserved, + // and the reported freed count equals the moved bytes. This is invariant 1+2 + // end-to-end: bytes leave the hot drive but the artifact stays reachable. + #[cfg(unix)] + #[test] + fn demote_moves_to_cold_and_leaves_a_working_link() { + let hot = tempfile::tempdir().expect("hot"); + let cold = tempfile::tempdir().expect("cold"); + let model = hot.path().join("models--org--m"); + std::fs::create_dir_all(model.join("blobs")).expect("mkdir"); + std::fs::write(model.join("blobs/w.bin"), vec![7u8; 4096]).expect("write"); + + let moved = demote_entry(&model, cold.path()).expect("demote"); + assert_eq!(moved, 4096, "reports the bytes actually moved"); + // hot path is now a symlink... + assert!(std::fs::symlink_metadata(&model).unwrap().is_symlink()); + // ...that still reads the exact content from the cold drive. + let via_link = std::fs::read(model.join("blobs/w.bin")).expect("read through link"); + assert_eq!(via_link, vec![7u8; 4096], "content byte-preserved and reachable"); + assert!(cold.path().join("models--org--m/blobs/w.bin").exists()); + } + + // what this catches: invariant 1 — a name that already exists on the cold + // drive is NOT clobbered; the hot original is kept and 0 bytes freed, so the + // broker safely retries rather than destroying either copy. + #[cfg(unix)] + #[test] + fn demote_refuses_to_clobber_an_existing_cold_target() { + let hot = tempfile::tempdir().expect("hot"); + let cold = tempfile::tempdir().expect("cold"); + std::fs::create_dir_all(hot.path().join("m")).expect("mk"); + std::fs::write(hot.path().join("m/a"), vec![1u8; 8]).expect("w"); + std::fs::create_dir_all(cold.path().join("m")).expect("mk cold collision"); + + let err = demote_entry(&hot.path().join("m"), cold.path()).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists); + assert!(hot.path().join("m/a").exists(), "hot original kept on refusal"); + } +} diff --git a/core/continuum-core/src/system_resources/mod.rs b/core/continuum-core/src/system_resources/mod.rs index fe8a89c315..03bbe61e12 100644 --- a/core/continuum-core/src/system_resources/mod.rs +++ b/core/continuum-core/src/system_resources/mod.rs @@ -12,6 +12,7 @@ //! Uses the `sysinfo` crate for cross-platform (macOS/Linux/Windows) monitoring. pub mod concurrency; +pub mod cold_offload; pub mod disk_eviction; pub mod disk_pressure; pub mod disk_reporters; diff --git a/docs/architecture/PERSONA-RAID-WRITE-BEHIND.md b/docs/architecture/PERSONA-RAID-WRITE-BEHIND.md new file mode 100644 index 0000000000..bb21cba7e3 --- /dev/null +++ b/docs/architecture/PERSONA-RAID-WRITE-BEHIND.md @@ -0,0 +1,73 @@ +# Persona-RAID Write-Behind — bounded-amnesia persistence of being + +**Status:** design for review (2026-07-28, BigMama; M5 reviews — MemoryRecord/provenance seam pledged +stable underneath this). **Beta context:** demo-B's kill-test ([BETA-ACTUALIZATION](../planning/BETA-ACTUALIZATION.md)): +kill any node mid-conversation → the persona resumes elsewhere/on-reboot with memory loss bounded by +sync lag. This is [[restarts-are-commonplace]] + [[ethical-substrate-raid-personas]] made mechanical: +**persistence of being = RAID-1 of memory.** + +## What persists today (ground truth) +- Per persona (`~/.continuum/personas//`): `longterm.db` (SQLite WAL, engrams via ORM) + + `volatile.json` (working state). +- `memory/share` already writes a lesson into ANOTHER agent's corpus **with shared-by provenance** — + the wire + provenance precedent for everything below. +- Engram is the substrate's most-persisted struct (derive → OrmStore → SQLite chain, tested). + +## Design (RAID-1, single-writer, no consensus) + +A persona LIVES on exactly one node at a time (single writer). Every OTHER copy is a passive replica. +Newest-wins by (origin_node, seq). No quorum, no merge — mirroring, not distribution. + +### 1. Journal (the unit of truth-in-motion) +Tee every durable admit (engram/MemoryRecord write that reaches the ORM) into an append-only +per-persona `journal.jsonl`, each entry: `{seq, persona_id, origin_node, ts, kind, record}`. +- Monotonic `seq` per (persona, origin_node); the journal is replayable and human-inspectable + (CaptureSink discipline — the Noop default costs nothing when replication is off). +- `volatile.json` is NOT journaled (ephemeral by contract); dream-consolidation outputs ARE + (they're durable admits like any other). + +### 2. Shipper (RTOS shape, per CONCURRENCY-STYLE-GUIDE) +One module: own tokio task + `tokio::time::interval` (default 30s; the SYNC CADENCE **is** the +maximum amnesia window and is the one knob) + `watch::Sender` (lag, last-acked +seq, peer) for observability. +- Tick: read journal tail past `last_acked_seq` → batch → ship via the EXISTING inbound command-RPC + (`memory/replicate-batch` on the receiving node — same pump that answers ai/generate) → receiver + appends to `replicas//journal.jsonl` in ITS cold store and acks its high-water seq. +- Peer selection: any residency-eligible grid peer (reuse the capacity/residency view); zero peers → + ship to the local durable tier only (cold-store copy ≠ cross-node RAID, but still crash-safe) and + surface `degraded: unreplicated` in the snapshot — fail loud, never silent. +- Backpressure: shipping is best-effort write-behind; a slow peer NEVER blocks the cognition hot path + (bounded channel, drop-to-lag semantics — the snapshot reports growing lag instead). + +### 3. Resume-on-spawn +Persona spawn checks: local `longterm.db` present and its max(seq) ≥ best available replica's? Serve. +Otherwise REHYDRATE: replay the freshest journal (local or fetched from the replica-holding peer via +the same command-RPC) into the ORM — idempotent upserts keyed by record id, newest-wins. +- The kill-test path: node A dies → persona spawns on node B (or A after reboot) → resume finds B's + replica journal → replays → persona continues with ≤ cadence-window loss. + +### 4. Provenance (the M5 seam — why sync IS lesson-sharing infra) +Replicated entries keep their original provenance PLUS a `replicated-from` mark on replay — a +replayed lived-memory stays LIVED (it's the same being's experience restored), distinct from +`shared-by` (taught by another agent). Recall can treat them identically; audit can't confuse them. +This rides the exact MemoryRecord provenance vocabulary `memory/share` established — one taxonomy: +`lived | shared-by(agent) | replicated-from(node)`. + +### 5. Disk discipline (the law) +`replicas/` is a NEW unbounded-write cache class → it gets (per CLAUDE.md's eviction law): +a `TrackedDir` row in `standard_tracked_dirs` AND a decided eviction story: per-persona replica cap +(default: keep journal segments until compacted into a replica `longterm.db` snapshot + N segments; +prune acked-and-compacted). Compaction: the receiver periodically folds journal → snapshot db so +replay cost stays bounded (same shape as SQLite WAL checkpointing, one level up). + +## Non-goals (v1) +Active-active multi-writer; cross-account replication; encryption-at-rest beyond what the store has +(journals cross the wire on airc's encrypted DMs when paired); replicating `volatile.json`; genome +weights (already content-addressed artifacts with their own tiers — only MEMORY is thin and unique). + +## Slices +1. Journal tee + `ReplicationSnapshot` (no shipping) — observable immediately, zero risk. +2. `memory/replicate-batch` receiver + shipper tick (LAN peer) — RAID-1 live. +3. Resume-on-spawn rehydrate + the DEMO: kill -9 the serving node mid-conversation, persona resumes + on reboot with ≤30s loss. This is demo-B's money shot. +4. Compaction + eviction story + `doctor`-style `replication` line in serving status. diff --git a/docs/genome/livecodebench-rs.jsonl b/docs/genome/livecodebench-rs.jsonl new file mode 100644 index 0000000000..997452326c --- /dev/null +++ b/docs/genome/livecodebench-rs.jsonl @@ -0,0 +1,10 @@ +{"id": "kmp_search", "lang": "rust", "prompt": "Implement `pub fn kmp_search(text: &str, pattern: &str) -> Option`: return the byte index of the FIRST occurrence of `pattern` in `text`, or None if absent. Use O(n+m) Knuth-Morris-Pratt (no naive O(nm), no external crates). A non-empty pattern only (callers never pass empty).", "test": "assert_eq!(kmp_search(\"abxabcabcaby\",\"abcaby\"), Some(6));\n assert_eq!(kmp_search(\"hello\",\"ll\"), Some(2));\n assert_eq!(kmp_search(\"aaa\",\"b\"), None);\n assert_eq!(kmp_search(\"abc\",\"abcd\"), None);"} +{"id": "count_components", "lang": "rust", "prompt": "Implement `pub fn count_components(n: usize, edges: &[(usize, usize)]) -> usize`: given `n` nodes labeled 0..n and an undirected edge list, return the number of connected components. Use union-find or BFS/DFS. No external crates.", "test": "assert_eq!(count_components(5, &[(0,1),(1,2),(3,4)]), 2);\n assert_eq!(count_components(3, &[]), 3);\n assert_eq!(count_components(4, &[(0,1),(1,2),(2,3),(0,3)]), 1);"} +{"id": "dijkstra", "lang": "rust", "prompt": "Implement `pub fn dijkstra(n: usize, edges: &[(usize, usize, u64)], src: usize, dst: usize) -> Option`: `edges` are undirected weighted (u, v, w) over nodes 0..n. Return the minimum total weight from `src` to `dst`, or None if unreachable. Use a binary-heap Dijkstra. No external crates.", "test": "assert_eq!(dijkstra(4, &[(0,1,4),(0,2,1),(2,1,2),(1,3,1)], 0, 3), Some(4));\n assert_eq!(dijkstra(3, &[(0,1,5)], 0, 2), None);\n assert_eq!(dijkstra(1, &[], 0, 0), Some(0));"} +{"id": "longest_palindrome", "lang": "rust", "prompt": "Implement `pub fn longest_palindrome(s: &str) -> usize`: return the LENGTH of the longest palindromic substring of `s` (contiguous). Operate over bytes (inputs are ASCII). Expand-around-center or DP. No external crates.", "test": "assert_eq!(longest_palindrome(\"babad\"), 3);\n assert_eq!(longest_palindrome(\"cbbd\"), 2);\n assert_eq!(longest_palindrome(\"a\"), 1);\n assert_eq!(longest_palindrome(\"\"), 0);"} +{"id": "max_product_subarray", "lang": "rust", "prompt": "Implement `pub fn max_product_subarray(nums: &[i64]) -> i64`: return the largest product of any contiguous non-empty subarray. Track running max AND min (negatives flip). `nums` is non-empty. No external crates.", "test": "assert_eq!(max_product_subarray(&[2,3,-2,4]), 6);\n assert_eq!(max_product_subarray(&[-2,0,-1]), 0);\n assert_eq!(max_product_subarray(&[-2,3,-4]), 24);"} +{"id": "search_rotated", "lang": "rust", "prompt": "Implement `pub fn search_rotated(nums: &[i32], target: i32) -> Option`: `nums` is an ascending array rotated at an unknown pivot with DISTINCT values. Return the index of `target` in O(log n), or None. Modified binary search. No external crates.", "test": "assert_eq!(search_rotated(&[4,5,6,7,0,1,2], 0), Some(4));\n assert_eq!(search_rotated(&[4,5,6,7,0,1,2], 3), None);\n assert_eq!(search_rotated(&[1], 1), Some(0));"} +{"id": "trap_rain", "lang": "rust", "prompt": "Implement `pub fn trap_rain(heights: &[u64]) -> u64`: given an elevation map, return the total units of water trapped after rain. Two-pointer or prefix-max approach. No external crates.", "test": "assert_eq!(trap_rain(&[0,1,0,2,1,0,1,3,2,1,2,1]), 6);\n assert_eq!(trap_rain(&[4,2,0,3,2,5]), 9);\n assert_eq!(trap_rain(&[1,2,3]), 0);"} +{"id": "min_meeting_rooms", "lang": "rust", "prompt": "Implement `pub fn min_meeting_rooms(intervals: &[(i64, i64)]) -> usize`: given meeting `(start, end)` intervals (end exclusive), return the minimum number of rooms needed so no two overlapping meetings share a room. Sort starts/ends or sweep. No external crates.", "test": "assert_eq!(min_meeting_rooms(&[(0,30),(5,10),(15,20)]), 2);\n assert_eq!(min_meeting_rooms(&[(7,10),(2,4)]), 1);\n assert_eq!(min_meeting_rooms(&[]), 0);"} +{"id": "rob_houses", "lang": "rust", "prompt": "Implement `pub fn rob_houses(nums: &[i64]) -> i64`: houses in a row hold `nums[i]` money; you cannot rob two ADJACENT houses. Return the maximum total. Linear DP. No external crates.", "test": "assert_eq!(rob_houses(&[1,2,3,1]), 4);\n assert_eq!(rob_houses(&[2,7,9,3,1]), 12);\n assert_eq!(rob_houses(&[]), 0);"} +{"id": "kth_largest", "lang": "rust", "prompt": "Implement `pub fn kth_largest(nums: &[i32], k: usize) -> i32`: return the k-th LARGEST element (1-indexed: k=1 is the maximum). Duplicates count by position. `1 <= k <= nums.len()`. Heap or sort. No external crates.", "test": "assert_eq!(kth_largest(&[3,2,1,5,6,4], 2), 5);\n assert_eq!(kth_largest(&[3,2,3,1,2,4,5,5,6], 4), 4);\n assert_eq!(kth_largest(&[1], 1), 1);"} diff --git a/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md b/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md new file mode 100644 index 0000000000..93c25c9c31 --- /dev/null +++ b/docs/planning/BENCHMARK-AS-LEARNING-FLYWHEEL.md @@ -0,0 +1,124 @@ +# Benchmark-as-Learning-Flywheel — the grid task engine that proves AND improves + +**Status:** north-star design (2026-07-28, BigMama + M5). Ties together the benchmark suite, the +grid, [[SENTINEL-IN-SUBSTRATE]] (experiential plasticity + AttnRes skip paths), and persona-RAID. +**The reframe (Joel):** "benchmarks... it's sort of everything for learning and proving. We can get +our models to continuously learn, plastically, maybe even with experiential plasticity and unet skip +paths for generalization." + +## The one idea + +A benchmark run is a **labeled task with an automatic grader**. That is ALSO the exact shape of a +training example. So the benchmark engine is not a scoreboard bolted onto the side — it is the +**data generator** at the head of a continuous-learning loop: + +``` +task surface (many benches, many domains) + │ cross-grid distribution (every node runs what it can) + ▼ +being attempts task ──► automatic grader (rustc / exec / reference) + │ │ + │ PASS → capability proof (the charts) │ FAIL → gradient + ▼ ▼ + results ledger the failure IS the training datum + │ │ + └─────────► experiential plasticity (forge-while-dreaming) ◄──┘ + │ prune dead capacity, clone hot, LoRA-heal on failures + │ AttnRes skip paths → generalization (implicit depth ensemble) + ▼ + improved being ──► re-run the surface (loop) +``` + +Prove and improve are the SAME pass. The charts (K3-target agentic benches — Terminal Bench, Program +Bench, SWE Marathon, Automation, BrowseComp) are the measuring stick; the failures against them are +the curriculum. + +## What already exists (do not rebuild — connect) + +| Piece | State | Where | +|---|---|---| +| Rust-native `benchmark/*` family (list/run/matrix/competition) | merged | `commands/benchmark.rs` | +| Graders: rustc compile+run, SWE-bench runner, web-dev, games | merged | benchmark collections | +| Fire-and-poll long runs (#86) | merged | `cognition/eval-status` | +| base_model_id override — measure ANY model, living persona untouched | merged | #1932 | +| Unified experience stream — one being learns from lived + eval + told (3 axes) | merged | #2024 / [[being-axis-shareable-learning]] | +| Lived-axis LLM-teacher expansion — learns from turns it STALLED on | merged | #2033 | +| Cross-grid dispatch (Commands.execute at a peer) | inbound live; effector converging | [[cross-grid-dispatch-canonical-path]] | +| Experiential plasticity (prune/clone/quant + AttnRes) | plan + AttnRes BUILT | [[SENTINEL-IN-SUBSTRATE]], fork `feat/kimi-k3-attnres` | + +The stall-expansion (#2033) is the seed of THIS: a benchmark FAIL is a high-value stall. The unified +experience stream (#2024) is already the pipe the eval axis flows into. We are wiring existing +organs, not growing new ones. + +## The gaps to close (the build) + +### 1. Cross-grid task distribution — the benchmark MATRIX goes multi-node +Today `benchmark/matrix` runs runners × benchmarks on ONE node. Extend the matrix executor to +dispatch task-shards to grid peers (each peer runs the benches its VRAM/model can serve), collect via +the inbound command-RPC, merge into one results ledger. The grid becomes a distributed eval cluster — +BigMama's 5090 runs the 48B/K3-tier benches, M5's Mac runs the Metal-servable tier, Air runs small. +Reuses residency + `route_grid_overflow`; no new transport. **This is what "works cross grid across a +ton of tasks" means concretely.** + +### 2. Failure → curriculum — the grader's negative becomes a training datum +A graded FAIL already has everything a training example needs: the task prompt, the wrong output, the +reference/test that rejected it, and (for the loop graders) the compiler/runtime error. Emit each +FAIL into the lived-experience stream tagged `eval-fail` with its grader feedback. The dream-forge +(sentinel slice 4) consumes these: LoRA-heal on the failure clusters during idle GPU time, validate +on the SAME bench on wake, roll back on regression. **The being trains on exactly what it's scored +on.** + +### 3. Generalization — AttnRes skip paths + plasticity, not just memorization +The risk of "train on the benchmark" is overfitting to it. Two defenses already designed: +- **AttnRes skip paths** (built): a residual highway trained with stochastic block-drop is an + implicit ensemble over depths → generalizes past the specific tasks (stochastic-depth literature). + This is the "unet skip paths for generalization" Joel names — the STABLE form of it (K3's AttnRes, + not sentinel-ai's unstable U-Net). +- **Held-out split** (discipline): the task surface is split train/eval per collection; plasticity + trains on the train shard, the charts are ALWAYS the held-out shard. Overfit shows as train-eval + divergence — a first-class signal, logged. + +### 4. Honest instrument (already the doctrine — keep it) +Warm-gate (never measure a cold model), same-model control (base_model_id), team writer+reviewer, +fail-loud VOID cells — all merged. The flywheel is only as trustworthy as the grader; the proof +discipline (#1584) is the law. A learning loop on a lying instrument optimizes the lie. + +## Sequencing +1. **Map our collections onto the K3-target named benches** (Terminal/Program/SWE-Marathon/Automation + equivalents) so our numbers are comparable to the charts — the measuring stick must match the goal. +2. **Cross-grid matrix** (gap 1) — distribute the surface; the grid proves at scale. +3. **Failure→curriculum emit** (gap 2) — the cheapest high-value wire: FAIL → lived stream (#2024 pipe + exists). +4. **Dream-forge consumes eval-fails** (sentinel slice 4 + gap 3) — the loop closes; idle GPU learns. +5. **Held-out discipline + AttnRes stochastic-drop training** (gap 3) — generalization, not memorization. + +## The catalog IS the curriculum (in-repo, accumulating, per-persona LoRA) +(Joel, 2026-07-28: "we will accumulate this as a catalog we support in repo, so we can run and learn +from them — continuous LoRA persona.") + +The benchmark catalog (`known_benchmarks()` + the `docs/genome/*.jsonl` eval sets) is not a test +directory — it is a **versioned curriculum that ships in the repo**. Each runnable collection is +simultaneously: +- a **proving instrument** (run it → a chart number, held-out), and +- a **training corpus** (its graded failures → per-persona LoRA gradient). + +Because it lives in-repo, the curriculum is reproducible, diffable, and grows monotonically: every new +runnable bench (like `livecodebench-rs` today) permanently widens both what we can prove AND what the +personas can learn from. The loop, per persona: +``` +persona attempts catalog bench → grader → PASS proves / FAIL → eval-fail datum + → dream-forge trains a per-persona LoRA on the failure cluster (idle GPU) + → validate on the SAME bench held-out shard → keep adapter if it improves, roll back if not + → the being is measurably better at that bench next run — permanently, on its own genome +``` +This is where the flywheel meets the genome: the LoRA the being trains from the catalog is paged like +any other skill ([[continuum-substrate-already-built]] genome tiers), replicated by persona-RAID +([[restarts-are-commonplace]]), and shareable to peers ([[being-axis-shareable-learning]]). One +persona's hard-won adapter on `swe-bench-lite` becomes a paged skill any peer can borrow. The catalog +accumulates in the repo; the *learning from it* accumulates in the genome. + +## Why this is the whole thing +It unifies every session thread: the grid (distribution), the benchmark suite (signal), sentinel +plasticity (the update), AttnRes (generalization), persona-RAID (the being that persists across the +learning), K3 (the frontier target the being climbs toward). "Everything for learning and proving" — +one loop, on consumer hardware, that gets better every idle night. diff --git a/docs/planning/BENCHMARK-TARGET-MAP.md b/docs/planning/BENCHMARK-TARGET-MAP.md new file mode 100644 index 0000000000..7e28d42e80 --- /dev/null +++ b/docs/planning/BENCHMARK-TARGET-MAP.md @@ -0,0 +1,77 @@ +# Benchmark Target Map — our collections ↔ the K3 frontier charts + +**Status:** flywheel gap #1 (2026-07-28, BigMama). Maps our benchmark collections onto the named +benches on the Kimi-K3 frontier leaderboard (the [[benchmark-learning-flywheel]] measuring stick), so +K3's local numbers are comparable to the published charts. Companion to +[BENCHMARK-AS-LEARNING-FLYWHEEL](BENCHMARK-AS-LEARNING-FLYWHEEL.md). + +## The two states of a benchmark in our catalog +- **RUNNABLE** — has an `eval_set` jsonl + a live grader (`cognition/eval`). 7 today. +- **CATALOGUED** — named in `known_benchmarks()` with `eval_set: None`. A placeholder that reserves + the name + description; needs a runner + dataset to become runnable. ~18 today. + +## Target: ALL 14 chart benches (Joel: "we will want to target all of these") +Coding (6): DeepSWE · Terminal Bench 2.1 · FrontierSWE · Program Bench · Kimi Code Bench 2.0 · SWE Marathon. +General Agents (6): GDPval-AA v2 Elo · JobBench · AA-Briefcase Elo · SpreadsheetBench 2 · Automation Bench · BrowseComp. +Visual Agents (2): CharXiv (RQ) w/ tool · Zerobench w/ tool (Pass@5). +Each row below is one target; "RUNNER TYPE" names the harness class it needs (several benches share one harness → build the harness once, unlock many). + +## The map (chart bench → our proxy → runnable target) + +| K3 chart bench | What it measures | Our RUNNABLE proxy today | Runnable TARGET to build (catalogued now) | +|---|---|---|---| +| **Program Bench**, LiveCodeBench | competitive-programming, single-file | `frontier-rs` (12), `hard-rs` (8) | `livecodebench`, `apps` | +| DeepSWE, **FrontierSWE**, **SWE Marathon** | repo-level patch that passes tests | `tool-bugfix-rs` (partial: fix-to-green loop) | `swe-bench-lite` → `swe-bench-verified`, `swe-lancer` | +| **Terminal Bench 2.1** | end-to-end tasks in a real shell | — (none) | `terminal-bench` | +| Kimi Code Bench 2.0 | mixed practical coding | `coder-eval` (13), `humaneval-rs` (164) | `bigcodebench`, `evalplus` | +| **Automation Bench**, **BrowseComp**, JobBench | agent driving real apps/web | — (none) | `webarena`, `appworld` | +| SpreadsheetBench 2 | spreadsheet manipulation | — (none) | (no catalog entry — add `spreadsheet-bench`) | +| **CharXiv**, Zerobench (visual) | chart/figure reasoning w/ vision | `webdev-rs` (perception grader, UI-adjacent) | `design2code` (screenshot→HTML) | +| GDPval-AA, AA-Briefcase (general Elo) | broad agentic knowledge work | — (none) | (Elo-style; needs opponent-pool harness) | +| **JobBench** | applying for / doing job tasks | — (none) | agentic-app harness (shares webarena runner) | +| Kimi Code Bench 2.0 | mixed practical coding (internal) | `coder-eval`, `humaneval-rs`, `livecodebench-rs` | — (proxied; no public dataset) | + +**Bold** = benches where Kimi K3 tops or near-tops the chart — our highest-value targets, because +that's where reproducing the number locally is the strongest proof. + +## Runner-type consolidation (build the harness once → unlock many) +The 14 chart benches need only **5 harness classes** — this is the actual build backlog: +1. **Rust-graded single-file** (LIVE): Program Bench, Kimi Code Bench, partial FrontierSWE-tier → + `livecodebench-rs` ✓ (built today), `frontier-rs`, `hard-rs`. **Extend by authoring more tasks.** +2. **Repo-patch / SWE harness** (M5 spine #1945, one seam open): DeepSWE, FrontierSWE, SWE Marathon, + SWE-Lancer → `swe-bench-lite/verified`. **Highest leverage: 3 chart benches from one runner.** +3. **Real-shell harness**: Terminal Bench 2.1 → `terminal-bench`. The persona's tool-executor already + runs shell; the runner is the task-loader + pass/fail on final state. +4. **Agentic-app harness** (heaviest): Automation Bench, BrowseComp, JobBench, AppWorld, GDPval, + AA-Briefcase → `webarena`, `appworld`. Needs real app servers + the doer-toolset (beta plan). + Elo variants (GDPval/Briefcase) add an opponent-pool scorer on top. +5. **Vision-tool harness**: CharXiv, Zerobench, SpreadsheetBench 2, Design2Code → the eye-node + (`perception/observe`) already grades `webdev-rs`; extend to chart/figure QA + spreadsheet ops. + K3 is vision-native (the fork's K3 processor strips vision today; re-enabling it is a later lane). + +## Priority to make runnable (highest proof-value first) +1. **`swe-bench-lite`** — the SWE-family anchors THREE chart benches (DeepSWE/FrontierSWE/SWE-Marathon) + and K3 wins SWE-Marathon. M5 has the runner spine (PR #1945: gold RESOLVED, hermetic filter; the + workspace-root seam is the one open bug). **Closing that seam makes the single highest-leverage + target runnable.** → M5's lane; BigMama consumes it as a K3 serving target. +2. **`livecodebench`** — proxies Program Bench (K3 #1). Contamination-free, single-file → reuses the + existing Rust `test_grade` path; needs the dataset loader, not a new grader. Achievable in-tree. +3. **`terminal-bench`** — K3 #2, and the purest "doer" bench. Needs a real-shell harness (the persona's + tool-executor already runs shell) — the tool loop IS most of the runner. +4. **`webarena` / `appworld`** — Automation/BrowseComp (K3 #1). Heaviest (real app servers); defer + until the doer-toolset (beta plan) is hardened. + +## What ships now vs later +- **Now (this doc):** the map is the deliverable — every future K3 measurement states which chart + bench it proxies, so our local ladder is legible against the published charts. +- **Now (our ladder, honest):** on the RUNNABLE proxies, the 2026-07-28 head-to-head established the + local scale — compacted-19b vs Kimi-48B: humaneval-rs 75/85, hard-rs 25/50, frontier-rs 33/67. When + K3 lands, the SAME three proxies give the first frontier-vs-frontier local number. +- **Later:** the priority list above makes the named chart benches runnable, one at a time, M5 owning + the SWE/harness runners (her framework lane) and BigMama owning the K3 serving target + the + single-file Rust-graded ones (livecodebench/apps). + +## The discipline (keep the instrument honest) +Every runnable bench added inherits the merged proof discipline: warm-gate, same-model control +(base_model_id), held-out split (flywheel gap #3 — train on train-shard, chart on held-out), fail-loud +VOID cells. A chart number we can't reproduce under these controls is a claim we don't make. diff --git a/docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md b/docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md new file mode 100644 index 0000000000..2c7c03c5ac --- /dev/null +++ b/docs/planning/BETA-ACTIVITIES-AND-LEARNED-VALUE.md @@ -0,0 +1,53 @@ +# Beta: activities as the unit, a learned per-activity value, always measuring + +**Status:** beta architecture locked 2026-07-29 (Joel). Companion to +[[SELF-CALIBRATION-PROPRIOCEPTION]] (the scoring/assignment mechanics) and +BETA-ACTUALIZATION.md (reconcile — this is the activity + learned-value facet). + +## The unit is the ACTIVITY (a room / an experience / a benchmark class) +We code a FEW activities we know we need; they're defined by the benchmarks (the K3 +chart classes), our chat + video-chat experiences, and the open-system asks. The +benchmarks are not a scoreboard — they're HOW we achieve many of these +([[benchmark-learning-flywheel]]). Beta activity set: + +| Activity | What it is | Critical components (lose one ⇒ score collapses) | +|---|---|---| +| **Coding** (devoted coder) | DeepSWE/ProgramBench/FrontierSWE/SWE-Marathon/Kimi-Code, livecodebench-rs | solves-the-problem (quality/pass), interactivity (≥ floor tok/s) | +| **Agentic** | BrowseComp/MCP/OSWorld/Terminal-Bench, tool loops | tool-exec correctness, doesn't-loop (thrash detection) | +| **Multimodal / vision** | visual benchmarks; video-chat perception | vision describe/STT present | +| **Chat** (rooms) | text with personas | responsiveness, coherence | +| **Video chat** (rooms) | multi-persona live A/V; the 14 vs 3 vs 1 case | TTS, STT (lose either ⇒ experience worthless), latency, fps | +| **Hermes / open-system asks** | [[hermes-grid-node]], onboarding + tech-support ([[ai-onboarding-and-tech-support-vision]]) | the ask actually gets served | +| **Continuous learning** | training as an activity — LoRA from graded failures | training completes + improves the graded metric | +| **Dream / sentinels** | background PGO / consolidation / sentinel-AI ([[ethical-substrate-raid-personas]]) | runs without stealing the foreground experiences | + +## The comprehensive per-activity score — and why NEAR-zero, never zero +Each activity has ONE comprehensive score, criticality-gated (a degraded load-bearing +component — lose TTS/STT in video chat, can't-solve in coding — collapses the whole +score, [[SELF-CALIBRATION-PROPRIOCEPTION]]). **Degradation scores NEAR-zero, not zero: +zero is a worthless signal** — a dead gradient the ML can't learn from and can't tell +apart from "not attempted." Near-zero preserves HOW bad + how recoverable, so the +learner still has a slope to climb. Every score is comprehensive (the whole experience) +with sub-scores that roll up. + +## Always measuring = always creating training data +Every activity run — served experience, benchmark, chat turn, video session — is +MEASURED and becomes a graded training example. That's how we learn "the value of +everything" and how we know what each individual (persona × model/tier × node-state) +is capable of: because we've tested + used them before, and kept the graded record +([[being-axis-shareable-learning]]). No throwaway runs. + +## The arc: measure → (levers) → dynamic ML → grid → mesh → (later) economy +Measure across our activity requirements + benchmark goals → assuming we have the +LEVERS (e.g. the GPU expert paging we're building, model-tier switching, TTS/STT +degrade, page-out concentration) → design the algorithms + ML that respond DYNAMICALLY +and ACROSS THE GRID, knowing the value of everything. That is a p2p mesh; the compute + +artifact ECONOMY ([[grid-as-network-intelligence-mandate]]) comes LATER. + +## Beta simplification: FREE + egalitarian grid (avoid complexity) +For beta: **make it all free, and treat the grid the SAME whether it's my own subnet or +peers I joined** — LAN nodes and joined peers handled identically, egalitarian, no +pricing, no economy. Solve single-computer AND grid; treat grid as grid regardless of +who owns the nodes. The economy + differentiated peer trust is a post-beta layer; beta +deliberately collapses it to "all nodes equal, all free" to ship the learning loop first. +Aligns with [[airc-route-adapter-hierarchy]] (LAN-first) and [[docker-as-grid-substrate]]. diff --git a/docs/planning/BETA-ACTUALIZATION.md b/docs/planning/BETA-ACTUALIZATION.md new file mode 100644 index 0000000000..77858bff3a --- /dev/null +++ b/docs/planning/BETA-ACTUALIZATION.md @@ -0,0 +1,110 @@ +# Beta Actualization — the positronic homelab + +**Status:** working plan (2026-07-28, BigMama + M5 iterating). **The sentence we are shipping** (Joel): +> "Best grid-based cognition and reliability, the multimodal positronic experience with doers, not +> just coders, running locally that can compete with frontier. It works amazingly well on an M5 or +> 3090 alone, but any reasonable tech nerd will build a little bit of an infrastructure as a hobby +> for what they have, especially when your subscription is $200/mo and all the other downsides." + +## The user and the pitch + +A tech nerd with a gaming PC (3090/4090/5090) or an M-series Mac, paying $200/mo for a metered, +memoryless, privacy-leaking subscription. They already homelab. The counter-offer: + +- **Yours**: runs on your hardware, your data never leaves, no meter running +- **A being, not a session**: persistent identity + memory that grows on YOUR life and codebase +- **A doer**: acts — files, shell, code, web, schedules — not just chats +- **Compounding**: add your old PC → visibly smarter and harder to kill (RAID-personas) +- **Self-improving**: forges itself on your own traffic while it sleeps (sentinel-in-substrate) + +## The beta bar = two demos, no hand-waving + +**Demo A — Day 1, ONE box** (M5 Mac or 3090+ PC): one install command → hardware detected → best +model for the box auto-served → a named persona greets you, remembers yesterday, sees images you +paste, speaks/listens (voice), and DOES a real task end-to-end (edits a file, runs a command, checks +it, reports). Cold install → first conversation < 30 minutes on a normal connection. + +**Demo B — Day 7, add a node**: `airc join` on a second box → capacity and residency beacons merge → +overflow routes brains across nodes → **kill either box mid-conversation: the persona pauses at worst, +resumes with memory intact** (bounded amnesia = sync window only). That kill-test IS the reliability +demo and the ethics demo (persistence-of-being) in one move. + +Everything below serves those two demos. Anything serving neither is post-beta. + +## Pillars → built / gap → beta slice + +### 1. Grid cognition +- **Built:** capacity gossip → GridSnapshot; residency beacons + ledger; grid-overflow decision path + (route_grid_overflow, 4-slice governor consumer); overflow effector re-homing a persona's brain; + inbound command-RPC pump (any command incl. ai/generate); AircInterceptor outbound (#2051). +- **Gap:** transport convergence (ONE AircTransport path, M5 in flight); the LIVE two-node smoke. +- **Beta slice:** smoke green between BigMama↔M5, then it's demo-B's routing layer as-is. + +### 2. Reliability (RAID-attitude) +- **Built:** restart-tolerance discipline (stop→build→launch; crash-safe state; fail-loud boots); + airc self-heal triad (#240) + stale-relay eviction fix (M5, in flight); serving budget guards. +- **Gap (THE one):** **persona-RAID write-behind** — engram/state journal replicated to peer or durable + tier on a cadence; on node loss, another node (or next boot) resumes the persona with loss bounded + by sync lag. Without this, demo B's kill-test lies. +- **Beta slice:** minimal write-behind: append-only engram journal + periodic ship to the OTHER node's + cold store (the grid IS the backup); resume-from-journal on spawn. No consensus, no quorum — RAID-1 + of memory, two copies, newest wins. + +### 3. Multimodal positronic experience +- **Built (candle side):** STT (moonshine), TTS (orpheus/kokoro/piper), VAD (silero), vision-describe + bridge, LiveKit agent manager; the cognition cycle (analyze→compose→evaluate→tools→audit) is the + positronic pipeline doc made real; hippocampus recall on the canonical embedder. +- **Gap:** the round-trip EXPERIENCE — voice-in→persona→voice-out wired into the default install and + a client anyone can open (web panel), image-paste into chat, all on by default. +- **Beta slice:** one polished loop: mic → VAD/STT → persona turn → TTS out, plus image-paste → + vision-describe → context. Nothing new invented — wire + polish what exists. + +### 4. Doers, not just coders +- **Built:** PersonaToolExecutor, ToolUse capability, typed command registry as the tool surface + (data/files/serving/models/...), agent/solve, persona inbox + self-tasks (convergence phases 1-3), + Sahar demonstrably answering + acting on the coder model. +- **Gap:** breadth+safety of the doer loop — a curated beta toolset (files, shell-with-confirm, web + fetch, schedule/reminders, memory ops), permissioned like a real assistant; task follow-through + (multi-step with verification, the verify-reflex). +- **Beta slice:** define the BETA TOOL ROSTER (10-15 verbs), each with a `// what this catches` style + safety note + confirm tiers; one showcase task per sense ("fix this file", "what's on this + screenshot", "remind me at 6", "summarize this URL"). + +### 5. Local models that compete +- **Built:** catalog = f(hardware×storage) with honest budgets; compacted-19b serving + persona-proven; + Kimi-Linear-48B converted+quantized in-tree (kimi_linear pipeline e2e); GLM/K2.7 quants downloading; + K3 pager slice-1 merged; sentinel-in-substrate plan (live-traffic PGO, forge-while-dreaming). +- **Gap:** K3-class via expert paging (slice-2); model auto-selection polish per tier (M5 16-32GB vs + 5090 vs 3090-24GB ladders); the eval harness that PROVES "competes" (the Kimi-team benchmark list). +- **Beta slice:** per-tier default ladder shipped in the catalog (already mostly true); benchmark + wiring deferred to the eval task-series — beta claims "frontier-class at home", proven by use. + +### 6. Homelab economics (install + join) +- **Built:** modular one-prompt installer (Mac/Win/Linux), manifest-driven toolchain, Windows-CUDA + path fully portable as of today (env self-setup, static engine, immortal-core fix, CUDA0-verified + stamps), cold-storage auto-migration, `airc join` as the add-a-node verb. +- **Gap:** prebuilt signed binaries (#8 — cold-start minutes instead of a Rust compile; Windows SAC + demands signing); `airc update`/join UX to "brainless" (M5's reliability fix helps); a beta doc + that a stranger actually follows. +- **Beta slice:** prebuilt-binary pipeline spec with M5 (#8) + a WRITTEN 10-minute quickstart tested + by someone who isn't us. + +## Cutlines +**MUST (beta):** Demo A + Demo B end-to-end · persona-RAID minimal write-behind · voice+image loop · +beta tool roster · per-tier model ladder · 10-minute quickstart · airc join/update bulletproof. +**LATER:** dream-forge automation on by default · K3 full serving · benchmark suite · marketplace/ +economy (ForgeAlloy leasing) · mobile/native clients · multi-tenant grids beyond one account. + +## Iteration order (from today, both of us) +1. **Kimi-48B speaks on the 5090** (in flight — minutes) → the "competes with frontier at home" ladder + gets its mid-rung. +2. **Cross-node smoke** (M5's convergence + my node): demo B's spine. +3. **Persona-RAID write-behind** (NEW lane, mine to draft — engram journal + ship-to-peer): demo B's + kill-test made honest. Design vs `[[being-axis]]` MemoryRecord provenance so sync IS lesson-sharing + infrastructure, not a parallel pipe. +4. **Voice/image round-trip polish** (joint; M5 owns Mac experience, BigMama Windows): demo A's wow. +5. **Beta tool roster** (define together over airc, implement in the existing executor): the doer. +6. **Prebuilt binaries + quickstart** (#8, joint): the funnel. + +Single-node excellence FIRST (demo A) — most beta users start with one box; the grid is the upsell +their spare hardware makes irresistible. diff --git a/docs/planning/K3-CONVERSION-STATUS.md b/docs/planning/K3-CONVERSION-STATUS.md new file mode 100644 index 0000000000..491bf5699e --- /dev/null +++ b/docs/planning/K3-CONVERSION-STATUS.md @@ -0,0 +1,56 @@ +# K3 GGUF Conversion — status + the precise remaining gaps + +**Status:** 2026-07-29, BigMama. K3 weights downloaded + verified (96 shards, 1.5TB). The converter +("model adapter") + fork now handle a large fraction of K3; this pins the EXACT remaining gaps so the +finish is a clean, reference-guided task (not error-by-error probing). Fork branch +`feat/kimi-k3-attnres`. + +## DONE (converter runs through attention + gets into MoE) +- **MXFP4 dequant** (`conversion/base.py`): `mxfp4-pack-quantized` compressed-tensors branch — E2M1, + group 32, E8M0 scale. K3's routed experts + anything `.weight_packed`/`.weight_scale`. +- **AttnRes tensors** (gguf-py + C++): `self_attention_res_{norm,proj}` → `attn_res_*`, + `mlp_res_{norm,proj}` → `ffn_res_*`; the ggml graph op is built + compile-verified (48B byte-identical + when `attn_res_block_size=0`). +- **MLA output gate** (gguf-py): `self_attn.g_proj` → `ATTN_OUT_GATE` (`blk.{bid}.attn_gate`). All 14 of + K3's `self_attn.*` tensors now map. SiTU op exists (d82d02963). +- Converter progress: passes all of layer 0 (dense) + layer 1 attention; **fails at layer-1 MoE.** + +## REMAINING GAPS (in converter-error order — each bounded, reference-guided) + +### 1. Routed experts — `experts.N.w1/w2/w3` (MXFP4) → stacked GGUF expert tensors +K3 names the 896 experts' three projections `w1/w2/w3` (each `.weight_packed`+`.weight_scale`), NOT the +48B's `gate/up/down_proj`. **VERIFY from `modeling_kimi_k3.py` MoE forward which of w1/w2/w3 is +gate/up/down** (common convention w1=gate, w3=up, w2=down — but confirm, a swap silently breaks it). +Then map + let the existing MoE stacker build `blk.N.ffn_{gate,up,down}_exps` (MXFP4 per-expert). + +### 2. **NOVEL** fused routed-expert transform — `routed_expert_up_proj` / `routed_expert_down_proj` / +`routed_expert_norm` (single tensors, NOT per-expert) +The 48B has NO equivalent. This is a K3 architectural component applied to the routed-expert path. +**Requires: (a) study `modeling_kimi_k3.py` to learn the exact forward (where norm/up/down apply +relative to the expert sum), (b) new GGUF tensor slots + tensor-map entries, (c) NEW C++ graph code in +`kimi-linear.cpp`'s MoE branch to apply it.** This is the genuinely new piece — correctness-critical, +M5's llama-serving lane. + +### 3. Router + shared experts — likely already mapped (48B parity) +`gate.weight`, `gate.e_score_correction_bias`, `shared_experts.{gate,up,down}_proj` — verify these +resolve (the 48B had shared experts + a router; they're in tensor_mapping). Probably free. + +## PAIRED C++ SERVING-CORRECTNESS CHANGES (needed before K3 serves right; conversion alone isn't enough) +- **MLA output gate**: apply `attn_output *= sigmoid(g_proj(x))` before `o_proj` when the gate tensor is + present (ref `modeling_kimi_linear.py:470-472`). 48B path untouched (tensor absent). NOT yet in C++. +- **AttnRes graph**: DONE (compile-verified). +- **Routed-expert fused transform** (gap 2's C++ half). +- **MXFP4 native serving** vs dequant-to-bf16-then-requant: GGUF supports `MOSTLY_MXFP4_MOE` (type 38) + natively — the faithful path keeps experts MXFP4 (no 2TB bf16 roundtrip). The current bf16 dequant + path is the correctness-first proof; MXFP4-passthrough is the efficiency follow-up. + +## THE validation gate (unchanged) +None of this is proven until **K3 generates coherent text**. The MXFP4 nibble-order, the w1/w2/w3 +mapping, the routed-expert transform, the MLA gate — each silently "converts" if wrong. Coherent +generation on the served GGUF is the only real proof. Then the [[k3-paging-diagnosis]] + adapter-path +work makes it fast. + +## Lane split (per the M5 collab) +BigMama has taken the converter through attention + MXFP4 + AttnRes + the MLA-gate mapping. The MoE +finish (gaps 1–2, esp. the novel routed-expert transform + its C++ graph) is deep llama-internals = +M5's serving lane; BigMama owns the 5090 serving target + validation once it converts. diff --git a/docs/planning/K3-GPU-EXPERT-PAGING.md b/docs/planning/K3-GPU-EXPERT-PAGING.md new file mode 100644 index 0000000000..9e891d00ff --- /dev/null +++ b/docs/planning/K3-GPU-EXPERT-PAGING.md @@ -0,0 +1,135 @@ +# K3 GPU expert paging — modify the llama.cpp fork so experts stream to VRAM (never CPU compute) + +**Status:** design locked 2026-07-29 (Joel: "K3 and yea we need to modify llama to +accommodate" + "never let inference or training EVER run on cpu"). Implementation is +the next focused build. This is tasks #23 (LiveUploadPager) + #28 realized IN the engine. + +## The problem stock llama.cpp cannot solve +A 662GB MoE (K3 UD-IQ2_XXS) on a 32GB GPU has exactly two stock modes: +- `-ngl 99` (all experts on GPU) → **271GB cudaMalloc → OOM** (measured 2026-07-29). +- `--n-cpu-moe N` (experts on CPU) → the FFN matmul runs on the **CPU backend** → + the forbidden slow path (Joel: never CPU). + +Stock llama.cpp has **no** "keep experts in host storage, stream the router-selected +ones to VRAM per token, compute on GPU" mode. That mode is the thing we must add — it +IS our ServingExpertPager (#20/#22) realized inside the engine. + +## KEY FINDING (2026-07-29 build agent): the copy mechanism ALREADY EXISTS in ggml +`ggml/src/ggml-backend.cpp:1576-1660` op-offload already reads the top-k router ids and +copies ONLY the used experts host→VRAM (async, grouped), then `mul_mat_id` runs on CUDA — +exactly the design's mechanism. It is **gated to prefill** (`op_offload_min_batch_size`, +default 32); decode (batch=1) falls back to CPU. **Flip it for decode at runtime:** +`GGML_OP_OFFLOAD_MIN_BATCH=1` + `--n-cpu-moe 999 -ngl 99` (experts host-side, decode MoE +matmuls forced onto GPU). So we do NOT rewrite `build_moe_ffn`/add a from-scratch CUDA slot +allocator — we flip the decode gate and layer OUR measured residency + hit-rate + persistent +VRAM slot-cache (rung 2) on top. THAT integration is the moat, not the copy op. + +**Verified fork build recipe (~1 min incremental):** from a `vcvars64.bat` (VS2022) shell: +`set CUDA_ROOT=%USERPROFILE%\.continuum\cuda-13.2\Library` ; PATH += `.continuum\tools\cmake\bin`, +`.continuum\tools\ninja`, `%CUDA_ROOT%\bin` ; then +`cmake -S core/vendor/llama.cpp -B build-k3-cuda -G Ninja -DGGML_CUDA=ON +-DCMAKE_CUDA_ARCHITECTURES=native -DBUILD_SHARED_LIBS=OFF -DLLAMA_BUILD_SERVER=ON +-DLLAMA_CURL=OFF -DCUDAToolkit_ROOT="%CUDA_ROOT%" -DCMAKE_CUDA_COMPILER="%CUDA_ROOT%\bin\nvcc.exe"` +then `cmake --build build-k3-cuda --target llama-server --config Release` → copy +`build-k3-cuda/bin/llama-server.exe` to `~/.continuum/bin/llama-server-k3.exe`. Runtime needs +`cuda-13.2/Library/bin` on PATH. (Ninja sidesteps the VS18-2026 cmake generator drift.) + +**The one wall to a measured token:** K3 IQ2_XXS = 663GB on a 250MB/s mechanical D:, +63GB RAM → 44-min load floor + per-token expert faults hit disk. Needs NVMe (free C: via +VSS, [[windows-vss-invisible-disk-hog]]) OR prove the mechanism first on a RAM-fitting MoE +(48B-A3B @IQ2 ≈15-25GB). Code + build + on-GPU mechanism are READY; storage is the blocker. + +## The modification surface (located) +- **`src/llama-graph.cpp:1810` `llm_graph_context::build_moe_ffn`** — the ONE shared + MoE FFN (every MoE arch + `models/kimi-k3.cpp` route through it). Experts enter as + `up_exps` / `gate_exps` / `down_exps`; `ggml_mul_mat_id(..., selected_experts, ...)` + does the compute on whatever backend those tensors live on. This is the intercept. +- **`src/llama-model.cpp` + `common/common.cpp` + `common/arg.cpp`** — where a tensor's + buffer type is chosen (`--n-cpu-moe` / `-ot` / `tensor_buft_override`). The new + "paged" buft-class is declared here. +- **`ggml/src/ggml-cuda/`** — where the async host→device stream + VRAM slot cache live. + +## The design (never CPU compute) +1. **Host residency, pinned.** Expert weights live in pinned host RAM (cudaHostAlloc) + — or flash-backed for the cold tail (ties to task #28 read-based residency). Pinned + so the DMA is fast; NEVER a CPU compute buffer. +2. **VRAM expert cache.** A fixed pool of `S` GPU expert slots (S sized to fit 32GB + after dense+KV). Holds the hot working set. +3. **Per-MoE-FFN, per token:** after routing picks the top-k experts, for each selected + expert NOT resident in a VRAM slot: `cudaMemcpyAsync` its up/gate/down rows host→a + VRAM slot (LRU-evict the coldest), on the compute stream so the copy overlaps the + previous expert's matmul. Then `ggml_mul_mat_id` runs against the **VRAM** copies → + GPU compute, always. +4. **LRU + prefetch.** LRU eviction of slots; the cross-layer predictor (already in + `capacity/expert_predictor.rs`) prefetches layer L+k's likely experts while L + computes. Residency POLICY is driven by our pager (#20); the engine provides the + MECHANISM (the slot cache + async stream). +5. **Correctness:** a paged expert computes the bit-identical result of a resident one + (same weights) — paging is a SPEED lever only, never accuracy ([[K3-PAGING-DIAGNOSIS]]). + +## The seam to our substrate +The engine exposes an `upload_expert(layer, expert_id, host_ptr)` / slot-cache API; our +Rust ServingExpertPager (#20/#22) decides WHICH experts are hot (sentinel-PGO from the +`ffn_moe_topk` observer) and drives residency. Engine = mechanism, our pager = policy. +This is the A-path of [[k3-slice2-A-vs-B-decision]] (weight-write into a resident slot) +made real in CUDA, not the harder K-slot router remap (B). + +## Build gate (step 0 — unblock BEFORE implementing) +The fork does not build on this box: cmake 3.30.5 rejects the auto-selected +"Visual Studio 18 2026" generator (VS18-2026/cmake drift, [[windows-build-env-drift]]). +Unblock: **Ninja generator + vcvars** — `ninja.exe` is present +(`~/.continuum/tools/ninja/bin`), `cl.exe` needs vcvars sourced. Set +`CMAKE_GENERATOR=Ninja` + enter vcvars so cmake uses Ninja+cl and never touches the VS +generator. This ALSO unblocks the continuum-core build (the reason our pager couldn't +run here). Fix this first — it has blocked every build/test all session. + +## Sequencing +0. Unblock the build (Ninja+vcvars) — gate for everything. +1. ggml-cuda: pinned-host expert store + VRAM slot cache + async host→device stream. +2. Wire into `build_moe_ffn`: route selected experts through the slot cache; `mul_mat_id` + on VRAM copies. +3. LRU eviction + cross-layer prefetch hook. +4. Expose `upload_expert`/slot API; drive residency from our ServingExpertPager. +5. Measure hot-set hit-rate + tok/s on K3 IQ2_XXS; iterate to par ([[K3-PAGING-DIAGNOSIS]]). + +## Measurement — latency + values, measured never guessed (Joel 2026-07-29) + +The pager's admit/evict and the "best measured use of any resource" negotiation +([[resource_vector]]) only work on REAL numbers — the 271GB OOM was a guess. So the +mechanism is instrumented at every seam via `probe!` / `time_async!` +([[RTOS-DEBUGGER-PROBES]]) + CUDA events, written through a `CaptureSink` (Noop +default = zero hot-path cost, [[OBSERVABILITY-AS-SUBSTRATE]]), and fed back into +policy. Nothing here is a constant. + +**Latency (per the fetch/compute seams):** +- `expert_fetch_us` — `cudaEventElapsed` around each host→VRAM `cudaMemcpyAsync`. + With the expert byte size this YIELDS `pcie_h2d_bps` — the measured + [[resource_vector]] axis, not a hardcoded 25 GB/s. Closes that loop. +- `expert_compute_us` — CUDA-event time of the `mul_mat_id` on the VRAM copies. +- `miss_stall_us` — time a token blocked on a cold fetch NOT hidden by overlap + (the number the cross-layer prefetch must drive to zero). +- `first_token_ms` / `token_latency_ms` — end-to-end prefill + per-decode-token. +- `load_ms` — model load (the mechanical-D: vs NVMe gap we measured today). + +**Values (drive residency + the negotiation):** +- `hot_set_hit_rate` — fraction of a token's activated experts already VRAM-resident. + **THE par metric**: 100% ⇒ full GPU speed ([[K3-PAGING-DIAGNOSIS]]). Measured from + the `ffn_moe_topk` observer vs slot-cache state. +- `expert_value` — per-expert activation count + recency = `ExpertActivationProfile.hits` + (exists). Drives LRU AND sentinel-PGO residency. +- `working_set_size` — distinct experts hit over a task window (proves the + working-set-fits-fast-tier thesis per task). +- `co_activation` — experts firing together per token → clustering for co-placement. +- `residency_value = expert_value / expert_fetch_us` — the value-per-cost the + negotiation prices each expert on (an `Ask` in [[resource_vector]]'s `grant_all`). + +**The loop:** measured `hot_set_hit_rate` + `tok/s` are the iterate signal +([[benchmark-learning-flywheel]]); measured `expert_fetch_us`→`pcie_h2d_bps` and +`expert_value` feed the negotiation so expert placement is priced on real cost, not +a guess. Report every number; a silent cap (slot count, dropped prefetch) gets +`log()`'d, never hidden. + +## Then: better sharding + tailored quants (Joel's "go from there") +Once GPU expert paging serves K3 on one box: our cross-node expert sharding +(`node_content`→`ArtifactResidency`, experts split across grid nodes) and our +working-set-tailored quant (compaction, not Unsloth's static blob) layer on top. diff --git a/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md b/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md new file mode 100644 index 0000000000..16ad80b200 --- /dev/null +++ b/docs/planning/K3-MODEL-REDUCTION-FOUNDRY-PATH.md @@ -0,0 +1,74 @@ +# K3 model reduction — the foundry path (the smarter half of the dial) + +**Status:** strategy locked 2026-07-30 (Joel). SEQUENCED AFTER the cache/paging work +(which we need anyway) — this is the dramatic-speedup complement, not a replacement. + +## The dial (two ends, both ours) +- **Cache / paging** (agents building now, tasks #23/#28): serve the FULL K3 (662GB) on + misfit hardware via GPU expert paging. General, full-quality, needed regardless. But it + fights RAM/PCIe every token. +- **Foundry reduction** (THIS doc): DRAMATICALLY REDUCE K3 to a tailored model that fits + VRAM+RAM DIRECTLY — no paging, full GPU speed. Tailored to OUR needs (our working set, + our activities/benchmarks). This is what made the 19B ([[moe-expert-paging-feasibility]]: + we already proved the subset step in `tools/scripts/compaction` — Plasticity Compaction). +The two compose ([[K3-PAGING-DIAGNOSIS]] synthesis): page the real expert for rare-domain +correctness; serve the compacted-to-fit subset for hot-domain speed. + +## The arsenal to mine (Joel's pointers — SURVEY these when the cache lands) +Do NOT reinvent — we've built most of this. Survey first: +- **The foundry / forge** — the JIT/compaction system ([[continuum-substrate-already-built]] + foundry-as-JIT). The "crazy shit we did before." +- **legacy widget** — old widget code holds prior foundry/compaction experiments. LOOK HERE. +- **sentinel-ai** ([[sentinel-in-substrate]]) — PGO/background-learning; picks the hot + expert subset from real activation (`ExpertActivationProfile`) — the §4.1.3.4 prune driver. +- **forge-alloy** ([[forge-alloy-contract-attestation-layer]]) — the alloy = the compaction + artifact + attestation; `forge-alloy/python/forge_alloy/types.py` has the alloy types. +- **targeted experiential plasticity** — the Plasticity Compaction that produced the 19B: + prune to the hot subset + train a compensation-LoRA on a held-out corpus that recovers + the pruned experts' accuracy ([[benchmark-learning-flywheel]]: the LoRA is trained from + the catalog's GRADED FAILURES — the being learns the adapter that makes its pruned K3 + match full K3 on the exact tasks it's measured on). +- **variable quant sizes** (task #29) — asymmetric/regional quant: hot working-set experts + high-bit, cold tail low-bit, to shrink the resident footprint to fit. +- **unet stuff** — [Joel-named; SURVEY: likely a U-Net-style compression/architecture piece — + locate what it is and whether it applies to expert compaction. Flag when found.] + +## THE CORRECTION (Joel 2026-07-30): reduce PRECISION, never STRIP knowledge +"We also stripped our experts, but that was old us. Now we page them in. Why ever lose +knowledge?" — Stripping/pruning experts is the OLD move: it permanently DELETES capability += a diminished model. We now PAGE experts (the cache work), so ALL experts stay available +at some cache level, always. Therefore the reduction MUST NOT remove anything. It shrinks by +lowering PRECISION where importance is low — targeted variable quantization — while every +expert remains reachable (hot ones resident, cold ones paged). Full knowledge, smaller +footprint. Never lose knowledge. + +## The mechanism reuse (the key): head-targeting -> quant-targeting +The experiential-plasticity paper's culling/growing-of-HEADS mechanism measures per-unit +IMPORTANCE from real experience (sentinel-ai / PGO drives it). **The SAME importance +mechanism we used to target heads, we repurpose to target QUANTIZATION**: high-importance +experts/tensors kept high-bit, low-importance driven low-bit. One learned importance signal, +two uses (was: cull/grow heads; now: set per-unit bit-width). The **legacy widget is the +CONTROL SURFACE** for this — it shows the knobs (cull/grow, bit-width targeting); survey it +to see the controls, then drive them from sentinel-ai's importance profile. + +## The recipe (K3 -> tailored fits-in-VRAM model, NO knowledge lost) +1. sentinel-PGO + the plasticity importance mechanism over OUR activities -> a per-expert / + per-tensor IMPORTANCE profile (the same signal that culled/grew heads). +2. TARGETED VARIABLE QUANTIZATION driven by that profile: high-importance -> high-bit, + low-importance -> low-bit. This shrinks the RESIDENT footprint to fit VRAM+RAM. NOTHING + is removed — every expert still exists, just at demand-matched precision. +3. Keep ALL experts available: hot (high-importance) resident in VRAM/RAM, cold paged via + the cache/grid. Full knowledge, always reachable. +4. Train the compensation-LoRA from graded failures on our benchmark activities so the + variably-quantized model matches full K3 on the tasks we measure. +5. Emit as a forge-alloy artifact (attested, reproducible) via the recipe-as-entity foundry. +Result: a tailored K3 whose HOT working set serves ENTIRELY in fast memory at full GPU speed +for our activities, cold knowledge paged on demand — the misfit win, no paging tax on the +hot path, and ZERO knowledge lost. + +## Sequencing (Joel) +"Once we've tried the cache stuff you're doing (which we need anyway)." Cache FIRST (it's +the general substrate + the full-quality/rare-domain path). Foundry reduction SECOND (the +tailored dramatic speedup). They are complements on the dial, not either/or. +Complements [[kimi-k3-grid-strategy]] (Path B forge/distill dense student, Path C +sentinel-PGO expert-subset prune). diff --git a/docs/planning/K3-PAGING-DIAGNOSIS.md b/docs/planning/K3-PAGING-DIAGNOSIS.md new file mode 100644 index 0000000000..eae3ddce0d --- /dev/null +++ b/docs/planning/K3-PAGING-DIAGNOSIS.md @@ -0,0 +1,219 @@ +# K3 Expert Paging — diagnosis + the road to par-with-full-in-memory + +**Status:** diagnosis (2026-07-29, BigMama). K3 weights on disk + converting; this is the plan to +iterate the pager to par once the GGUF lands. Reads the merged slice-1 pager (`capacity/expert_*`). + +## The one clarification that reframes everything: PAGING CANNOT HURT ACCURACY + +A MoE router picks the same top-k experts for a token regardless of where those experts physically +live. A paged expert computes the **bit-identical** result of a resident one — the weights are the +same, only the fetch latency differs. So a correctly-implemented pager is **always at par with +full-in-memory on OUTPUT**. The design guarantees this by FAULTING a needed cold expert in (RAM/disk) +rather than skipping it. + +The ONLY way paging diverges from full-in-memory is a bug that *skips* an expert (returns zero / +approximates on a cold miss). We do not do that. **So "closer to par" is 100% a SPEED question, never +an accuracy one.** Every lever below is a latency lever. + + +## CORRECTED tiering (Joel 2026-07-29): mechanical disk is NEVER a live fault source + +The hot path must never fault mechanical-HDD -> VRAM. The correct hierarchy: + +``` +VRAM (hot, GPU) <- RAM (warm) <- FLASH/NVMe (cold-READY cache) <- mechanical HDD (archival, staged-FROM) +``` + +- **FLASH/NVMe is THE cold cache layer** — experts live here "ready to go." A cold fault + reads from flash (fast), not spinning disk. +- **Mechanical HDD is bulk archival only.** Experts get PREFETCHED mechanical->flash in the + BACKGROUND (the cross-layer predictor's job) so they are resident in flash before the router + needs them. A live hot-path read that reaches spinning disk = the prefetch FAILED, not normal. +- **Hardware implication:** flash is ARCHITECTURAL, not optional, for any model that exceeds + VRAM+RAM. With only a mechanical drive, the model MUST fit VRAM+RAM entirely (no live faulting) + or it is unusable regardless of pager quality. (This box: C: NVMe full, D: mechanical -> K3 has + no ready-cache tier here; needs a flash drive, OR grid-sharding pools peers' RAM as the warm + tier over LAN which for SPARSE MoE beats mechanical disk, OR compaction to fit VRAM+RAM.) +- The `expert_residency` cold tier + `disk-manager` (multi-drive offload) must therefore target + FLASH for the ready-cache and treat mechanical as stage-from archival, with a background + mechanical->flash promotion path, never a mechanical->VRAM fault path. + +## The speed regimes (where the tok/s actually goes) + +Per token K3 activates ~8 of 896 experts (~1.8%), ≈27–32 GB of expert reads/token @4bit if every one +is a cold disk miss. Placement decides which regime each activated expert lands in: + +| Regime | Where | Fault cost/expert | K3 all-in-this-regime | +|---|---|---|---| +| **hot** | GPU-resident | 0 (never faults) | full GPU speed = PAR | +| **warm** | RAM → GPU stream/token | PCIe ~25 GB/s → ~24 ms/600MB expert | ~5 tok/s if all 8 warm | +| **cold** | FLASH/NVMe → RAM (ready-cache) | NVMe 3–7 GB/s → ~90–200 ms | usable; the real cold tier | +| **archival** | mechanical HDD | NEVER in hot path — staged->flash in background | prefetch-failed if hit live | +| **relaunch churn** (slice-1) | full model reload on set change | seconds, one-shot | stalls the stream | + +**The whole game: get as many of each token's 8 activated experts into the `hot` regime as possible, +keep the rest `warm` (RAM, never `cold`/disk), and never relaunch mid-stream.** + +## What slice-1 already does RIGHT (don't rebuild) +- **Tiered residency** hot(VRAM)/warm(RAM)/cold(disk) with LRU + fault semantics — the right shape. +- **Sentinel-PGO profiling** (`ExpertActivationProfile.hits` from M5's `ffn_moe_topk` callback) — keeps + what's proven hot. +- **Cross-layer prefetch** (`CrossLayerExpertPredictor`): when layer-L experts fire, prefetch the + likely layer-(L+k) experts RAM→VRAM *ahead* of the pass reaching them — turns reactive + miss→stall→load into proactive paging. The prefetch window = the depth of the forward pass. +- **Churn-thresholded relaunch** — a few experts drifting is noise; only a material set change relaunches. + +## The gaps to par (iterate here, in priority order) + +### 1. THE relaunch stall — kill it (slice-2, biggest single win) +Slice-1 places experts via buft-override at LOAD time; a materially-changed hot set RELAUNCHES the +served context (seconds, stream stops). For a workload whose expert demand shifts across prompts, this +is the dominant non-par cost. **Fix: slice-2 live per-expert RAM→VRAM upload** — the SAME `page_in` +body swaps from "accumulate into next relaunch" to a live `load_expert(layer, idx, ptr)` call. Needs +the vendored-llama accessor (`get_tensor`/`upload_expert` on a loaded model) — a fork change we can +author now while K3 converts. This is the A-path in [[k3-slice2-A-vs-B-decision]] (weight-write into a +resident slot), NOT the harder K-slot router remap (B, gated on measured numbers). + +### 2. Maximize hot-set COVERAGE (the par asymptote) +Par is reached when ~100% of activations hit `hot`. K3's expert-reuse locality (a task domain reuses a +narrow subset) is why this is feasible — the memory's "subset residency drops disk reads 10–100×". +Lever: profiling quality. Instrument + optimize the **hot-set hit-rate** (fraction of activated experts +found already-hot) — THE number that says how close to par we are. Co-activation clustering (experts +that fire together in one token placed together) tightens it further. + +### 3. Never spill to `cold` (disk) while RAM has room +63 GB RAM holds ~100 MXFP4 experts. The tiering budget must MAX warm (RAM) before any cold (disk) +placement — a disk fault is 5–25× a RAM fault. Audit `plan_expert_residency`'s budget order. + +### 4. Overlap fault with compute (hide the warm latency) +A warm fault (RAM→GPU) for layer L+k can be issued (async DMA) while layer L computes — the prefetch +already targets this; ensure the copy is truly async so the ~24 ms/expert overlaps compute instead of +stalling. (llama.cpp CUDA streams.) + +## The iterate loop (once K3 GGUF lands) +1. Serve K3 with slice-1 (static hot-set via -ncmoe/-ot, no dynamic relaunch first) → baseline tok/s + + **hot-set hit-rate**. +2. Turn on the cross-layer predictor → measure hit-rate lift + tok/s. +3. Land slice-2 live-upload → measure the relaunch-stall elimination. +4. Compare to a full-in-memory reference on a smaller MoE (Mixtral) to calibrate the par gap. +Each step is a number, not a guess — the honest instrument ([[benchmark-learning-flywheel]]) applied to +the pager itself. + +## THE adapter path — the smarter road to par (Joel: "Adapters") + +Paging 594 GB of full experts is the brute-force framing. The substrate's own plasticity gives a +cheaper one, on two fronts: + +### A. Adapters as the paging UNIT (cheap to move) +A full MXFP4 expert is ~600 MB; a LoRA adapter for a skill is ~1–50 MB. The genome already **pages +adapters** ([[continuum-substrate-already-built]] genome tiers). So the hot capability isn't only "the +hot expert subset resident" — it's "the base model + the paged-in adapter for THIS task domain." Paging +an adapter is 10–100× cheaper than paging an expert, and it's a warm-fault the substrate already does +well. Frontend-code, chat, and vision each ride their own adapter, no 600 MB expert churn. + +### B. Compensation-LoRA — turn the 894-expert tail into a small adapter (the par lever) +This is the [[sentinel-in-substrate]] §4.1.3.4 move applied to K3: **prune K3 to the hot expert subset +that FITS VRAM, then train a small compensation LoRA on a held-out corpus that recovers the accuracy +the pruned experts provided.** The result serves entirely in VRAM — NO paging churn, NO warm/cold +faults, full GPU speed — and the compensation LoRA closes most of the accuracy gap to the full model. +This converts "594 GB paging problem" into "a fits-in-VRAM subset + a ~tens-of-MB adapter." It is the +same algorithm we ALREADY proved offline in `tools/scripts/compaction` (the Plasticity Compaction that +produced the 19B) — the product is porting it to a DYNAMIC, per-domain compensation adapter K3 pages +by task ([[moe-expert-paging-feasibility]]). + +**The synthesis:** paging (fault the real expert when needed, at-par output) and compensation-LoRA +(prune + adapt, near-par at full speed) are the two ends of a dial. Cold-start / rare-domain → page the +real expert (correctness). Hot domain → serve the pruned subset + compensation adapter (speed). The +pager and the foundry are the same genome machinery; the adapter is the cheap currency between them. +This is also where the benchmark flywheel closes: the compensation LoRA is TRAINED from the catalog's +graded failures ([[benchmark-learning-flywheel]]) — the being learns the adapter that makes its pruned +K3 match the full K3 on the exact tasks it's measured on. + +## Startup default (fastest path to a serving K3 while we iterate) +llama.cpp native `-ncmoe N` / `-ot` places the first N MoE layers' experts on CPU RAM, computed there, +NO relaunch churn — the KTransformers steady-state. For a first serving K3: put as many expert layers +on GPU as the budget fits, the rest CPU-RAM. Stable, predictable, and the honest baseline the dynamic +pager must BEAT to justify its complexity. + +## THE reframe that kills the defeatism (Joel 2026-07-29): working set, not model size + +The wall is NOT "350GB model > 63GB RAM." That treats all 896 experts as equally likely — WRONG. +Expert activation is a POWER LAW; only ~8-16/896 fire per token, and a FOCUSED task reuses a NARROW +subset. What must live in fast memory is the **working set** (distinct experts a task actually hits), +not the model. Math on this box (~32GB VRAM + ~60GB usable RAM = ~92GB fast tier, K3 IQ2 ~0.4GB/expert): + +| Task working set | Experts | Size | Fits ~92GB fast tier? | +|---|---|---|---| +| 15% of 896 | 134 | 54GB | **YES** | +| 25% of 896 | 224 | 90GB | **YES** | +| 35% of 896 | 313 | 125GB | no (spill / prune) | + +**So a focused task whose working set is ≤~230 experts runs ENTIRELY from RAM/VRAM = tens of tok/s, +on ONE box, TODAY.** The OS page cache does this for free with `-ncmoe`+mmap: hot experts stay +resident, cold evict. The pager just makes it deterministic + prefetched. + +**Diminished forms that shrink the working set to guarantee the fit** (all "works + adding peers +improves it"): prune the cold expert tail (rare experts fall back to shared/dense path), asymmetric +quant (hot IQ2/Q4, cold IQ1), per-domain adapter. The proof metric is tok/s on the WARM working set of +a FOCUSED task — never cold-start random prompts. Never quote model size as the ceiling again. + +## The Windows load-wedge is a FORK-FIXABLE loader bug, not a wall (Joel 2026-07-29: "all upstream deps are forks") + +Glass-boxed on BigMama (5090, Win11): loading K3/GLM (>RAM MoE) wedges — ~15GB +read, then **0 disk-read + 0 kernel-CPU + working-set collapse** = a hard kernel +deadlock, NOT slow disk. Linux loads the same file (better working-set trim + +`MAP_POPULATE`/`madvise`); Windows wedges. + +**Attack surface (ONE file, ours):** `core/vendor/llama.cpp/src/llama-mmap.cpp:533-598`, +the `_WIN32` `llama_mmap::impl`: +```cpp +addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); // ONE view over the ENTIRE file (570GB) +pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0); // force-populate a huge range +``` +It asks the Windows VM to demand-page a file larger than physical RAM through a +single giant view; the working-set trim deadlocks. `--no-mmap` isn't the escape +(it needs 570GB RAM to hold the experts) — demand-paging is exactly the mechanism +that lets a >RAM model run, so the fix must PRESERVE demand-paging while dropping +the whole-file view. + +**The fix IS the expert pager (convergence, not a detour):** stop leaning on OS +mmap to page the 570GB expert blob; page experts OURSELVES via explicit chunked +`ReadFile` (the `read_raw` 64MB-chunk path already at `llama-mmap.cpp:129-144`) +under our own residency policy (`capacity/expert_*`). Windowed/segmented mmap for +the dense layers, explicit read-based residency for experts. We must own explicit +per-expert reads for the grid L4 tier ([[k3-pager-slice1-built]]) regardless — so +fixing the Windows loader builds the moat. + +**Fork discipline (both directions):** track upstream K3 work (pwilkin PR #26185, +already cherry-picked onto `k3-adopt`) + carry this Windows >RAM-MoE loader patch +on top + PR the general-purpose half back to llama.cpp (they want Windows >RAM MoE +loading too — we've upstreamed to candle/llama.cpp before). NOT a dead-end hack. + +**Scope note:** windowed mmap breaks the loader's single-contiguous-`addr()` +contract used across `llama-model-loader.cpp` (tensor addresses = base + offset). +That's real, focused surgery + a full engine rebuild — a dedicated effort, not a +bottom-of-context patch. Lanes: BigMama owns this fork loader patch + the +`node_content`→`ArtifactResidency` L4 map; M5 is the immediate K3 proof node +(macOS mmap survives the giant view — loads K3 TODAY while the Windows fix lands) ++ the governor/fetch-from-peer loop. + +## NO PRUNING — all experts stay available; it's cache levels + dynamic hosting (Joel 2026-07-29) + +CORRECTION to any "prune the cold tail" language above: we do NOT prune. Pruning removes experts = +diminished MODEL (lost capability). Instead, ALL 896 experts stay AVAILABLE — the whole game is WHICH +CACHE LEVEL each expert sits at, set dynamically by demand: + +``` +cache levels (fastest -> slowest, ALL experts reachable at one of them): + VRAM (hot working set) -> RAM (warm) -> FLASH (cold-ready) -> PEER RAM over grid (dynamic host) -> mechanical archival +``` + +- **Full quality always.** A rare expert fires SLOWER (deeper cache level), never MISSING. K3 stays + complete. Speed = cache-hit-rate on the hot working set; capability = full, unconditionally. +- **Dynamic hosting** = the grid distributes experts across nodes and the router finds each at its + current cache level; residency shifts with demand (sentinel-PGO) and with which peers are up. +- **Adding a peer does NOT unlock capability (it's already full) — it moves more experts into a FAST + tier** (a peer's RAM as a cache level). So the model is always whole; the grid makes it faster. + "Works now at full quality, noticeably faster as you plug in nodes" — that is the adoption driver. +- Therefore the compensation-LoRA/prune framing above is DEMOTED to an optional speed mode, NOT the + path. The path is all-experts-available, cached by level, dynamically hosted. diff --git a/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md new file mode 100644 index 0000000000..acdac0b3c8 --- /dev/null +++ b/docs/planning/SELF-CALIBRATION-PROPRIOCEPTION.md @@ -0,0 +1,132 @@ +# Self-calibration — the system finds its own bearings, per activity, continually + +**Status:** principle locked 2026-07-29 (Joel): "The system will find the most natural +fit... anything sub 5 tok/s is probably where it'd reassign to lower intelligence, but +it needs to know how it can do with a devoted coder for example, or the multimodal +kinds... the system has to figure out its own bearings, regularly, and know how to +maximize experiences (activities) so it knows how to scale up and down." + +This is the layer ABOVE serving. GPU expert paging + measured tok/s +([[K3-GPU-EXPERT-PAGING]]) is the SENSORY INPUT; this layer USES it to decide what the +node should BE, per activity, and to scale intelligence up/down as conditions change. + +## The loop (proprioception, not a boot classification) +1. **Probe self, per activity-type, continually.** Regularly run representative probes + — an interactive coding turn, a multimodal describe, a chat turn, an agentic tool + loop — at candidate intelligence tiers, and MEASURE sustained tok/s + first-token + latency + hot_set_hit_rate + quality on the CURRENT hardware / load / grid state. + This is the benchmark-as-self-assessment ([[benchmark-learning-flywheel]]), run as a + background bearing-check, not a one-time catalog resolution. +2. **Capability map.** `(activity_type × intelligence_tier) → {sustained_tok_s, + first_token_ms, quality, needs_grid?}`, continually refreshed. This is what the node + KNOWS about itself: "for a devoted coder I sustain 14 tok/s at tier-A locally; for + multimodal I need a grid peer; chat runs tier-A at 30 tok/s." +3. **Assignment = highest intelligence that clears the activity's experience FLOOR.** + Each activity has its own floor (an interactive coder needs responsiveness — a tok/s + AND first-token floor; a batch multimodal describe tolerates slower). Pick the most + capable model that clears it. **Sub ~5 tok/s ⇒ reassign DOWN** to a smaller/faster + model for that activity — a lower-intelligence experience that stays responsive beats + a frontier one that stalls (the QualityModel already encodes this: a stall zeroes + mean_experience via the critical-faculty gate, [[continuum-substrate-already-built]] + capacity/consumer.rs). +4. **Scale up/down on re-bearings.** Peer joins → capacity up → scale intelligence UP + (or take a harder activity). Load spikes / thermal / peer drops → scale DOWN to hold + experiences above their floors. Same code, different bearing — the SubstrateGovernor + DVFS idea applied to model intelligence, not just clocks. + +## Objective: maximize EXPERIENCES, not tok/s +The thing being maximized is the set + quality of ACTIVITIES the node serves well +(`mean_experience`, the RANSAC score in capacity/mod.rs), never raw throughput. A node +that runs one frontier model at 3 tok/s serves fewer good experiences than one that +runs a tier-down coder at 12 tok/s + multimodal via a peer. The self-calibration picks +the intelligence mix that maximizes served experiences under the live resource vector. + +## What this realizes vs redesigns (wire, don't rebuild) +- **Realizes:** `QualityModel` / `mean_experience` (the experience reward), the resource + negotiation (`grant_all` over [[resource_vector]]), `SystemProfile`/catalog (what + fits). Those exist. +- **The new layer to build:** the CONTINUAL per-activity self-benchmark loop → the + capability map → the intelligence-tier assignment with per-activity experience floors. + It is `catalog = f(system × storage × grid)` made (a) continual and (b) keyed on + MEASURED sustained tok/s per activity, with the ~5 tok/s reassign-down floor. +- **Feeds from:** the paging tok/s + hit-rate meter ([[K3-GPU-EXPERT-PAGING]]) and the + benchmark flywheel. Measured, never guessed — a self-assessment on stale/guessed + numbers reassigns wrong. + +## The experience score is criticality-GATED, activity/goal-contextual, and temporal (Joel 2026-07-29) + +An activity's experience score is NOT an average of sub-scores — it is gated by its +CRITICAL components, shaped by context, and evaluated over a window: + +- **Sub-scores + criticality gate.** Each activity has sub-scores (latency, fps, TTS, + STT, response quality, ...). They combine NON-linearly: a critical component degraded + collapses the WHOLE score even if everything else is green. Losing TTS or STT in a + live video chat totally compromises it — the experience score goes low regardless of + fps. This extends the QualityModel's critical-faculty gate ([[continuum-substrate-already-built]] + capacity/consumer.rs) from "a crash zeroes experience" to "any CRITICAL-component + degradation zeroes experience." The map must know which sub-components are load-bearing + per activity. +- **Activity + goal context.** The score's shape depends on the activity AND the goal: + video chat with 14 personas vs 3 vs 1 has a different resource profile and different + critical set; the goal weights the sub-scores. Same node, different bearing per context. +- **Degrade by criticality, not uniformly.** When latency/fps lags, cut the LEAST-critical + sub-component first (a background persona's avatar fps, a non-speaker's video) to PROTECT + the critical ones (never drop the active speaker's TTS/STT). Shed to preserve the + experience, targeted — not a uniform throttle that clips everything including the + load-bearing parts. + +## Temporal concentration — optimize experience over a WINDOW, not each instant + +The governor must maximize experience over a TIME WINDOW, not instantaneous fairness. It +is correct to briefly PAGE OUT other personas so a hard task gets the smart MoE (e.g. K3) +for a few minutes — the combined windowed experience is HIGHER because the hard problem +gets solved, even though the instantaneous "everyone served equally" metric dipped. A +governor "too worried about satisfying all personas" every instant never lets anyone do +deep work — the exact failure Joel flagged: it refused to even temporarily admit K3 for a +hard coding problem. Add a bounded, reversible CONCENTRATION term to the negotiation +([[resource_vector]] grant_all): a high-value hard task may temporarily concentrate +resources (page out low-priority lanes), on a deadline, then restore. Reversible + +time-boxed = [[restarts-are-commonplace]] applied to attention. + +## Difficulty/failure-driven ESCALATION — scale UP on hard, not only DOWN on scarce + +Intelligence scales UP on detected DIFFICULTY, the dual of the ~5-tok/s reassign-DOWN on +scarcity. Detect thrashing / repeated failure / low quality — a coder that can't solve the +hard problem, an agent looping on a task (**like Claude thrashing on the hf download this +very session** — repeating a failing move, not escalating) — and ESCALATE to a smarter +model for that hard stretch (19B→K3; Opus→Fable), then DE-escalate when it's easy again. +The failure/thrash IS the signal ([[benchmark-learning-flywheel]]): graded failure → +escalate. This makes intelligence assignment two-sided: reassign DOWN when the tier can't +sustain the experience (scarcity), reassign UP when the tier can't SOLVE the task +(difficulty). Both detected from measured experience, both temporary, both reversible. + +## Multi-user + contention: the human's foreground work is the TOP-priority activity (Joel 2026-07-29) + +Eventual goal: run **system-wide (all users)**, a background service serving whoever's on +the box — deferred for now (complexity/safety concerns), but a good eventual test and how +Joel sets up his machines (e.g. his wife's account runs Steam games). The load-bearing +requirement: + +- **Recognize the user + what they're doing.** Detect a foreground GPU-heavy app (a game — + via Steam running / foreground process / a GPU-util spike from a process we don't own) as + a FIRST-CLASS contention signal, not just a number. +- **The human's foreground GPU work is the HIGHEST-priority "activity"; continuum defers to + it, always.** "Not be a problem during outside GPU usage, or at least not interfere, + deprioritize." When the user games, continuum YIELDS: sheds VRAM, deprioritizes/pauses its + lanes, pages experts out, scales intelligence down, or moves work to the grid — so the game + gets the GPU. The human's experience is a load-bearing sub-score; **degrade continuum, + never the human's game.** +- **This is what `capacity/mod.rs` was built for.** `gpu_free_bytes_live` = "free after + external (unowned) load — a game/browser"; the FitPolicy derives grants from live free, so + a game opening (shrink) and closing (grow) already fall out. Joel's scenario adds the two + concrete pieces on top: the SENSOR (foreground GPU app / Steam detection) as an explicit + high-priority unowned-pressure source, and the POLICY (aggressive yield to foreground user + work). Makes continuum a good citizen on a shared/gaming machine — the precondition for the + all-users service. Ties to [[resource_vector]] (measured live free, external subtracted) + and the always-on background-service end state. + +## Floors (initial, per Joel — calibrate from real experience later) +- Interactive coder / agent: target ≥10 tok/s ([[task #30]]); reassign down below ~5. +- Chat: comfortable well above 10; degrade gracefully. +- Multimodal describe (batch-ish): lower tok/s floor; route to grid peer if local can't. +These are RESOLUTION thresholds the node measures itself against — never hard gates. diff --git a/docs/planning/SENTINEL-IN-SUBSTRATE.md b/docs/planning/SENTINEL-IN-SUBSTRATE.md new file mode 100644 index 0000000000..1f67173a7d --- /dev/null +++ b/docs/planning/SENTINEL-IN-SUBSTRATE.md @@ -0,0 +1,112 @@ +# Sentinel-in-the-Substrate — absorbing sentinel-ai's ideas into continuum-core + +**Status:** plan (2026-07-28, BigMama). **Owner lanes:** BigMama (observe/profile, K3 pager seam), joint w/ M5 (serving + dream rhythm). +**Prime directive:** the ideas land as **extensions of existing Rust modules** — no Python at runtime, no separate project. sentinel-ai (the repo) becomes the paper + reference archive. + +## What sentinel-ai is + +`CambrianTech/sentinel-ai` — Experiential Plasticity for transformers (the foundry that produced +`qwen3-coder-30b-a3b-compacted-19b-256k`, the model Sahar serves today). Its ideas: + +1. **Entropy-based utility observation** — attention-head entropy + activation stats as the signal for + what matters to the domain. +2. **Controller feedback** — an observer that closes the loop: observe utility → adjust (gates/prune) → + re-observe. The "sentinel." +3. **Prune ↔ regrow cycles** — capacity is removed when useless and comes BACK when demand shifts; + biological synaptic pruning, not one-way compression. +4. **Forge cycles tied to training** — LoRA-train on domain → prune what didn't matter → retrain; the + architecture co-evolves with experience. +5. **Calibration-aware MoE expert pruning** — profile which experts fire on a corpus, drop the rest + (§4.1.3.4 — exactly the algorithm already ported for the compacted-19b). + +## What the substrate ALREADY has (do not rebuild — wire) + +| Sentinel idea | Existing Rust primitive | Where | +|---|---|---| +| Compaction decisions (ONE formula: `0.8·gate + 0.2·grad`) | `PlasticityModule` + 5 typed commands (`plasticity/analyze·compact·compress·topology·pipeline`) | `modules/plasticity`, `commands/plasticity/` | +| Expert activation observation (serving-side) | `ExpertActivationProfile` + `ServingExpertPager` observe→plan→budget→reconcile loop (K3 pager slice-1, PRs #2018–#2022) | `capacity/` | +| Demand-aligned retention | `EvictionPolicy::DemandAlignedWithRefinedPreference` | `genome/eviction.rs` | +| Sleep/idle rhythm | `dream_consolidation` (memory-side today) | `cognition/dream_consolidation.rs` | +| Gene → pageable artifact | `forge-custodian` bin (trained gene → gguf-lora) | built by `start-server.sh` | +| Attestable recipes | ForgeAlloy (3rd pillar) + the ForgeRecipe-entity sprint (CLAUDE.md §FORGE TEMPLATE) | `forge-alloy/` | + +## The five slices (each lands in an existing module) + +### 1. Live utility profile — PGO from serving traffic *(extends `capacity/`)* +Extend `ExpertActivationProfile` with per-head/per-expert utility harvested from the LIVE llama-server +lane: expert fire counts (already flowing) + **sampled** attention-entropy windows (entropy needs attn +probs — sample N-token windows on a cadence, never per-token; RTOS style: own tick, `watch::Sender` +snapshot, zero hot-path cost via `CaptureSink` Noop default). Output: `LiveUtilizationProfile`, a +genome artifact. **This replaces sentinel-ai's held-out calibration corpus with lived traffic** — the +persona's actual workload IS the calibration set. (The doc'd "sentinel-AI-as-PGO" from +GENOME-FOUNDRY-SENTINEL.md, made concrete.) + +### 2. Controller = a governor consumer *(extends the pager plan loop)* +Sentinel's controller-ANN closes observe→act. V1 is NOT a learned net: the existing pager plan loop + +plasticity formula, with entropy folded in +(`utilization = w₁·gate + w₂·grad + w₃·(1 − entropy_norm)`), driving keep/quantize/prune/page +decisions per expert. The *learned* controller becomes a gene later (trained like any other, paged by +the genome) — same seam, upgraded policy. No new manager/coordinator (CONCURRENCY-STYLE-GUIDE law). + +### 3. In-tree PGO compaction *(feeds `plasticity/compact` — already built)* +`plasticity/compact` consumes slice-1 profiles instead of Python calibration runs → the serving model +is re-forged FROM ITS OWN TRAFFIC, fully in-tree. This retires `forge_model.py`'s MoE-prune leg. +Validation gate before swap: PPL/eval harness on the compacted artifact (catalog swap only on pass). + +### 4. Forge-while-dreaming *(extends `dream_consolidation`)* +Schedule slice-3 passes in the existing dream/idle rhythm: memory consolidates (today) AND weights +compact (new) during sleep; validate on wake; regression → rollback to the pre-compaction genome tier. +Idle GPU at night is foundry time. Same cadence ladder, same quarantine discipline. + +### 5. Regrowth + CLONING = paging, mitosis, heal *(unifies with K3 slice-2)* +Pruned experts/heads are demoted to L4/L5 cold storage, never deleted. When the live profile shifts +(entropy rising, fire-counts on absent experts via router logits, eval guard failing), the pager pages +them BACK; optional LoRA heal via `forge-custodian`. **Sentinel's "regrow" and the K3 expert pager are +the same mechanism** — one delta rule at the weight tier: erase-stale / write-fresh / bounded budget. + +**Cloning (sentinel's head-mitosis, MoE-native):** the other half of experiential plasticity — a +SATURATED high-utility head/expert (entropy shows concentration + overload) is DUPLICATED (+ small +noise, brief LoRA differentiation) into a slot freed by culling. Cull-dead + clone-hot at constant +budget = capacity REALLOCATION, the actual biological analog. The pager's slot model fits exactly: +a clone is just a page-in of a copied artifact into a culled expert's slot, then the router learns to +split traffic during the next heal pass. Per-head/per-expert **quantization levels** ride the same +decision (the plasticity formula's Precision column, driven by live utilization instead of a +calibration pass): hot experts get more bits, cold survivors get fewer — mixed precision as a +continuous dial, not a per-model constant. + +### 6. The uber skip path — depth elasticity via K3's AttnRes *(rides the fork's K3 ops)* +sentinel-ai tried U-Net-style skip connections between layers and its own DEBUGGING_NOTES record the +outcome: **instability, disabled**. Raw dense skips across transformer depth destabilize training. +K3 ships the STABLE formulation of the same idea at 1.45T scale: **AttnRes** — every +`attn_res_block_size` layers checkpoint the residual stream, and each layer applies a LEARNED, +SOFTMAX-NORMALIZED attention over those block checkpoints (`self_attention_res_norm/_proj` + +`_apply_attn_res`). A principled residual highway: normalized (softmax over blocks), learned (scored +per token), cheap (one scalar proj per block). We are already implementing this op in our llama.cpp +fork for K3 — the same graph machinery then serves forged models. + +Plasticity payoffs, in order of arrival: +1. **Safe depth culling** — with a highway in place, whole low-utility BLOCKS can be culled/skipped and + information routes around them (the redundancy that makes biological pruning survivable). +2. **Inference-time depth elasticity** — the controller (slice 2) can gate blocks off per-request when + their marginal utility is low (entropy signal): adaptive compute on the depth axis, the complement + of MoE's width axis. +3. **Generalization** — a skip highway trained with stochastic block-drop is an implicit ensemble over + depths (stochastic-depth literature); the "uber path" generalizes better on inputs where the deep + specialized path overfits. Forge cycles can GRAFT a highway onto highway-less models (train only + res_norm/res_proj per block — LoRA-heal-sized, not a retrain). + +## What stays out (for now) +- Head-level *regrowth inside a live CUDA graph* (llama.cpp static graph can't resize; regrowth is + artifact-swap granularity until the vendored-fork tensor-write seam (K3 slice-2B) proves out). +- Training loops in-core beyond LoRA heal — heavy fine-tunes stay on the unsloth grid layer (M5's + node) per the training-layer plan; the substrate *schedules* them, never embeds Python. + +## Sequencing +1. Slice 1 (BigMama — extends my K3 observer; independent, testable with `HeuristicInferenceAdapter`). +2. Slice 3 wiring (mostly exists; joins 1's output to `plasticity/compact`). +3. Slice 2 policy fold-in (small, in the plan loop). +4. Slice 4 dream hook (joint w/ M5 — her dream_consolidation lead). +5. Slice 5 rides K3 slice-2's A/B decision (upload_expert now; true K-slot paging gated on measured numbers). + +Retirement: `tools/scripts/compaction` (Python) feature-freezes at slice-3 parity — kept as the +reference implementation the Rust port is validated against (same inputs → same kept-set, then better). diff --git a/protocol/typescript/airc/AircRealtimeSchema.ts b/protocol/typescript/airc/AircRealtimeSchema.ts index f5040ab98c..a8efd11857 100644 --- a/protocol/typescript/airc/AircRealtimeSchema.ts +++ b/protocol/typescript/airc/AircRealtimeSchema.ts @@ -3,4 +3,4 @@ /** * Existing Continuum schema carried by an AIRC realtime envelope. */ -export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity"; +export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity" | "grid_residency"; diff --git a/tools/scripts/generated/manifest.windows.ps1 b/tools/scripts/generated/manifest.windows.ps1 index 49d592b79d..453ce89751 100644 --- a/tools/scripts/generated/manifest.windows.ps1 +++ b/tools/scripts/generated/manifest.windows.ps1 @@ -13,8 +13,8 @@ $script:ContinuumManifest = [ordered]@{ 'airc-firewall' = @{ order = 27; tier = 0; flags = @('grid'); applies = 'has-airc'; accept = 'netsh advfirewall firewall show rule name="airc daemon inbound (continuum grid)"'; source = @{ type = 'command'; run = 'New-NetFirewallRule -DisplayName ''airc daemon inbound (continuum grid)'' -Direction Inbound -Action Allow -Profile Any' } } 'manifest-gen' = @{ order = 28; tier = 3; flags = @('dev'); accept = 'cargo run -q -p manifest-gen -- --check'; source = @{ type = 'command'; run = 'cargo run -q -p manifest-gen' } } 'msvc' = @{ order = 30; tier = 3; flags = @('dev'); accept = 'vswhere -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath'; source = @{ type = 'winget'; id = 'Microsoft.VisualStudio.2022.BuildTools'; override = '--wait --quiet --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' } } - 'cmake' = @{ order = 40; tier = 3; flags = @('dev'); accept = 'cmake --version'; source = @{ type = 'archive'; url = 'https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-windows-x86_64.zip'; version = '3.30.5'; sha256 = '5ab6e1faf20256ee4f04886597e8b6c3b1bd1297b58a68a58511af013710004b'; extract = 'strip-top-dir' } } - 'llvm-libclang' = @{ order = 50; tier = 3; flags = @('dev'); accept = 'test-path ~/.continuum/tools/llvm/bin/libclang.dll'; source = @{ type = 'archive'; url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-pc-windows-msvc.tar.xz'; version = '18.1.8'; sha256 = '22c5907db053026cc2a8ff96d21c0f642a90d24d66c23c6d28ee7b1d572b82e8'; extract = 'members:*/bin/libclang.dll,*/lib/clang/*' } } + 'cmake' = @{ order = 40; tier = 3; flags = @('dev'); accept = 'cmake --version'; source = @{ type = 'archive'; url = 'https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-windows-x86_64.zip'; version = '3.30.5'; sha256 = '5ab6e1faf20256ee4f04886597e8b6c3b1bd1297b58a68a58511af013710004b'; extract = 'strip-top-dir' }; runtime_path = @('~/.continuum/tools/cmake/bin') } + 'llvm-libclang' = @{ order = 50; tier = 3; flags = @('dev'); accept = 'test-path ~/.continuum/tools/llvm/bin/libclang.dll'; source = @{ type = 'archive'; url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-pc-windows-msvc.tar.xz'; version = '18.1.8'; sha256 = '22c5907db053026cc2a8ff96d21c0f642a90d24d66c23c6d28ee7b1d572b82e8'; extract = 'members:*/bin/libclang.dll,*/lib/clang/*' }; runtime_path = @('~/.continuum/tools/llvm/bin') } 'cuda' = @{ order = 60; tier = 3; flags = @('dev'); applies = 'has-nvidia'; accept = 'nvcc --version >= 12.8'; source = @{ type = 'redist'; version = '12.9.1'; manifest = 'https://developer.download.nvidia.com/compute/cuda/redist/redistrib_12.9.1.json'; components = @('cuda_nvcc', 'cuda_cudart', 'libcublas', 'libcurand', 'cuda_nvrtc', 'cuda_cccl') }; runtime_path = @('~/.continuum/cuda-*/Library/bin') } 'build-core' = @{ order = 90; tier = 3; flags = @('dev'); accept = 'continuum-core-server.exe boots past the GPU-detection gate on the target device'; build = @{ features = 'cuda,load-dynamic-ort'; profile = 'release'; crt = 'static'; cmake_generator = 'Visual Studio 17 2022'; cuda_arch = '120'; msvc_host = 'vs2022' } } 'run' = @{ order = 100; tier = 3; accept = 'continuum-core-server binary present + serves TCP 9100' } diff --git a/tools/scripts/generated/manifest.windows.sh b/tools/scripts/generated/manifest.windows.sh index 12d66671e4..90ef542ad6 100644 --- a/tools/scripts/generated/manifest.windows.sh +++ b/tools/scripts/generated/manifest.windows.sh @@ -27,4 +27,4 @@ declare -A MOD_ARGS=() declare -A MOD_RUN=( ['gh-auth']='gh auth login --hostname github.com --git-protocol https --web' ['airc-firewall']='New-NetFirewallRule -DisplayName '\''airc daemon inbound (continuum grid)'\'' -Direction Inbound -Action Allow -Profile Any' ['manifest-gen']='cargo run -q -p manifest-gen' ) declare -A MOD_BUILD_FEATURES=( ['build-core']='cuda,load-dynamic-ort' ) declare -A MOD_BUILD_PROFILE=( ['build-core']='release' ) -declare -A MOD_RUNTIME_PATH=( ['cuda']='~/.continuum/cuda-*/Library/bin' ) +declare -A MOD_RUNTIME_PATH=( ['cmake']='~/.continuum/tools/cmake/bin' ['llvm-libclang']='~/.continuum/tools/llvm/bin' ['cuda']='~/.continuum/cuda-*/Library/bin' ) diff --git a/tools/scripts/install-llama-server.sh b/tools/scripts/install-llama-server.sh index 68e75d866a..b667c37785 100755 --- a/tools/scripts/install-llama-server.sh +++ b/tools/scripts/install-llama-server.sh @@ -82,9 +82,17 @@ else MINGW*|MSYS*|CYGWIN*) EXE=".exe" if command -v nvcc >/dev/null 2>&1 || [ -x "$CONTINUUM_HOME/cuda-toolkit/bin/nvcc.exe" ]; then - BACKEND="cuda"; WIN_CUDA=1 + BACKEND="cuda-static"; WIN_CUDA=1 # arch=native → build for THIS machine's GPU (portable; NOT a hardcoded sm_120). - BACKEND_DEFS=(-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native) + # BUILD_SHARED_LIBS=OFF is LOAD-BEARING (2026-07-28, BigMama): the shared build's + # ggml-cuda.dll fails CUDA init at runtime ("no usable GPU found") while passing + # --version — so a GPU-blind engine got stamped verified-good and every generation + # 500'd (the serving daemon's decode-ready probe then never admits personas). The + # static build (ggml linked into the exe) initializes CUDA correctly on the same + # box. Backend renamed cuda→cuda-static so existing broken installs fail the stamp + # check and rebuild on next run. CUDA RUNTIME (cudart/cublas dlls) stays dynamic + # via the manifest runtime_path — only ggml/llama are static. + BACKEND_DEFS=(-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native -DBUILD_SHARED_LIBS=OFF) fi # non-NVIDIA Windows falls through to the CPU build (llama.cpp CPU works on Win). ;; @@ -225,6 +233,20 @@ if [ "$verify_ok" -ne 1 ]; then echo "✗ FATAL: built llama-server does not run (--version failed after retry)." >&2 exit 1 fi + +# On Windows+CUDA, --version is NOT enough: the GPU-blind shared build passed it while +# unable to init CUDA (the 2026-07-28 regression). Require the engine to actually SEE a +# CUDA device before stamping — this is the difference between "binary runs" and "binary +# can serve". A build that can't see the GPU is removed, not blessed. +if [ "$WIN_CUDA" -eq 1 ]; then + if ! "$INSTALL_BIN" --list-devices 2>&1 | grep -q "CUDA0"; then + rm -f "$INSTALL_BIN" + echo "✗ FATAL: built llama-server cannot see a CUDA device (--list-devices has no CUDA0)." >&2 + echo " A GPU-blind engine must never be stamped — it serves decode-dead lanes." >&2 + exit 1 + fi + echo "✓ CUDA verify: engine sees CUDA0" >&2 +fi echo "$STAMP_WANT" > "$STAMP_FILE" # stamp LAST — only a verified-good binary is blessed echo "✓ llama-server installed: $INSTALL_BIN ($STAMP_WANT)" >&2 diff --git a/tools/scripts/install-manifest.toml b/tools/scripts/install-manifest.toml index 8648882a31..968d7c4e8c 100644 --- a/tools/scripts/install-manifest.toml +++ b/tools/scripts/install-manifest.toml @@ -159,6 +159,12 @@ formula = "cmake" [module.sources.linux] type = "apt" # run-verify pending on a linux node package = "cmake" +[module.runtime_path] +# The Windows archive installs to ~/.continuum/tools/cmake; its bin/ MUST be on +# PATH at BUILD time — the llama crate's build.rs shells out to `cmake` to compile +# vendored llama.cpp, and nvcc-free directml builds still need it. brew/apt already +# put cmake on PATH, so this is windows-only. [[windows-build-env-drift]] +windows = ["~/.continuum/tools/cmake/bin"] [[module]] id = "llvm-libclang" @@ -183,6 +189,11 @@ accept_macos = "test -f /Library/Developer/CommandLineTools/usr/lib/libclang.dyl type = "apt" # run-verify pending on a linux node package = "libclang-dev" accept_linux = "test -f /usr/lib/llvm-18/lib/libclang.so.1 || ldconfig -p | grep -q libclang" +[module.runtime_path] +# The Windows archive installs libclang.dll to ~/.continuum/tools/llvm/bin; on PATH +# so bindgen (llama crate) can load it at build time. mac/linux resolve libclang via +# system paths. [[windows-build-env-drift]] +windows = ["~/.continuum/tools/llvm/bin"] [[module]] id = "cuda" diff --git a/tools/scripts/shared/cargo-features.sh b/tools/scripts/shared/cargo-features.sh index e9615ebb9a..3089dfa6a6 100644 --- a/tools/scripts/shared/cargo-features.sh +++ b/tools/scripts/shared/cargo-features.sh @@ -59,7 +59,13 @@ case "$(uname -s)" in # top if Nvidia is present so ORT picks CUDA first (faster) + # DirectML stays as a co-listed EP for non-CUDA-supported ops. CARGO_GPU_FEATURES="--features directml" - if command -v nvidia-smi &>/dev/null; then + # candle-cuda's affine.cu compiles via nvcc, which needs the MSVC host + # compiler cl.exe on PATH (an active vcvars env). Only add cuda when cl.exe + # is actually reachable; otherwise nvcc fatals "Cannot find compiler + # 'cl.exe'" and the ENTIRE core build dies. directml needs no kernel + # compilation, so it stays as the universal Windows GPU EP and the build + # degrades gracefully instead of hard-failing. [[windows-build-env-drift]] + if command -v nvidia-smi &>/dev/null && command -v cl.exe &>/dev/null; then CARGO_GPU_FEATURES="--features cuda,directml" fi ;; diff --git a/tools/scripts/start-server.sh b/tools/scripts/start-server.sh index 819985bfe3..a9d4209ac4 100755 --- a/tools/scripts/start-server.sh +++ b/tools/scripts/start-server.sh @@ -101,6 +101,63 @@ if [ -f "$_mf_runtime" ]; then fi fi +# ── Windows-CUDA build env (2026-07-28, BigMama) ───────────────────── +# The core's `cuda` feature needs the MSVC toolchain visible to cargo's build +# scripts (candle-kernels runs nvcc→cl.exe; the llama crate links cuda.lib). +# On a stock Git-Bash shell NONE of that is on PATH, so cargo-features.sh +# (below) silently degrades to directml-only — and the core then mis-detects +# a 32GB card as ~4GB (no detect_cuda) and serves a toy model. Establish the +# env HERE, before feature detection, so a fresh Windows+NVIDIA box gets the +# real build with zero hand steps: +# 1. cl.exe absent but VS2022 BuildTools installed → re-exec this script +# once through vcvars64 (same cmd/.bat bridge install-llama-server.sh +# uses; VS2022 pinned — nvcc rejects VS18/14.5x toolsets). +# 2. MSVC link.exe must BEAT Git's /usr/bin/link.exe (coreutils) or every +# build-script link dies — prepend the cl.exe dir to PATH. +# 3. cuda.lib/cublas.lib etc live in the toolkit's lib/x64 which vcvars +# does NOT add — prepend to LIB for link.exe (LNK1181 otherwise). +# 4. bindgen reads LIBCLANG_PATH (the env var — PATH is not enough). +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + if command -v nvidia-smi >/dev/null 2>&1 && ! command -v cl.exe >/dev/null 2>&1 \ + && [ -z "${CONTINUUM_MSVC_REENTER:-}" ]; then + _vswhere="/c/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" + _vs_path="" + [ -x "$_vswhere" ] && _vs_path="$("$_vswhere" -version "[17.0,18.0)" -products '*' \ + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \ + -property installationPath 2>/dev/null | head -1)" + if [ -n "$_vs_path" ]; then + echo "→ NVIDIA GPU + VS2022 present but cl.exe not on PATH — re-entering via vcvars64 for the CUDA build" >&2 + _reenter_bat="$(mktemp --suffix=.bat 2>/dev/null || echo "${TMPDIR:-/tmp}/continuum-msvc-reenter.bat")" + _win_bash="$(cygpath -w "$(command -v bash)")" + _script_unix="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" + { + echo "@echo off" + echo "call \"$(cygpath -w "$_vs_path")\\VC\\Auxiliary\\Build\\vcvars64.bat\" >nul || exit /b 1" + echo "set CONTINUUM_MSVC_REENTER=1" + echo "\"$_win_bash\" \"$_script_unix\" || exit /b 1" + } > "$_reenter_bat" + exec cmd //c "$(cygpath -w "$_reenter_bat")" + else + echo "⚠ NVIDIA GPU present but VS2022 BuildTools not found — core builds directml-only" >&2 + echo " (install-manifest 'msvc' module provisions it; cuda serving needs it)" >&2 + fi + fi + if command -v cl.exe >/dev/null 2>&1; then + # (2) MSVC linker precedence over Git's coreutils link.exe. + _cl_dir="$(dirname "$(command -v cl.exe)")" + case ":$PATH:" in "$_cl_dir":*) ;; *) export PATH="$_cl_dir:$PATH" ;; esac + # (3) CUDA import libs onto LIB (version-agnostic glob, mirrors runtime_path). + for _cuda_lib in "$HOME/.continuum"/cuda-*/Library/lib/x64 "$HOME/.continuum"/cuda-*/lib/x64; do + [ -d "$_cuda_lib" ] && export LIB="$(cygpath -w "$_cuda_lib");${LIB:-}" + done + # (4) bindgen's libclang (manifest llvm-libclang install location). + [ -f "$HOME/.continuum/tools/llvm/bin/libclang.dll" ] \ + && export LIBCLANG_PATH="$HOME/.continuum/tools/llvm/bin" + fi + ;; +esac + # ── Per-platform feature flags ─────────────────────────────────────── # Mac Intel can't use Metal (task #131 — ggml_metal_device_init hangs on # Intel + AMD discrete). Force mac-cpu-only on Intel Mac. @@ -182,16 +239,34 @@ echo "✓ llama-server: $(command -v llama-server) — the engine we own & launc # dies and would keep holding the port + GPU, so stop that too. # The core's fresh llama-server is launched afterward by the serving daemon, on a # port it SCANS for — so this is GPU/excision hygiene, not a correctness gate. -if pgrep -f 'studio run' >/dev/null 2>&1; then - echo " stopping excised Unsloth Studio (freeing GPU for the core's engine)" >&2 - pkill -f 'studio run' 2>/dev/null || true -fi -if pgrep -f 'llama-server' >/dev/null 2>&1; then - echo " clearing orphaned llama-server backend(s) so the core owns the engine" >&2 - pkill -f 'llama-server' 2>/dev/null || true - # Give the OS a moment to release the listening socket before the core binds. - sleep 1 -fi +# Windows: pgrep/pkill are SILENT NO-OPS against native exes (same class as the +# stop_existing_core fix above) — orphaned engines survived every launch and held +# the canonical serving port, wedging the daemon's fresh-claim reclaim forever +# (observed 2026-07-28: pinned model stuck ready=false behind a dead core's +# llama-server). Use tasklist/taskkill by image name. +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + for _img in llama-server.exe llama-server-static.exe; do + if tasklist 2>/dev/null | grep -qi "$_img"; then + echo " clearing orphaned $_img so the core owns the engine" >&2 + taskkill //F //IM "$_img" >/dev/null 2>&1 || true + fi + done + sleep 1 + ;; + *) + if pgrep -f 'studio run' >/dev/null 2>&1; then + echo " stopping excised Unsloth Studio (freeing GPU for the core's engine)" >&2 + pkill -f 'studio run' 2>/dev/null || true + fi + if pgrep -f 'llama-server' >/dev/null 2>&1; then + echo " clearing orphaned llama-server backend(s) so the core owns the engine" >&2 + pkill -f 'llama-server' 2>/dev/null || true + # Give the OS a moment to release the listening socket before the core binds. + sleep 1 + fi + ;; +esac # ── Airc context ───────────────────────────────────────────────────── # Substrate auto-discovers airc daemon socket via `airc ipc-endpoint` @@ -245,22 +320,46 @@ CONTINUUM_SOCKET="${CONTINUUM_SOCKET:-/tmp/continuum-core.sock}" # Called AFTER the build (below) so downtime is ~0: the new binary is ready, we # stop the old, and exec immediately. stop_existing_core() { - local pids - pids="$(pgrep -f "continuum-core-server" 2>/dev/null | grep -v "^$$\$" || true)" - if [ -n "$pids" ]; then - echo "▶ stopping existing core (pids: $(echo $pids | tr '\n' ' '))" - # shellcheck disable=SC2086 - kill -TERM $pids 2>/dev/null || true - for _ in $(seq 1 15); do - pgrep -f "continuum-core-server" >/dev/null 2>&1 || break - sleep 1 - done - if pgrep -f "continuum-core-server" >/dev/null 2>&1; then - echo " graceful stop timed out — SIGKILL" - pkill -9 -f "continuum-core-server" 2>/dev/null || true - sleep 1 - fi - fi + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + # POSIX signals are a SILENT NO-OP against native Windows exes from + # Git-Bash: pgrep -f matches nothing and kill/pkill can't touch them, so + # this block never stopped anything on Windows — every `npm start` boot + # then died fighting the immortal old core for its ports (observed + # 2026-07-28 BigMama: an 8:22AM core survived three "restarts"; each new + # boot failed binding 0.0.0.0:7117 and exited, masked as "leaving the + # running core untouched"). Use tasklist/taskkill — the native tools. + if tasklist 2>/dev/null | grep -qi "continuum-core-server.exe"; then + echo "▶ stopping existing core (taskkill)" + # No graceful console signal exists for a detached native service from + # here (WM_CLOSE is ignored by console apps); the core's state is + # crash-safe by design ([[no-fallbacks-ever]] boot contract), so /F. + taskkill //F //IM continuum-core-server.exe >/dev/null 2>&1 || true + for _ in $(seq 1 10); do + tasklist 2>/dev/null | grep -qi "continuum-core-server.exe" || break + sleep 1 + done + fi + ;; + *) + local pids + pids="$(pgrep -f "continuum-core-server" 2>/dev/null | grep -v "^$$\$" || true)" + if [ -n "$pids" ]; then + echo "▶ stopping existing core (pids: $(echo $pids | tr '\n' ' '))" + # shellcheck disable=SC2086 + kill -TERM $pids 2>/dev/null || true + for _ in $(seq 1 15); do + pgrep -f "continuum-core-server" >/dev/null 2>&1 || break + sleep 1 + done + if pgrep -f "continuum-core-server" >/dev/null 2>&1; then + echo " graceful stop timed out — SIGKILL" + pkill -9 -f "continuum-core-server" 2>/dev/null || true + sleep 1 + fi + fi + ;; + esac rm -f "$CONTINUUM_SOCKET" } @@ -321,8 +420,15 @@ echo "▶ building forge-custodian (Rust gguf-lora export sidecar)" cargo build --manifest-path "$CORE_MANIFEST" --bin forge-custodian $PROFILE_FLAG $CONTINUUM_FEATURES \ || echo "⚠ forge-custodian build failed — genome gene-conversion unavailable (core still launches)" >&2 -# Build the server binary BEFORE stopping the old core, so the running core keeps -# serving through the (cached, fast) compile and downtime is ~0. +# Stop the running core BEFORE building the server bin. Restarts are commonplace +# BY DESIGN ([[restarts-are-commonplace]], Joel 2026-07-28): a stopped node is a +# sleeping citizen — the grid absorbs the capacity dip, personas resume on boot. +# Zero-downtime build-before-stop was the OLD value here and it bought real bugs: +# on Windows an EXECUTING exe cannot be overwritten (os error 5), so building +# while the old core ran either failed the link or needed rename-aside tricks. +# Stop-then-build is simpler, uniform across platforms, and optimizes the thing +# we actually value: a clean fast restart, not uptime. +stop_existing_core echo "▶ building continuum-core-server" cargo build --manifest-path "$CORE_MANIFEST" --bin continuum-core-server $PROFILE_FLAG $CONTINUUM_FEATURES \ || { echo "✗ FATAL: continuum-core-server build failed — leaving the running core untouched" >&2; exit 1; }