From 3152989400372d27b834ca8936aa7c6169d7f5e7 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 09:49:44 -0500 Subject: [PATCH 01/19] =?UTF-8?q?feat(capacity):=20expert=5Fobserve=20harn?= =?UTF-8?q?ess=20=E2=80=94=20glass-box=20LIVE=20MoE=20expert=20routing=20(?= =?UTF-8?q?#230/#229)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs a GGUF MoE through the core/llama FFI with LiveExpertObserver attached (the existing cb_eval → ffn_moe_topk seam), generates N tokens to drive real routing, then dumps the model-intrinsic affinity: hot/cold expert distribution + co-occurrence + prefetch candidates. That affinity is the INPUT to expert prefetch (#227), grid placement (#180), compaction, and distillation (#233). Uses the in-process FFI (not the live llama-server lane) because affinity is model-intrinsic — valid data, zero risk to live serving. First run (Qwen3-Coder-30B-A3B, 96 tokens, Metal): 43,640 activations, 6116/6144 expert-slots fired (~99.5%), hottest expert only 0.20% (~12x uniform), 6092 colder share 95.8%. FINDING: for an 8/128 (6.25%-active) MoE, activation over a generation is BROAD, not tiny-hot — so the paging win is the tier ladder + affinity placement, not a small resident set. Prefetch predictor returned 0 candidates over 96 tokens (needs more data). K3 (1.8% active) should be far more concentrated — same harness will quantify it when weights land. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/bin/expert_observe.rs | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 core/continuum-core/src/bin/expert_observe.rs diff --git a/core/continuum-core/src/bin/expert_observe.rs b/core/continuum-core/src/bin/expert_observe.rs new file mode 100644 index 0000000000..99ceec65bd --- /dev/null +++ b/core/continuum-core/src/bin/expert_observe.rs @@ -0,0 +1,114 @@ +//! expert_observe — glass-box the LIVE MoE expert routing (#230 / #229). +//! +//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached (the +//! already-built `cb_eval` → `ffn_moe_topk` seam in `core/llama/src/safe.rs`), generates +//! N tokens to drive REAL routing, then dumps the affinity data — hot/cold expert +//! distribution + co-occurrence + prefetch prediction. That affinity is the INPUT to +//! expert prefetch (#227), grid placement (#180), compaction, and distillation (#233). +//! +//! WHY this and not the live llama-server path: expert affinity is MODEL-INTRINSIC (which +//! experts co-fire for given inputs), so an in-process FFI run gathers valid data without +//! touching the live-persona serving lane. (The live llama-server path would need a +//! separate fork patch to emit routing; this harness needs neither.) +//! +//! Usage: +//! cargo run -p continuum-core --features metal,accelerate --bin expert_observe -- [n_tokens] [prompt] + +use std::path::PathBuf; +use std::sync::Arc; + +use continuum_core::capacity::expert_observer::LiveExpertObserver; +use llama::{Batch, ContextParams, ExpertObserver, Model, ModelParams, Sampler}; + +fn main() { + let args: Vec = std::env::args().collect(); + let model_path = args + .get(1) + .expect("usage: expert_observe [n_tokens] [prompt]"); + let n_tokens: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(96); + let prompt = args.get(3).map(|s| s.as_str()).unwrap_or( + "Write a Rust function to reverse a string, then explain how it works step by step:\n", + ); + + // The observer is the sink: `cb_eval` calls `observe(layer, selected_experts, n_used)` + // per MoE layer per token from inside the compute thread. + let observer = LiveExpertObserver::new(); + + let model = Model::load( + PathBuf::from(model_path), + ModelParams { + n_gpu_layers: -1, + use_mmap: true, + }, + ) + .expect("load model"); + println!("Loaded {model_path} (vocab={})", model.n_vocab()); + + let mut ctx = model + .new_context(ContextParams { + n_ctx: 4096, + n_batch: 512, + n_seq_max: 1, + expert_observer: Some(observer.clone() as Arc), + ..Default::default() + }) + .expect("context"); + + // Prefill. + let prompt_tokens = model.tokenize(prompt, true, false).expect("tokenize"); + let mut batch = Batch::allocated(512, 1); + let last = (prompt_tokens.len() - 1) as i32; + for (i, tok) in prompt_tokens.iter().enumerate() { + batch.push(*tok, i as i32, &[0], i as i32 == last); + } + ctx.decode(&batch).expect("prefill decode"); + + // Generate — every decoded token drives the router → observer tallies real selections. + let mut sampler = Sampler::greedy(); + let mut n_cur = batch.n_tokens(); + let mut n_decoded = 0usize; + for _ in 0..n_tokens { + let token = sampler.sample(&ctx, -1); + if model.is_eog_token(token) { + break; + } + batch.clear(); + batch.push(token, n_cur, &[0], true); + ctx.decode(&batch).expect("gen decode"); + n_cur += 1; + n_decoded += 1; + } + + // Dump the affinity — the fuel for the whole optimization catalog. + let total = observer.total_hits(); + let hits = observer.snapshot_hits(); + let (_seen, cooccur) = observer.snapshot_cooccurrence(); + let predicted = observer.predicted(); + + println!("\n=== EXPERT AFFINITY (model-intrinsic, {n_decoded} tokens observed) ==="); + println!("total expert activations : {total}"); + println!("distinct experts fired : {}", hits.len()); + println!("co-occurring pairs seen : {}", cooccur.len()); + println!("prefetch candidates : {} experts", predicted.len()); + + let mut ranked: Vec<(String, u64)> = hits + .iter() + .map(|(k, v)| (format!("{k:?}"), *v)) + .collect(); + ranked.sort_by(|a, b| b.1.cmp(&a.1)); + let show = ranked.len().min(24); + println!("\ntop {show} hottest experts (of {} fired):", hits.len()); + for (id, h) in ranked.iter().take(show) { + let pct = if total > 0 { 100.0 * *h as f64 / total as f64 } else { 0.0 }; + println!(" {id:<28} {h:>8} ({pct:.2}%)"); + } + // The tail: how concentrated is activation? (the paging headroom) + if ranked.len() > show { + let tail: u64 = ranked.iter().skip(show).map(|(_, h)| *h).sum(); + let tail_pct = if total > 0 { 100.0 * tail as f64 / total as f64 } else { 0.0 }; + println!( + " ... {} colder experts share the remaining {tail_pct:.2}% (the page-out tail)", + ranked.len() - show + ); + } +} From afef13294034f7f0b6bde6d4be92774790c4703a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 12:37:00 -0500 Subject: [PATCH 02/19] fix(chat): ChatModule executor fails loud per-request, not process panic (#201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ChatModule::executor()` did `.cloned().expect(...)` — a hard panic if a `chat/poll`, `chat/send`, or `persist_posted` landed before `start_server` called `install_executor_on_all` (a boot race). Panicking there SIGABRTs the whole core and takes every other module down with it, for a per-request contract violation that only concerns that one request. Convert `executor()` to `Result, String>` returning the SAME loud, contract-naming message, and `?`-propagate it in the 3 callers (all already `Result<_, String>`). Faithful to [[no-fallbacks-ever]] — still loud, still names `install_executor_on_all`, no silent default — while satisfying #26 (faculties degrade, never panic): a command that races boot fails loudly to its caller instead of crashing the process. Regression test: a pre-install `poll()` returns the loud error naming the contract instead of panicking. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/modules/chat/mod.rs | 49 +++++++++++++++------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/core/continuum-core/src/modules/chat/mod.rs b/core/continuum-core/src/modules/chat/mod.rs index 09998d01df..9758d62d38 100644 --- a/core/continuum-core/src/modules/chat/mod.rs +++ b/core/continuum-core/src/modules/chat/mod.rs @@ -121,18 +121,23 @@ impl ChatModule { Self { executor_slot } } - /// Resolve the executor for the current call. Panics if the - /// executor was never installed — that's a boot ordering bug - /// (`start_server` must call `install_executor_on_all` BEFORE - /// any chat command can dispatch). Per [[no-fallbacks-ever]]: - /// the panic message names the contract so the operator sees - /// the actual problem. - fn executor(&self) -> Arc { - self.executor_slot.cloned().expect( + /// Resolve the executor for the current call. Returns a loud, + /// contract-naming error if the executor was never installed — a + /// boot-ordering bug (`start_server` must call + /// `install_executor_on_all` BEFORE any chat command dispatches). + /// Per [[no-fallbacks-ever]] the message still names the contract + /// so the operator sees the real problem — but the blast radius is + /// THIS request, not the whole core: a command that races boot + /// fails loudly to its caller instead of `.expect()`-panicking the + /// process and taking every other module down with it + /// (#201 boot-race / #26 faculties degrade, never panic). + fn executor(&self) -> Result, String> { + self.executor_slot.cloned().ok_or_else(|| { "ChatModule: CommandExecutor not installed — \ start_server must call install_executor_on_all \ - before any chat command can dispatch (task #224)", - ) + before any chat command can dispatch (task #224)" + .to_string() + }) } /// `chat/poll` — return recent messages, optionally filtered by @@ -151,7 +156,7 @@ impl ChatModule { /// 5. Normalize back to chronological order for display regardless /// of query direction. pub async fn poll(&self, params: ChatPollParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let limit = params.limit.unwrap_or(DEFAULT_POLL_LIMIT); // ── Phase 1: resolve the anchor timestamp if the caller @@ -284,7 +289,7 @@ impl ChatModule { /// could be the dedup id) but the design conversation is its /// own scope. pub async fn send(&self, params: ChatSendParams) -> Result { - let executor = self.executor(); + let executor = self.executor()?; let message_id = Uuid::new_v4(); let now_ms = now_ms(); let now_iso = now_iso(now_ms); @@ -416,7 +421,7 @@ impl ChatModule { /// EXPECTED duplicate, never an error. A persona line's id is airc's /// `event_id` (stable across replay), so restarts can't double-store either. pub async fn persist_posted(&self, payload: Value) -> Result<(), String> { - let executor = self.executor(); + let executor = self.executor()?; let field = |k: &str| -> Result { payload .get(k) @@ -995,6 +1000,24 @@ mod tests { assert!(result.after_message_id.is_none()); } + // what this catches: #201 boot-race — a chat command that arrives BEFORE + // install_executor_on_all runs fails LOUD to its caller (naming the contract), + // instead of .expect()-panicking the whole core and taking every other module + // down with it. Regression for the .expect() → Result conversion in executor(). + #[tokio::test] + async fn command_before_executor_installed_fails_loud_not_panics() { + let chat = ChatModule::new(); // executor slot deliberately NOT installed + let err = chat + .poll(ChatPollParams::default()) + .await + .expect_err("poll before install_executor_on_all must fail, not panic"); + assert!(err.contains("not installed"), "loud contract error, got: {err}"); + assert!( + err.contains("install_executor_on_all"), + "error names the contract so the operator sees the real problem: {err}" + ); + } + // ── chat/poll: latest-N path (no anchor) ────────────────────────── #[tokio::test] From ca95b75c8cc67f5f7a2e24cae6ddb82adedb36cb Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 12:39:58 -0500 Subject: [PATCH 03/19] test(cognition): align tool-surface test name+doc with the deleted shrink-cliff (#212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test body was already updated (74acbb36c / #206) to pin the ABSENCE of the tight-window "discovery-pair only" shrink cliff — it asserts the full native surface (edit_file/bash/grep) is never window-amputated on an 8192 window. But the test NAME (`..._is_a_category_index_plus_discovery_pair`) and its header comment still described the DELETED behavior ("the per-turn tool PAYLOAD is the two-tool DISCOVERY PAIR"), contradicting the code below them. Rename to `tool_surface_is_a_category_index_plus_the_unamputated_native_surface` and rewrite the header to state what the test actually pins: the system prompt carries a category index, the full native surface rides beside it un-amputated, and a regression means either the ~150-schema dump or the amputation cliff came back. Doc/name only — body and assertions unchanged, still green. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/llm_deliberation_faculty.rs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs index f5249b4c08..eb91e0a82b 100644 --- a/core/continuum-core/src/cognition/llm_deliberation_faculty.rs +++ b/core/continuum-core/src/cognition/llm_deliberation_faculty.rs @@ -1734,19 +1734,20 @@ mod tests { ); } - // what this catches: progressive disclosure — the per-turn tool PAYLOAD is the - // two-tool DISCOVERY PAIR (`commands/list` + `commands/help`), not the whole - // authorized registry, and the system prompt carries only a CATEGORY INDEX, not - // every tool. The old dump injected ~150 full schemas / one-liners (~4–5k tokens) - // into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute. Now the - // surface is a tiny category index inside the system prompt + the two-tool native - // offering, so even a huge tool set leaves system + user + the offered tools well - // within the served window. Invariant: the category index (not tool names) rides - // the system prompt, the native offering is exactly the discovery pair, and the - // whole prompt + its tools + reserve fit the window. A regression means the dump - // came back. + // what this catches: progressive disclosure — the system prompt carries only a + // CATEGORY INDEX (names, not the ~150 full schemas), and the native offering is the + // FULL authorized coding surface, which is NEVER window-amputated. Two failure modes + // this pins the absence of: (1) the old dump that injected ~150 schemas (~4–5k + // tokens) into EVERY turn, overflowing n_ctx → 400 "exceeds context size" → mute; + // (2) the later "tight window ⇒ discovery-pair only" shrink cliff (deleted in #206 / + // 74acbb36c) that amputated the native surface to `commands/list`+`commands/help` on + // a tight window, stranding native-tool models in a help loop with 0 edits. Now the + // category index rides the system prompt, the full native surface rides beside it + // un-amputated, and the budget (`prompt_view_within`) reserves the specs' tokens and + // trims VOLATILE context so prompt + tools + reserve fit the window. A regression + // means either the dump came back or the amputation cliff did. #[test] - fn tool_surface_is_a_category_index_plus_discovery_pair() { + fn tool_surface_is_a_category_index_plus_the_unamputated_native_surface() { let persona = Uuid::new_v4(); let adapter: Arc = Arc::new(HeuristicInferenceAdapter::new()); // A tool set whose FULL schemas would dwarf the window — the live shape From 97011f4f82102fb4eb4e3bfd2fc26a0d1379dfcf Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 12:55:47 -0500 Subject: [PATCH 04/19] =?UTF-8?q?feat(serving):=20elastic=20demand-driven?= =?UTF-8?q?=20context=20window=20=E2=80=94=20thread=20the=20ceiling,=20sto?= =?UTF-8?q?p=20baking=20it=20at=20launch=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The served window was capped by a STATIC constant: window_for(lanes).min(BOOTSTRAP_WORKING_SET). So a hard coding task was clamped to 16k even when the model supports 128k and the budget allows it — exactly the "set in stone at launch" anti-pattern, when the whole system is a live, continuously-re-decided negotiation. Thread the demand ceiling as a parameter: plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil). The window sizes UP to what the budget allows (window_for), then caps DOWN to what the TASK needs. A hard task passes a high ceiling → the window grows; a simple turn passes a low one → it shrinks so more lanes fit (the multi-persona concurrency win). This is the local half of "resources ebb and flow with demand"; the grid_overflow_lanes producer is the scale-out half — same demand→grant loop. plan_serving(...) stays as a thin cold-start wrapper passing BOOTSTRAP_WORKING_SET as the prior, so every existing caller is unchanged and behavior is identical until a demand producer threads live demand — the rail is laid without touching the live-GPU-gated fit math. OOM-safe by construction: window_for already bounds the window to the budget, so a higher ceiling only raises the cap TOWARD that bound, never past it. Growing for a hard task can't crash a lane. Test: cold=16k (prior), high-ceiling=64k (grew for the hard task), low-ceiling=8k (shrank for the simple one). 22 serving_plan tests green, no regression. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 75 ++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 1bdbf43dec..3976c18b24 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -310,12 +310,25 @@ pub struct ServingPlan { /// rendered, and the room degenerated into a greeting loop. 2 slots would /// have doubled every mind's window with zero lost concurrency. /// +/// `demand_ceil` is the LIVE demand ceiling for the served window — the ELASTIC, +/// per-task upper bound. `window_for` sizes the window UP to what the budget +/// allows; this caps it DOWN to what the task actually needs. A hard coding task +/// passes a high ceiling and the window GROWS (up to the budget/model bound); a +/// simple turn passes a low one so more lanes fit. It is NEVER a launch-baked +/// constant — callers thread live per-persona/per-task demand (measured p95 + +/// headroom, or a task's explicit request). [`plan_serving`] supplies +/// [`BOOTSTRAP_WORKING_SET`] only as the cold-start prior until that telemetry +/// exists (#234). OOM-safe: `window_for` already bounds the window to the budget, +/// so a higher ceiling only raises the cap toward that bound, never past it. +/// [[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]] +/// /// The decision is pure classification on memory arithmetic — no model is /// loaded, no inference is run. -pub fn plan_serving( +pub fn plan_serving_with_demand( host: HostBudget, candidates: &[ModelFootprint], demand_lanes: u32, + demand_ceil: u32, ) -> Option { if candidates.is_empty() { return None; @@ -441,7 +454,7 @@ pub fn plan_serving( // never forces UP past what the model supports; `.max(MIN_SERVE_CTX)` keeps it // runnable. On a small host where window_for < BOOTSTRAP the cap is a no-op. let served_context_window = window_for(lanes as u64) - .min(BOOTSTRAP_WORKING_SET) + .min(demand_ceil.max(MIN_SERVE_CTX)) .max(MIN_SERVE_CTX); // The honest per-lane compute reserve AT the chosen window (floor + window-scaled), // reused by the packing math below AND reported to the board via @@ -501,6 +514,20 @@ pub fn plan_serving( }) } +/// Cold-start / no-live-demand convenience: [`plan_serving_with_demand`] with the +/// [`BOOTSTRAP_WORKING_SET`] prior as the demand ceiling. Callers that do not YET +/// thread live per-task demand use this; the elastic path (a hard task requesting +/// more context, or the #234 p95 telemetry) calls `plan_serving_with_demand` with +/// the measured/requested ceiling so the window ebbs and flows with real demand +/// rather than a baked constant. +pub fn plan_serving( + host: HostBudget, + candidates: &[ModelFootprint], + demand_lanes: u32, +) -> Option { + plan_serving_with_demand(host, candidates, demand_lanes, BOOTSTRAP_WORKING_SET) +} + /// Hysteresis wrapper around [`plan_serving`]: stops model THRASH from live- /// budget jitter. Keeps the `incumbent` model as long as it still fits the /// budget — switching DOWN only when the incumbent no longer fits (forced @@ -765,6 +792,50 @@ mod tests { ); } + // what this catches: the ELASTIC demand ceiling (Joel 2026-07-27: "context window sizes + // should ebb and flow depending on demands of the task and available resources — if it + // needs it larger for a moment, don't limit it"). The ceiling is threaded LIVE, not a + // launch-baked constant: a hard task passing a HIGH demand_ceil grows the served window + // PAST the BOOTSTRAP prior (up to the budget/model bound); a LOW ceiling shrinks it so + // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises + // the cap toward the budget bound, never past it. This is the "stop setting it in stone + // at launch" fix that opens the elastic-lease path. + #[test] + fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { + let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; + // Roomy host + high trained ceiling → window_for(1) far exceeds any of these ceilings, + // so the DEMAND ceiling (not the budget or the model) decides the served window. + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + + // Cold prior: default plan_serving caps at BOOTSTRAP_WORKING_SET. + let cold = plan_serving(host, std::slice::from_ref(&devstral), 1).unwrap(); + assert_eq!(cold.served_context_window, BOOTSTRAP_WORKING_SET); + + // Hard task demands more context → the window GROWS past the prior. + let big_ceil = 64_000; + let hot = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, big_ceil).unwrap(); + assert!( + hot.served_context_window > cold.served_context_window, + "a higher demand ceiling must GROW the window: hot {} ≤ cold {}", + hot.served_context_window, + cold.served_context_window + ); + assert!( + hot.served_context_window <= big_ceil, + "growth stays bounded by the demand ceiling (and the budget), never past it: got {}", + hot.served_context_window + ); + + // Simple turn demands little → the window SHRINKS below the prior, freeing memory. + let lean = + plan_serving_with_demand(host, std::slice::from_ref(&devstral), 1, 8_192).unwrap(); + assert_eq!( + lean.served_context_window, 8_192, + "a low demand ceiling shrinks the served window to it" + ); + } + // what this catches: lanes DEGRADE on a tight host — MAX_LANES is a sanity backstop, // the fit math is the real cap. A 4-persona demand on a budget that can't feed 4 warm // slots must serve FEWER (well-fed) lanes, never 4 starving ones that OOM. The window From c29e92dd91bfb2c4e74751fdfe13f5178cb21275 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:13:57 -0500 Subject: [PATCH 05/19] =?UTF-8?q?feat(serving):=20WorkingSetDemand=20?= =?UTF-8?q?=E2=80=94=20the=20live=20demand=20producer=20for=20the=20elasti?= =?UTF-8?q?c=20window=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The elastic served-window seam (plan_serving_with_demand) needed a demand PRODUCER that is measured, not guessed, and never launch-baked. WorkingSetDemand is it: a rolling observer of each turn's assembled-prompt token count that produces the live demand ceiling threaded into the plan. Two signals combine so it's both efficient AND never truncates: - demand_ceil() = max(floor, p95(recent prompts) + gen_headroom) — the sustained baseline that keeps the WARM lane sized to the persona's usual work, and ebbs back to the floor as lean turns roll through the window (a past hard session doesn't pin the window forever). p95, not max, so a lone spike can't over-provision a KV that swaps the box. - demand_for(measured_prompt) = max(baseline, measured_prompt + headroom) — the MEASURED current turn is never clamped. This is how "if it needs it larger for a moment, don't limit it" holds WITHOUT guessing: we already assembled the prompt, so we request exactly its size. plan_serving_with_demand still bounds it by the budget above, so an impossible prompt degrades honestly, never OOMs. Pure + fully unit-tested (cold→floor, sustained→grow, ebb-back, spike-excluded, current-turn-never-truncated). No serving loop, no GPU — the honest measurement under the elastic lease. Next slices wire it: observe TurnMetrics.input_tokens per turn, feed demand_for() into the live re-plan. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/cognition/mod.rs | 1 + .../src/cognition/working_set_demand.rs | 198 ++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 core/continuum-core/src/cognition/working_set_demand.rs diff --git a/core/continuum-core/src/cognition/mod.rs b/core/continuum-core/src/cognition/mod.rs index 1aac35dcf7..cbb9a2fade 100644 --- a/core/continuum-core/src/cognition/mod.rs +++ b/core/continuum-core/src/cognition/mod.rs @@ -77,6 +77,7 @@ pub mod response_orchestrator; pub mod response_validator; pub mod serving_plan; pub mod shared_analysis; +pub mod working_set_demand; pub mod self_repeat; pub mod should_respond; pub mod should_respond_module; diff --git a/core/continuum-core/src/cognition/working_set_demand.rs b/core/continuum-core/src/cognition/working_set_demand.rs new file mode 100644 index 0000000000..78b1a8d043 --- /dev/null +++ b/core/continuum-core/src/cognition/working_set_demand.rs @@ -0,0 +1,198 @@ +//! `working_set_demand` — the DEMAND PRODUCER for the elastic served window (#234). +//! +//! A persona's served context window should ebb and flow with what its turns +//! ACTUALLY use, not a launch-baked constant ([[serving-resources-are-elastic-per-task-leases-context-and-model-grow-for-hard-problems]]). +//! This observes the assembled-prompt token count of recent turns and produces a +//! live demand ceiling that threads into +//! [`plan_serving_with_demand`](super::serving_plan::plan_serving_with_demand) — +//! growing the window for a persona whose tasks got bigger (a hard coding session) +//! and ebbing it back when they get lean again, so many personas stay warm at a +//! lean window and one doing heavy work gets room to think. +//! +//! ## Measured, not guessed — and never truncate the current turn +//! +//! Two signals combine: +//! - a rolling **p95 baseline** ([`demand_ceil`]) sizes the WARM lane to the +//! persona's usual work (so it's already big when the next hard turn arrives); +//! - the **measured current prompt** ([`demand_for`]) guarantees THIS turn is +//! never truncated — we already assembled the prompt, so we know its exact +//! size and request precisely that + generation headroom. +//! +//! That is how "if it needs it larger for a moment, don't limit it" holds WITHOUT +//! guessing. The budget fit in `plan_serving_with_demand` still bounds the result +//! above, so an impossibly large prompt degrades honestly, never OOMs. p95 (not +//! max) for the baseline so a single rare spike doesn't pre-allocate a KV that +//! swaps the box — the spike is handled per-turn by the measured path instead. +//! +//! Pure + testable: token counts in, demand ceiling out. No serving loop, no GPU +//! here — this is the measurement that makes the elastic lease honest. + +use std::collections::VecDeque; + +/// Below this many samples [`demand_ceil`] returns the cold `floor` rather than a +/// p95 off 1–2 points: provisioning a window off a tiny sample swings it turn to +/// turn (thrash). A handful of turns is enough to trust the shape without waiting +/// so long the window never grows within a short session. +const MIN_SAMPLES_FOR_P95: usize = 4; + +/// Rolling observer of a persona's per-turn working-set size (assembled-prompt +/// tokens), producing the live demand ceiling for its served window. +pub struct WorkingSetDemand { + /// Recent assembled-prompt token counts, oldest at the front. Bounded to + /// `window`, so a past hard session doesn't pin the ceiling forever. + samples: VecDeque, + /// Rolling window length (turns). ≥ 1. + window: usize, + /// Cold-start floor AND the hard minimum — the ceiling never drops below this, + /// so a lull can't starve the next turn's first prompt. Callers pass the + /// serving cold prior (`serving_plan::BOOTSTRAP_WORKING_SET`). + floor: u32, + /// Added to the observed prompt to leave room for the turn's GENERATION (the + /// prompt is what's assembled; the model still needs to write its answer). + gen_headroom: u32, +} + +impl WorkingSetDemand { + pub fn new(window: usize, floor: u32, gen_headroom: u32) -> Self { + let window = window.max(1); + Self { + samples: VecDeque::with_capacity(window), + window, + floor, + gen_headroom, + } + } + + /// Record one completed turn's assembled-prompt token count. Oldest evicted + /// past `window` so demand EBBS back — a past hard session doesn't pin the + /// window forever. + pub fn observe(&mut self, prompt_tokens: u32) { + if self.samples.len() == self.window { + self.samples.pop_front(); + } + self.samples.push_back(prompt_tokens); + } + + /// The sustained baseline demand ceiling: `max(floor, p95(recent prompts) + + /// gen_headroom)`. Floored so a lull can't starve the next turn; p95 (not max) + /// so a lone spike doesn't over-provision. Returns `floor` until there is + /// enough evidence ([`MIN_SAMPLES_FOR_P95`]) to trust the shape. + pub fn demand_ceil(&self) -> u32 { + if self.samples.len() < MIN_SAMPLES_FOR_P95 { + return self.floor; + } + let mut v: Vec = self.samples.iter().copied().collect(); + v.sort_unstable(); + let idx = (((v.len() - 1) as f64) * 0.95).round() as usize; + let p95 = v[idx.min(v.len() - 1)]; + self.floor.max(p95.saturating_add(self.gen_headroom)) + } + + /// The demand ceiling for a SPECIFIC turn whose assembled prompt we've already + /// MEASURED: never below what this turn actually needs + /// (`current_prompt_tokens + gen_headroom`), and never below the sustained p95 + /// baseline (so the warm lane stays sized for the persona's usual work). This + /// is how "don't limit it for a moment" holds without guessing — we measured + /// the prompt, so we request exactly enough. `plan_serving_with_demand` still + /// bounds it by the budget above. + pub fn demand_for(&self, current_prompt_tokens: u32) -> u32 { + self.demand_ceil() + .max(current_prompt_tokens.saturating_add(self.gen_headroom)) + .max(self.floor) + } + + /// How many turns are in the rolling window (for observability / tests). + pub fn sample_count(&self) -> usize { + self.samples.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The serving cold prior (serving_plan::BOOTSTRAP_WORKING_SET) + one generation + // of headroom — the values a live caller threads in. + const FLOOR: u32 = 16_384; + const HEAD: u32 = 2_048; + + // what this catches: cold start returns the FLOOR — provisioning a window off + // 1–2 samples would thrash it turn to turn, so it grows only once there's + // enough evidence to trust the shape. + #[test] + fn cold_start_returns_the_floor_until_enough_samples() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + assert_eq!(d.demand_ceil(), FLOOR); + d.observe(40_000); + d.observe(40_000); + assert_eq!(d.demand_ceil(), FLOOR, "still too few samples to trust the shape"); + } + + // what this catches: a sustained hard-coding session GROWS the baseline to + // p95 + generation headroom — the "if it needs it larger, don't limit it" case, + // measured not guessed. + #[test] + fn sustained_large_working_set_grows_the_ceiling_past_the_floor() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(40_000); + } + assert_eq!(d.demand_ceil(), 40_000 + HEAD, "grows to p95(40k) + headroom"); + assert!(d.demand_ceil() > FLOOR); + } + + // what this catches: demand EBBS BACK — a past hard session doesn't pin the + // window forever; once lean turns roll through the window, the ceiling returns + // to the floor and frees memory for more lanes. + #[test] + fn ceiling_ebbs_back_to_floor_as_lean_turns_roll_through_the_window() { + let mut d = WorkingSetDemand::new(8, FLOOR, HEAD); + for _ in 0..8 { + d.observe(40_000); + } + assert!(d.demand_ceil() > FLOOR); + for _ in 0..8 { + d.observe(3_000); // fully replaces the window + } + assert_eq!(d.demand_ceil(), FLOOR, "lean turns pull the ceiling back to the floor"); + } + + // what this catches: a single rare SPIKE does not over-provision the WARM + // baseline — p95 (not max) excludes the outlier so one 120k turn among lean + // ones can't pre-allocate a KV that swaps the box. + #[test] + fn single_spike_does_not_over_provision_the_baseline() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..31 { + d.observe(3_000); + } + d.observe(120_000); // one outlier among 31 lean turns + assert!( + d.demand_ceil() <= (3_000 + HEAD).max(FLOOR), + "the lone spike must not lift the sustained baseline: {}", + d.demand_ceil() + ); + } + + // what this catches: the MEASURED current turn is NEVER truncated — a big prompt + // we already assembled requests exactly its size + headroom even before the p95 + // baseline has grown ("don't limit it for a moment", from measurement not a + // guess); a small turn still gets at least the warm baseline. + #[test] + fn demand_for_never_truncates_the_measured_current_turn() { + let mut d = WorkingSetDemand::new(32, FLOOR, HEAD); + for _ in 0..10 { + d.observe(20_000); // baseline ~22k + } + assert_eq!( + d.demand_for(60_000), + 60_000 + HEAD, + "a measured 60k turn requests its own size + headroom, above the baseline" + ); + assert_eq!( + d.demand_for(1_000), + d.demand_ceil(), + "a tiny turn still rides the warm sustained baseline" + ); + } +} From 8501c5e8ca3da3d7c2a41b42ace7e5d4d8e5fcb1 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:27:18 -0500 Subject: [PATCH 06/19] =?UTF-8?q?feat(serving):=20serving=20daemon=20is=20?= =?UTF-8?q?demand-aware=20=E2=80=94=20elastic=20window=20threaded=20end=20?= =?UTF-8?q?to=20end=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Threads the elastic demand ceiling through the LIVE serving decision path, not just the boot path: - plan_serving_stable now takes demand_ceil and forwards it to plan_serving_with_demand (both internal calls), so the hysteresis/ongoing-loop path is elastic too. - ServingDaemonModule holds a WorkingSetDemand aggregator (p95 of recent turns' assembled-prompt sizes, rolling window, floored at BOOTSTRAP_WORKING_SET), reads its demand_ceil() on every plan (compute_plan + publish_plan), and exposes observe_working_set(prompt_tokens) for the cognition turn path to feed. Inert-by-default: with no observations the aggregator returns the cold prior, so the served window is identical to before — behavior only changes once the emit is wired. A poisoned lock degrades to the prior, never panics. The whole plan API (boot + ongoing loop) is now demand-capable; the last hop is emitting each turn's input_tokens into observe_working_set so the window breathes with real demand. 74 serving tests green, no regression. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 15 ++++--- .../src/modules/serving_daemon.rs | 45 +++++++++++++++++-- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index 3976c18b24..e935227bc5 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -541,13 +541,14 @@ pub fn plan_serving_stable( candidates: &[ModelFootprint], incumbent: Option<&str>, demand_lanes: u32, + demand_ceil: u32, ) -> Option { // NB: do NOT `?`-bail here. A deep transient dip can leave `plan_serving` // with nothing fitting the depressed budget (`fresh` = None) while a model // is STILL resident and serving fine — its memory is its own. Tearing that // down to "nothing" is the exact harm we're guarding against, so `fresh` is // an Option we fall back to only when the incumbent genuinely can't hold. - let fresh = plan_serving(host, candidates, demand_lanes); + let fresh = plan_serving_with_demand(host, candidates, demand_lanes, demand_ceil); let Some(inc_id) = incumbent else { return fresh; }; @@ -599,7 +600,7 @@ pub fn plan_serving_stable( if let Some(m) = promoted.iter_mut().find(|m| m.model_id == inc_id) { m.capability_rank = u8::MAX; } - plan_serving(at_rest, &promoted, demand_lanes) + plan_serving_with_demand(at_rest, &promoted, demand_lanes, demand_ceil) } fn bytes_gb(bytes: u64) -> f64 { @@ -1062,7 +1063,7 @@ mod tests { fn stable_with_no_incumbent_equals_plain() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; assert_eq!( - plan_serving_stable(host, &pair(), None, MAX_LANES), + plan_serving_stable(host, &pair(), None, MAX_LANES, BOOTSTRAP_WORKING_SET), plan_serving(host, &pair(), MAX_LANES) ); } @@ -1075,7 +1076,7 @@ mod tests { // 10GB: big (9.7GB) fits a lane but exceeds the 0.9*10=9GB headroom bar. let host = HostBudget { usable_bytes: 10 * GB, perf_cores: 6 }; assert_eq!(plan_serving(host, &pair(), MAX_LANES).unwrap().base_model_id, "big", "fresh would pick big"); - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "hysteresis keeps incumbent — no flap"); assert!(stable.lanes >= 1, "lanes still re-tracked for the kept model"); } @@ -1085,7 +1086,7 @@ mod tests { #[test] fn stable_upgrades_when_better_model_fits_with_headroom() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; // big 9.7 << 0.9*20=18 - let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &pair(), Some("small"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "more capable + ample headroom → upgrade"); } @@ -1100,7 +1101,7 @@ mod tests { fn stable_forced_down_when_incumbent_gone_from_disk() { let host = HostBudget { usable_bytes: 20 * GB, perf_cores: 6 }; let only_small = vec![fp("small", 1, 4_000, 32_768, 1)]; // "big" no longer on disk - let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(host, &only_small, Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "small", "incumbent gone from disk → serve what's present"); } @@ -1124,7 +1125,7 @@ mod tests { "depressed-budget plain plan would flap to the smaller model" ); // With the incumbent credited its own weights back, the resident big stays. - let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES).unwrap(); + let stable = plan_serving_stable(dipped, &pair(), Some("big"), MAX_LANES, BOOTSTRAP_WORKING_SET).unwrap(); assert_eq!(stable.base_model_id, "big", "incumbent survives its OWN load dip — no flap"); assert!(stable.lanes >= 1, "kept model still gets ≥1 lane"); } diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index b32dd25112..ff94143298 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -23,8 +23,10 @@ use crate::cognition::model_resolver::types::HwCapabilityTier; use crate::cognition::serving_plan::{ - plan_serving, plan_serving_stable, HostBudget, ModelFootprint, ServingPlan, MIN_SERVE_CTX, + plan_serving, plan_serving_stable, plan_serving_with_demand, HostBudget, ModelFootprint, + ServingPlan, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, }; +use crate::cognition::working_set_demand::WorkingSetDemand; use crate::gpu::GpuMemoryManager; use crate::inference::llama_server::{ ensure_model_serving, serving_v1_url, AdapterEntry, EnsureOutcome, LlamaServerControl, @@ -239,6 +241,12 @@ pub struct ServingDaemonModule { /// that fit at pin time; budget can still shift under it, and then the plan /// degrades honestly (`fits_on_gpu = false`) rather than over-committing. pinned: watch::Sender>, + /// Live per-host working-set demand — the p95 of recent turns' assembled-prompt + /// sizes, threaded into the serving plan as the elastic `demand_ceil` so the + /// served window ebbs and flows with real demand instead of the baked prior + /// (#234). A Mutex (not atomic) for the rolling window; read only on the plan + /// tick, never a hot path. `observe_working_set` feeds it from the cognition turn. + working_set: std::sync::Mutex, } impl ServingDaemonModule { @@ -273,6 +281,10 @@ impl ServingDaemonModule { let (serving_tx, _srx) = watch::channel(ServingSnapshot::empty()); let (suppressed, _urx) = watch::channel(Arc::new(HashSet::new())); let (pinned, _prx) = watch::channel(None); + // Rolling window (recent turns) over which the served window's demand ceiling + // tracks the p95 assembled-prompt size. Long enough to hold a coding session's + // shape, short enough that demand ebbs back within a session (#234). + const WORKING_SET_WINDOW: usize = 64; Self { gpu, system, @@ -296,6 +308,11 @@ impl ServingDaemonModule { suppressed, pinned, lane_demand: Arc::new(std::sync::atomic::AtomicU32::new(1)), + working_set: std::sync::Mutex::new(WorkingSetDemand::new( + WORKING_SET_WINDOW, + BOOTSTRAP_WORKING_SET, + MIN_SERVE_CTX, // one generation of headroom above the assembled prompt + )), moe_serving: std::sync::Mutex::new(None), // Read ONCE here (single config entry point, no per-tick I/O). Off by default. measure_force_expert_budget_bytes: crate::config_env::read( @@ -313,6 +330,27 @@ impl ServingDaemonModule { .store(demand.max(1), Ordering::Relaxed); } + /// Feed one completed turn's assembled-prompt token count into the live + /// working-set demand (#234). Called from the cognition turn path; the next + /// plan tick sizes the served window to the p95 of recent turns. Cheap — a + /// brief lock on a bounded ring, off the serving hot path. + pub fn observe_working_set(&self, prompt_tokens: u32) { + if let Ok(mut w) = self.working_set.lock() { + w.observe(prompt_tokens); + } + } + + /// The live demand ceiling for the served window — the p95 baseline of recent + /// turns (floored at the cold prior). Read on the plan tick and threaded into + /// `plan_serving_with_demand` so the window grows for heavy work and ebbs back + /// when turns get lean. A poisoned lock degrades to the cold prior, never panics. + fn demand_ceil(&self) -> u32 { + self.working_set + .lock() + .map(|w| w.demand_ceil()) + .unwrap_or(BOOTSTRAP_WORKING_SET) + } + /// The current lane demand (≥ 1). /// Register serving's autonomic PLANNER to run on the memory authority's tick /// (MEMORY-AUTHORITY-DAEMON slice 1b). The lane plan — which model, how many lanes, @@ -559,10 +597,11 @@ impl ServingDaemonModule { /// to drive the spawner before the tick loop starts — single source of /// truth for "what model + how many lanes." pub fn compute_plan(&self) -> Option { - plan_serving( + plan_serving_with_demand( self.host_budget(), &self.live_candidates(), self.lane_demand(), + self.demand_ceil(), ) } @@ -1004,7 +1043,7 @@ impl ServingDaemonModule { .borrow() .as_ref() .map(|p| p.base_model_id.clone()); - match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand()) { + match plan_serving_stable(budget, candidates, incumbent.as_deref(), self.lane_demand(), self.demand_ceil()) { Some(plan) => { crate::probe!( class = "serving.plan", From e6b3f1c013c54222285fb3baf7fd80dee86cb287 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:33:55 -0500 Subject: [PATCH 07/19] =?UTF-8?q?feat(serving):=20close=20the=20elastic-wi?= =?UTF-8?q?ndow=20loop=20=E2=80=94=20turn=20demand=20feeds=20the=20served?= =?UTF-8?q?=20window=20(#234)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final hop. The persona turn path now feeds each completed turn's assembled-prompt size into the serving demand, so the window breathes with real work instead of a baked constant: - serving_daemon exposes a process-wide sink (SERVING_WORKING_SET, same install_serving_state shape) holding the daemon's WorkingSetDemand (now Arc), registered at initialize(). observe_serving_working_set(tokens) is a free function the cognition path calls with NO daemon handle — no cross-subsystem plumbing. - service_loop, where each turn's TurnMetrics is known, calls observe_serving_working_set(m.input_tokens) at turn completion. The whole loop is now live: turn completes -> observe -> p95 baseline updates -> next plan tick sizes the served window to real demand (plan_serving_with_demand) -> a lean chat turn keeps it small so more personas stay warm, a heavy coding turn grows it toward the model/budget ceiling. Safe/inert until a daemon boots (the sink registers at init), so it can't break anything; it activates on the next deploy. Full continuum-core lib compiles clean. This completes the #234 elastic serving substrate started with plan_serving_with_demand + WorkingSetDemand: producer, seam, plan threading, daemon-awareness, and now the emit — all built and validated in isolation, ready for a live burst to watch the window grow on a hard turn. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/modules/serving_daemon.rs | 30 +++++++++++++++++-- .../src/persona/service_loop.rs | 5 ++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index ff94143298..b1b19fc191 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -27,6 +27,26 @@ use crate::cognition::serving_plan::{ ServingPlan, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, }; use crate::cognition::working_set_demand::WorkingSetDemand; + +/// Process-wide sink for per-turn working-set observations — the serving daemon +/// installs its own `WorkingSetDemand` here at [`ServingDaemonModule::initialize`], +/// so the cognition turn path feeds each turn's assembled-prompt size WITHOUT a +/// daemon handle (the same process-wide-readable-seam shape as `install_serving_state`). +/// Unset until a daemon boots → [`observe_serving_working_set`] is a no-op and the +/// served window simply holds its cold prior. (#234) +static SERVING_WORKING_SET: OnceLock>> = OnceLock::new(); + +/// Feed one completed turn's assembled-prompt token count into the live serving +/// demand. Called from the persona turn path; the next plan tick sizes the served +/// window to the p95 of recent turns. No-op before a serving daemon has booted, so +/// the caller never needs to know whether serving is up. (#234) +pub fn observe_serving_working_set(prompt_tokens: u32) { + if let Some(ws) = SERVING_WORKING_SET.get() { + if let Ok(mut w) = ws.lock() { + w.observe(prompt_tokens); + } + } +} use crate::gpu::GpuMemoryManager; use crate::inference::llama_server::{ ensure_model_serving, serving_v1_url, AdapterEntry, EnsureOutcome, LlamaServerControl, @@ -246,7 +266,7 @@ pub struct ServingDaemonModule { /// served window ebbs and flows with real demand instead of the baked prior /// (#234). A Mutex (not atomic) for the rolling window; read only on the plan /// tick, never a hot path. `observe_working_set` feeds it from the cognition turn. - working_set: std::sync::Mutex, + working_set: Arc>, } impl ServingDaemonModule { @@ -308,11 +328,11 @@ impl ServingDaemonModule { suppressed, pinned, lane_demand: Arc::new(std::sync::atomic::AtomicU32::new(1)), - working_set: std::sync::Mutex::new(WorkingSetDemand::new( + working_set: Arc::new(std::sync::Mutex::new(WorkingSetDemand::new( WORKING_SET_WINDOW, BOOTSTRAP_WORKING_SET, MIN_SERVE_CTX, // one generation of headroom above the assembled prompt - )), + ))), moe_serving: std::sync::Mutex::new(None), // Read ONCE here (single config entry point, no per-tick I/O). Off by default. measure_force_expert_budget_bytes: crate::config_env::read( @@ -1557,6 +1577,10 @@ impl ServiceModule for ServingDaemonModule { // free functions + adapters read "what's live" as a pointer instead of // each probing /v1/models. Set-once (singleton daemon). let _ = crate::inference::llama_server::install_serving_state(self.subscribe_serving()); + // Install this daemon's working-set demand as the process-wide sink so the + // cognition turn path feeds per-turn prompt sizes without a daemon handle, and + // the served window ebbs and flows with real demand (#234). + let _ = SERVING_WORKING_SET.set(self.working_set.clone()); // Register serving as a MEASURED ResourceConsumer with the one per-machine // authority (#79). See `register_as_consumer` — this is monitor-not-reserve: // no lease acquired, `available` math untouched, the authority simply stops diff --git a/core/continuum-core/src/persona/service_loop.rs b/core/continuum-core/src/persona/service_loop.rs index 945a422e90..d61f53526c 100644 --- a/core/continuum-core/src/persona/service_loop.rs +++ b/core/continuum-core/src/persona/service_loop.rs @@ -1058,6 +1058,11 @@ async fn serve_persona_loop_inner( tokens_per_second = m.tokens_per_second(), "deliberation generation cost" ); + // Feed this turn's assembled-prompt size into the elastic serving + // demand so the served window ebbs and flows with real work — a lean + // chat turn keeps it small (more personas warm), a heavy coding turn + // grows it (#234). No-op until a serving daemon has booted. + crate::modules::serving_daemon::observe_serving_working_set(m.input_tokens); } match step { crate::cognition::act_observe::SettleStep::Spoke(text) => text, From 0ea9538a1ea7e589752c32b944c0164b05066097 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:43:47 -0500 Subject: [PATCH 08/19] =?UTF-8?q?feat(serving):=20opt-in=20KV=20cache=20qu?= =?UTF-8?q?antization=20=E2=80=94=20q8=5F0=20halves=20KV,=20feeds=20the=20?= =?UTF-8?q?elastic=20window=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds SERVING_KV_CACHE_TYPE (config, default f16/off): when set to q8_0 (or q4_0) the llama-server lane runs --cache-type-k/v , cutting resident KV ~in half at near-lossless quality. That frees memory the elastic window (#234) can spend on a bigger context or more warm lanes — faster for multiple personas AND more room for hard coding, the same "faster + best code" pair. OFF by default and safe-by-construction: absent / f16 → byte-identical f16 launch (no behavior change), so this can't destabilize a backend whose build lacks Metal KV-quant kernels — enabling it is an explicit operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). Follow-up (noted in code): to have the PLAN grow the window on the freed memory rather than leave it as extra headroom, footprint_for must scale kv_per_token by the quant factor. This slice is the safe enablement; that fit-math coupling is the next step, and wants a live burst on a KV-quant-capable backend to validate quality + the speedup. continuum-core lib compiles clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/inference/llama_server.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index e54e31cec5..0970f4c4b7 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -1354,6 +1354,25 @@ impl LlamaServerControl for LlamaServerProcess { // surfaces the real defect: a RAG budget that overshot the served // window ([[fallbacks-are-illegal-fail-loud]]). .arg("--no-context-shift"); + // KV CACHE QUANTIZATION (#232, opt-in field-proven technique). f16 KV is the + // default; q8_0 is ~half the resident KV footprint at near-lossless quality, + // freeing memory the elastic window (#234) can spend on a BIGGER context or MORE + // warm lanes — faster for multiple personas AND more room for hard coding. + // OFF by default: not every backend/build ships Metal KV-quant kernels, so this + // is an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). + // Set SERVING_KV_CACHE_TYPE=q8_0 (or q4_0) to enable; absent / `f16` → byte-identical + // f16 behavior. NOTE: to have the plan actually GROW the window on the freed memory + // (not just leave it as extra headroom), the fit math must also scale kv_per_token — + // that footprint coupling is the follow-up; this slice is the safe enablement. + if let Some(kv_type) = crate::config_env::read("SERVING_KV_CACHE_TYPE") + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty() && s != "f16") + { + cmd.arg("--cache-type-k") + .arg(&kv_type) + .arg("--cache-type-v") + .arg(&kv_type); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render From 175cd6d51f8c6e459c37e024393182db629f81ab Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:51:10 -0500 Subject: [PATCH 09/19] =?UTF-8?q?feat(serving):=20KV-quant=20fit=20couplin?= =?UTF-8?q?g=20=E2=80=94=20the=20window=20GROWS=20into=20the=20freed=20KV?= =?UTF-8?q?=20memory=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the KV-quant feature. The launcher flag (prior commit) makes a lane run q8_0 KV; this makes the PLAN know it: footprint_for scales kv_per_token by the quant divisor, so the served window is sized against the KV the lane WILL actually hold, and the elastic window (#234) grows into the freed memory instead of leaving it idle. - kv_divisor_for (pure, env-free, unit-tested): f16/unset/unknown → 1 (no change), q8_0 → 2, q4_0/q4_1 → 3. CONSERVATIVE by design — under the ideal ~3.5x for q4 — so the plan can never over-grow the window past the real KV and OOM (over-reserve = smaller window = safe). - Applied only in the config-aware footprint_for; footprint_from_parts stays pure so its tests are env-independent. Same SERVING_KV_CACHE_TYPE key as the launcher — one config, two consumers (flag + fit rate), documented to stay in sync. Default (f16 / unset) → divisor 1 → byte-identical: this can't change serving on a box that doesn't opt in. Test pins the mapping + the safe-default. Wants a live burst on a KV-quant-capable backend to confirm quality + the actual window growth. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/modules/serving_daemon.rs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index b1b19fc191..452f91e4c3 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -1349,12 +1349,38 @@ fn serving_footprint_fn(catalog: Arc) -> FootprintFn { pub fn footprint_for(model: &Model) -> Option { let path = crate::model_registry::artifacts::resolve_gguf_for_model(model)?; let weights_bytes = std::fs::metadata(&path).ok()?.len(); - footprint_from_parts( + let mut fp = footprint_from_parts( &model.id, weights_bytes, model.context_window, model.has(Capability::ToolUse), - ) + )?; + // KV CACHE QUANTIZATION (#232): a lane running quantized KV holds proportionally + // fewer bytes/token, so the plan can size a BIGGER window into the same budget — + // this is what turns the launcher's opt-in q8_0 flag into an actual window GROWTH. + // Divide the f16 rate by the quant factor; default (f16 / unset) → 1 → byte-identical. + // Keep the config key in sync with the launcher arg in inference/llama_server.rs — + // one SERVING_KV_CACHE_TYPE key, two consumers (launcher flag + this fit-math rate). + fp.kv_per_token = (fp.kv_per_token / kv_cache_quant_divisor()).max(1); + Some(fp) +} + +/// The resident-KV divisor implied by `SERVING_KV_CACHE_TYPE`, so the plan sizes the +/// served window against the KV the lane WILL actually hold, not the f16 default. (#232) +fn kv_cache_quant_divisor() -> u64 { + kv_divisor_for(crate::config_env::read("SERVING_KV_CACHE_TYPE").as_deref()) +} + +/// Pure KV-rate divisor for a cache-type string (testable without env). CONSERVATIVE by +/// design: q8_0 ≈ half of f16 → 2; q4_0/q4_1 ≈ a third → 3 (under the ideal ~3.5×, so the +/// plan never over-grows the window past the real KV and OOMs). Anything else / f16 → 1 +/// (no change). Over-reserve is a smaller window (safe); under-reserve is an OOM (fatal). +fn kv_divisor_for(cache_type: Option<&str>) -> u64 { + match cache_type.map(|s| s.trim().to_ascii_lowercase()).as_deref() { + Some("q8_0") => 2, + Some("q4_0") | Some("q4_1") => 3, + _ => 1, + } } /// Pure footprint estimate from the fields that drive it — split out from the @@ -1798,6 +1824,19 @@ mod tests { // what this catches: footprint estimate is honest about weights (passed // through), tool capability bumps the rank, KV is non-zero, and zero // weights → no footprint (we only offer what we can actually serve). + #[test] + fn kv_divisor_reflects_cache_type_conservatively() { + // what this catches: the #232 KV-quant fit-math coupling — the served window grows + // only when the lane actually runs quantized KV, and CONSERVATIVELY so the plan + // never over-grows past the real KV and OOMs. f16/unset/unknown must never scale. + assert_eq!(kv_divisor_for(None), 1, "unset never scales the window"); + assert_eq!(kv_divisor_for(Some("f16")), 1, "explicit f16 is the no-op default"); + assert_eq!(kv_divisor_for(Some("q8_0")), 2, "q8_0 ~ half of f16"); + assert_eq!(kv_divisor_for(Some(" Q8_0 ")), 2, "trimmed + case-insensitive"); + assert_eq!(kv_divisor_for(Some("q4_0")), 3, "q4_0 conservative, under the ideal ~3.5x"); + assert_eq!(kv_divisor_for(Some("garbage")), 1, "unknown type → no grow, never a bogus OOM"); + } + #[test] fn footprint_from_parts_is_footprint_aware() { let fp = footprint_from_parts("present", 3 * GB, 8192, true).unwrap(); From 10fed2798615a725590ef641aaab6636ae14b0fd Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:07:29 -0500 Subject: [PATCH 10/19] =?UTF-8?q?feat(serving):=20opt-in=20flash=20attenti?= =?UTF-8?q?on=20=E2=80=94=20faster=20prefill+decode,=20lower=20memory=20(#?= =?UTF-8?q?232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fused attention kernel is faster on BOTH prefill and decode and lowers peak memory — directly attacking the prefill-bound turn latency (#139) and freeing room the elastic window (#234) can spend. SERVING_FLASH_ATTN=1|on|true adds --flash-attn to the lane; absent → llama.cpp default (no flag), byte-identical. OFF by default: Metal/backend flash-attn support + quality vary by build, so it's an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). Composes with the KV-quant flag: enable both for the field-proven GLM-style speedup, then validate on a live burst. continuum-core lib compiles clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/inference/llama_server.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index 0970f4c4b7..7bd27ca9ef 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -1373,6 +1373,19 @@ impl LlamaServerControl for LlamaServerProcess { .arg("--cache-type-v") .arg(&kv_type); } + // FLASH ATTENTION (#232, opt-in field-proven technique). The fused attention kernel + // is faster on BOTH prefill and decode and lowers peak memory — directly attacking + // the prefill-bound turn latency (#139) and freeing room the elastic window (#234) + // can spend. OFF by default: Metal/backend flash-attn support + quality vary by build + // ([[verify-real-device-numbers-not-a-clamp-premise]]), so it's an operator opt-in, + // never a blind assumption. SERVING_FLASH_ATTN=1|on|true → enable; absent → llama.cpp + // default (no flag), byte-identical. + if crate::config_env::read("SERVING_FLASH_ATTN") + .map(|s| matches!(s.trim().to_ascii_lowercase().as_str(), "1" | "on" | "true" | "yes")) + .unwrap_or(false) + { + cmd.arg("--flash-attn"); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render From e47661aaf7fcd99917e4d012e457926c80faf408 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:08:07 -0500 Subject: [PATCH 11/19] =?UTF-8?q?feat(capacity):=20expert=5Fobserve=20?= =?UTF-8?q?=E2=80=94=20per-domain=20concentration=20+=20working-set-size?= =?UTF-8?q?=20+=20Jaccard=20(#230)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evolves the MoE glass-box harness from the pooled first cut to the measurement that actually decides the paging architecture (BigMama's three methodological guardrails): diverse multi-domain corpus, PER-DOMAIN concentration (top-K% activation share vs the uniform null), cross-domain hot-set Jaccard, shared-base-vs-own-only activation MASS split, and the working-set-size curve (experts resident for 50/80/90/95% of a domain's mass). Prefill-dominant sampling on realistic input, not a degenerating greedy loop. Ran live on Qwen3-Coder-30B-A3B (Metal): pooled top-10% = 18.6% (mild — the smear), but per-domain = 25–38% with near-disjoint hot sets (code↔prose Jaccard 0.05) and a tiny 38-expert universal core — i.e. paging is domain-working-set SWAPPING, not frequency tiering. This is the #180 evidence; the harness is the reusable probe for any MoE (incl. K3 at weight-drop). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/bin/expert_observe.rs | 370 ++++++++++++++---- 1 file changed, 299 insertions(+), 71 deletions(-) diff --git a/core/continuum-core/src/bin/expert_observe.rs b/core/continuum-core/src/bin/expert_observe.rs index 99ceec65bd..558e731222 100644 --- a/core/continuum-core/src/bin/expert_observe.rs +++ b/core/continuum-core/src/bin/expert_observe.rs @@ -1,49 +1,125 @@ //! expert_observe — glass-box the LIVE MoE expert routing (#230 / #229). //! -//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached (the -//! already-built `cb_eval` → `ffn_moe_topk` seam in `core/llama/src/safe.rs`), generates -//! N tokens to drive REAL routing, then dumps the affinity data — hot/cold expert -//! distribution + co-occurrence + prefetch prediction. That affinity is the INPUT to -//! expert prefetch (#227), grid placement (#180), compaction, and distillation (#233). +//! Runs a GGUF MoE through the core/llama FFI with a [`LiveExpertObserver`] attached, drives +//! REAL routing over a labelled multi-domain corpus, then reports the affinity that decides +//! the paging architecture (#180): PER-DOMAIN concentration + CROSS-DOMAIN hot-set overlap. //! -//! WHY this and not the live llama-server path: expert affinity is MODEL-INTRINSIC (which -//! experts co-fire for given inputs), so an in-process FFI run gathers valid data without -//! touching the live-persona serving lane. (The live llama-server path would need a -//! separate fork patch to emit routing; this harness needs neither.) +//! ## Methodology (BigMama's three guardrails, 2026-07-27) +//! 1. **Sample size.** ~7 activations/slot is far too few to tell power-law from uniform — +//! Poisson noise swamps the skew. We drive thousands of tokens so the head separates. +//! 2. **Clean samples, not a greedy loop.** A long greedy generation degenerates and +//! OVER-routes to the same experts — a false positive for locality. We PREFILL diverse +//! prompts (prefill routes every token on realistic input) with only short generation. +//! 3. **Diverse HOW — the knife-edge.** Pooling ACROSS domains measures the global +//! mixed-workload histogram; MoE specialization routes domains to different experts, so +//! pooling SMEARS toward uniform and answers the WRONG question. The paging architecture +//! serves ONE persona doing ONE coherent thing, so we measure: +//! (a) PER-DOMAIN concentration — one observer per domain = the coherent-session +//! paging headroom (top-K% activation share WITHIN a domain). +//! (b) CROSS-DOMAIN overlap — intersect each domain's hot set. Experts hot in EVERY +//! domain = the always-resident shared base (never paged); experts hot in ONE +//! domain = the pageable specialization tier. The money number: what fraction of a +//! domain's activation MASS lands on experts cold for the other domains — the +//! eviction win. //! //! Usage: -//! cargo run -p continuum-core --features metal,accelerate --bin expert_observe -- [n_tokens] [prompt] +//! cargo run --release -p continuum-core --features metal,accelerate --bin expert_observe -- [n_gen_per_prompt] +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; use continuum_core::capacity::expert_observer::LiveExpertObserver; use llama::{Batch, ContextParams, ExpertObserver, Model, ModelParams, Sampler}; -fn main() { - let args: Vec = std::env::args().collect(); - let model_path = args - .get(1) - .expect("usage: expert_observe [n_tokens] [prompt]"); - let n_tokens: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(96); - let prompt = args.get(3).map(|s| s.as_str()).unwrap_or( - "Write a Rust function to reverse a string, then explain how it works step by step:\n", - ); - - // The observer is the sink: `cb_eval` calls `observe(layer, selected_experts, n_used)` - // per MoE layer per token from inside the compute thread. - let observer = LiveExpertObserver::new(); - - let model = Model::load( - PathBuf::from(model_path), - ModelParams { - n_gpu_layers: -1, - use_mmap: true, - }, - ) - .expect("load model"); - println!("Loaded {model_path} (vocab={})", model.n_vocab()); +/// Labelled corpus: each domain gets its OWN observer, fed several diverse prompts so the +/// per-domain sample is dense. The point is a COHERENT workload per observer (what a persona +/// actually does in a session), not a global mixture. +const DOMAINS: &[(&str, &[&str])] = &[ + ( + "code", + &[ + "Implement a lock-free single-producer single-consumer ring buffer in Rust using \ + atomics with Release/Acquire ordering; explain why a SeqCst fence is unnecessary \ + and how the head/tail indices wrap a power-of-two capacity without a modulo in the \ + hot path.\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\n", + "Write a React hook useDebouncedResource taking an async fetcher and debounce \ + interval that cancels in-flight requests on key change, dedupes concurrent callers, \ + and surfaces loading/error/stale-while-revalidate state without tearing under \ + concurrent mode.\n\nimport { useEffect, useRef, useState } from 'react';\n", + "Given orders(id, customer_id, placed_at, total) and refunds(order_id, amount, \ + refunded_at), write SQL returning each customer's net revenue by month, excluding \ + months with over 40% refunded, ranking customers within each month by a window \ + function.", + ], + ), + ( + "prose", + &[ + "The lighthouse keeper had not spoken to another person in forty-one days when the \ + boat appeared on the horizon. He noticed first the wrongness in the grey, and only \ + afterward resolved the shape into a hull riding low, and set down the brass polish \ + and went to the door, the cold coming up through the stone under his socks.", + "She had rehearsed the apology on the train, each version softer than the last, until \ + the words lost their edges and became a kind of weather she carried into the room. \ + Her mother was at the sink with her back turned, and for a moment neither of them \ + moved, and the tap ran over a single white plate.", + "Write the opening of a short story about a cartographer who discovers that a river \ + on his oldest map no longer exists, and who sets out on foot to find where it went, \ + narrated in close third person with attention to the texture of the walking.", + ], + ), + ( + "math", + &[ + "Prove that every finite integral domain is a field. Fix a nonzero a and consider \ + x -> a x; show injectivity, conclude surjectivity from finiteness, hence an inverse \ + of a exists. Then exhibit an infinite integral domain that is not a field to show \ + finiteness is essential.", + "Derive the closed form for the variance of a sum of two correlated random variables \ + in terms of their individual variances and covariance, then generalize to n \ + variables and interpret the cross terms as the reason diversification reduces \ + portfolio variance.", + "State and prove the pigeonhole principle, then use it to show that among any 51 \ + integers chosen from 1 to 100 there must be two that are coprime, and separately two \ + whose difference is exactly 10.", + ], + ), + ( + "science", + &[ + "Explain how the sodium-potassium pump maintains a neuron's resting potential: three \ + sodium out for two potassium in, the ATP-driven conformational change, and how the \ + electrogenic imbalance plus leak channels and the Nernst equilibria set the roughly \ + -70 mV resting potential.", + "Describe why the sky is blue in terms of Rayleigh scattering: the inverse fourth \ + power wavelength dependence, why shorter wavelengths scatter more strongly, and why \ + sunsets are red because of the longer atmospheric path length near the horizon.", + "Explain the greenhouse effect at the level of molecular physics: which atmospheric \ + gases absorb in the infrared, why their vibrational modes couple to outgoing \ + longwave radiation while nitrogen and oxygen do not, and how re-emission warms the \ + surface.", + ], + ), + ( + "planning", + &[ + "Sketch the migration plan to decompose a Rust monolith into services over a \ + command-and-event bus: identify the seams where synchronous calls become async, how \ + you preserve transactional guarantees that relied on one process, and how you roll \ + out incrementally behind a facade without a big-bang cutover.", + "Write a JSON schema for a distributed-cache eviction policy supporting LRU, LFU, and \ + TTL tiers with per-key overrides, budgets in both bytes and percentage-of-pool, and \ + validation that every declared cache class has at least one eviction dimension so \ + none grows unbounded.", + "Draft a rollout plan for a feature flag that changes a checkout flow: staged \ + percentage ramp, the metrics that gate each stage, the automatic rollback trigger, \ + and how you keep the two code paths from diverging while the flag is live.", + ], + ), +]; +fn drive_prompt(model: &Model, observer: &Arc, prompt: &str, n_gen: usize) { let mut ctx = model .new_context(ContextParams { n_ctx: 4096, @@ -54,61 +130,213 @@ fn main() { }) .expect("context"); - // Prefill. - let prompt_tokens = model.tokenize(prompt, true, false).expect("tokenize"); - let mut batch = Batch::allocated(512, 1); - let last = (prompt_tokens.len() - 1) as i32; - for (i, tok) in prompt_tokens.iter().enumerate() { - batch.push(*tok, i as i32, &[0], i as i32 == last); + let tokens = model.tokenize(prompt, true, false).expect("tokenize"); + let mut n_cur: i32 = 0; + for chunk in tokens.chunks(512) { + let mut batch = Batch::allocated(512, 1); + let last_global = n_cur as usize + chunk.len() - 1; + for (i, tok) in chunk.iter().enumerate() { + let pos = n_cur + i as i32; + batch.push(*tok, pos, &[0], (n_cur as usize + i) == last_global); + } + ctx.decode(&batch).expect("prefill decode"); + n_cur += chunk.len() as i32; } - ctx.decode(&batch).expect("prefill decode"); - // Generate — every decoded token drives the router → observer tallies real selections. let mut sampler = Sampler::greedy(); - let mut n_cur = batch.n_tokens(); - let mut n_decoded = 0usize; - for _ in 0..n_tokens { + for _ in 0..n_gen { let token = sampler.sample(&ctx, -1); if model.is_eog_token(token) { break; } - batch.clear(); + let mut batch = Batch::allocated(1, 1); batch.push(token, n_cur, &[0], true); ctx.decode(&batch).expect("gen decode"); n_cur += 1; - n_decoded += 1; } +} + +/// Sorted-desc counts → activation share captured by the top `frac` of FIRED experts. +fn top_share(counts_desc: &[u64], total: u64, frac: f64) -> f64 { + if counts_desc.is_empty() || total == 0 { + return 0.0; + } + let k = ((counts_desc.len() as f64 * frac).ceil() as usize).max(1).min(counts_desc.len()); + let s: u64 = counts_desc.iter().take(k).sum(); + 100.0 * s as f64 / total as f64 +} + +fn main() { + let args: Vec = std::env::args().collect(); + let model_path = args + .get(1) + .expect("usage: expert_observe [n_gen_per_prompt]"); + let n_gen: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(48); + + let model = Model::load( + PathBuf::from(model_path), + ModelParams { n_gpu_layers: -1, use_mmap: true }, + ) + .expect("load model"); + println!("Loaded {model_path} (vocab={})", model.n_vocab()); + + // Drive each domain into its OWN observer → per-domain hit maps. + let mut per_domain = Vec::new(); + for (label, prompts) in DOMAINS { + let obs = LiveExpertObserver::new(); + for prompt in *prompts { + drive_prompt(&model, &obs, prompt, n_gen); + } + let hits = obs.snapshot_hits(); + println!( + " domain {label:<9} : {} activations across {} experts", + obs.total_hits(), + hits.len() + ); + per_domain.push((label.to_string(), hits)); + } + + // ---- PER-DOMAIN concentration (the coherent-session paging headroom) ---- + println!("\n=== PER-DOMAIN CONCENTRATION (top-K% share vs uniform null) ==="); + println!("{:<10} {:>8} {:>8} {:>8} {:>7} {:>7} {:>7}", "domain", "total", "fired", "mean", "top1%", "top10%", "top25%"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let mean = if counts.is_empty() { 0.0 } else { total as f64 / counts.len() as f64 }; + println!( + "{label:<10} {total:>8} {:>8} {mean:>8.1} {:>6.1}% {:>6.1}% {:>6.1}%", + counts.len(), + top_share(&counts, total, 0.01), + top_share(&counts, total, 0.10), + top_share(&counts, total, 0.25), + ); + } + println!("(uniform null → top1%≈1, top10%≈10, top25%≈25; materially above = per-session paging headroom)"); - // Dump the affinity — the fuel for the whole optimization catalog. - let total = observer.total_hits(); - let hits = observer.snapshot_hits(); - let (_seen, cooccur) = observer.snapshot_cooccurrence(); - let predicted = observer.predicted(); + // ---- WORKING-SET SIZE: how many experts resident to capture X% of a domain's mass ---- + // THE engineering number for #180: the resident working set you must keep hot; the rest + // pages to CPU/disk (misses on the cold tail are low-mass = infrequent). + let experts_for_mass = |counts_desc: &[u64], total: u64, frac: f64| -> usize { + if total == 0 { return 0; } + let target = frac * total as f64; + let mut acc = 0u64; + for (i, c) in counts_desc.iter().enumerate() { + acc += c; + if acc as f64 >= target { return i + 1; } + } + counts_desc.len() + }; + println!("\n=== WORKING-SET SIZE (experts resident for X% of a domain's mass; % of fired) ==="); + println!("{:<10} {:>10} {:>12} {:>12} {:>12}", "domain", "50%mass", "80%mass", "90%mass", "95%mass"); + for (label, hits) in &per_domain { + let total: u64 = hits.values().sum(); + let mut counts: Vec = hits.values().copied().collect(); + counts.sort_unstable_by(|a, b| b.cmp(a)); + let n = counts.len().max(1); + let cell = |f: f64| { let k = experts_for_mass(&counts, total, f); format!("{k} ({:.0}%)", 100.0 * k as f64 / n as f64) }; + println!("{label:<10} {:>10} {:>12} {:>12} {:>12}", cell(0.50), cell(0.80), cell(0.90), cell(0.95)); + } + println!("(the 50%-mass column = the tight resident set; 95%-mass = keep-everything-hot floor; the GAP is the pageable tail)"); - println!("\n=== EXPERT AFFINITY (model-intrinsic, {n_decoded} tokens observed) ==="); - println!("total expert activations : {total}"); - println!("distinct experts fired : {}", hits.len()); - println!("co-occurring pairs seen : {}", cooccur.len()); - println!("prefetch candidates : {} experts", predicted.len()); + // ---- CROSS-DOMAIN overlap: shared base vs pageable specialization ---- + // A domain's HOT SET = the smallest set of experts capturing 50% of that domain's mass. + let hot_set = |hits: &HashMap<_, u64>| -> HashSet<_> { + let total: u64 = hits.values().sum(); + let mut ranked: Vec<(_, u64)> = hits.iter().map(|(k, v)| (*k, *v)).collect(); + ranked.sort_unstable_by(|a, b| b.1.cmp(&a.1)); + let mut acc = 0u64; + let mut set = HashSet::new(); + for (k, v) in ranked { + if (acc as f64) >= 0.5 * total as f64 { + break; + } + acc += v; + set.insert(k); + } + set + }; + let hot_sets: Vec<(String, HashSet<_>)> = + per_domain.iter().map(|(l, h)| (l.clone(), hot_set(h))).collect(); - let mut ranked: Vec<(String, u64)> = hits + // Jaccard overlap matrix of hot sets. + println!("\n=== CROSS-DOMAIN HOT-SET JACCARD (top-experts-to-50%-mass) ==="); + print!("{:<10}", ""); + for (l, _) in &hot_sets { + print!("{l:>9}"); + } + println!(); + for (la, sa) in &hot_sets { + print!("{la:<10}"); + for (_lb, sb) in &hot_sets { + let inter = sa.intersection(sb).count(); + let uni = sa.union(sb).count(); + let j = if uni > 0 { inter as f64 / uni as f64 } else { 0.0 }; + print!("{:>8.2} ", j); + } + println!(); + } + + // Shared base = experts in the hot set of ALL domains. Specialization = hot in exactly one. + let n_domains = hot_sets.len(); + let mut membership: HashMap<_, usize> = HashMap::new(); + for (_l, s) in &hot_sets { + for k in s { + *membership.entry(*k).or_insert(0) += 1; + } + } + let shared_base: HashSet<_> = membership .iter() - .map(|(k, v)| (format!("{k:?}"), *v)) + .filter(|(_, &c)| c == n_domains) + .map(|(k, _)| *k) .collect(); - ranked.sort_by(|a, b| b.1.cmp(&a.1)); - let show = ranked.len().min(24); - println!("\ntop {show} hottest experts (of {} fired):", hits.len()); - for (id, h) in ranked.iter().take(show) { - let pct = if total > 0 { 100.0 * *h as f64 / total as f64 } else { 0.0 }; - println!(" {id:<28} {h:>8} ({pct:.2}%)"); - } - // The tail: how concentrated is activation? (the paging headroom) - if ranked.len() > show { - let tail: u64 = ranked.iter().skip(show).map(|(_, h)| *h).sum(); - let tail_pct = if total > 0 { 100.0 * tail as f64 / total as f64 } else { 0.0 }; + let specialized: HashSet<_> = membership + .iter() + .filter(|(_, &c)| c == 1) + .map(|(k, _)| *k) + .collect(); + println!( + "\nshared base (hot in ALL {n_domains} domains): {} experts", + shared_base.len() + ); + println!( + "domain-specialized (hot in exactly 1) : {} experts", + specialized.len() + ); + + // The money number: per domain, what fraction of activation MASS lands on the shared base + // vs on experts NOT in any OTHER domain's hot set (the pageable eviction win). + println!("\n=== ACTIVATION MASS: shared-base vs pageable-specialized (the eviction win) ==="); + println!("{:<10} {:>14} {:>22}", "domain", "on shared base", "on own-only specialists"); + for (li, (label, hits)) in per_domain.iter().enumerate() { + let total: u64 = hits.values().sum(); + // experts that are in THIS domain's hot set and NO other domain's hot set + let own_only: HashSet<_> = hot_sets[li] + .1 + .iter() + .filter(|k| membership.get(*k).copied().unwrap_or(0) == 1) + .copied() + .collect(); + let mut base_mass = 0u64; + let mut own_mass = 0u64; + for (k, v) in hits { + if shared_base.contains(k) { + base_mass += v; + } + if own_only.contains(k) { + own_mass += v; + } + } + let pct = |m: u64| if total > 0 { 100.0 * m as f64 / total as f64 } else { 0.0 }; println!( - " ... {} colder experts share the remaining {tail_pct:.2}% (the page-out tail)", - ranked.len() - show + "{label:<10} {:>13.1}% {:>21.1}%", + pct(base_mass), + pct(own_mass) ); } + println!( + "\nRead: high own-only mass = a coherent session concentrates on experts the OTHER\n\ + domains never touch → evict them when the workload isn't that domain. High shared-base\n\ + mass = a resident core you never page. That split IS the #180 paging architecture." + ); } From 98a6a16c822e8cc5deaebcb1153a54231c84180c Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:22:44 -0500 Subject: [PATCH 12/19] =?UTF-8?q?feat(capacity):=20ModelFootprint::grid=5F?= =?UTF-8?q?lease=5Frequest=20=E2=80=94=20serving=20demand=20=E2=86=92=20ca?= =?UTF-8?q?pacity=20LeaseRequest=20(grid-overflow=20bridge)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean half of the grid-overflow seam. serving_plan reasons about MODEL RESIDENCY (can a peer hold model M's weights + per-lane KV at the served window, and how many lanes). capacity/ reasons about CONCURRENCY SPIKES (does a peer have a free lane RIGHT NOW — LeaseRequest{want_concurrency, spike_bytes}). They're orthogonal and compose — residency is the eligibility gate, concurrency is the right-now admission. Neither absorbs the other. grid_lease_request(served_window, demand_lanes) is the one-directional map from the serving side into the capacity side: demand_lanes → want_concurrency (floored at 1), and the prefill compute spike at the live served window → spike_bytes (prefill_compute_reserve(window, 1) — the transient the peer must have free to accept the hop, distinct from the resident weights+KV the residency gate already proved). No transport, no placement policy here — just the honest projection so the grid-overflow router can ask a residency-eligible peer for a concurrency lease. Test grid_lease_request_maps_demand_and_the_prefill_spike pins both mappings. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/cognition/serving_plan.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/core/continuum-core/src/cognition/serving_plan.rs b/core/continuum-core/src/cognition/serving_plan.rs index e935227bc5..9651511c70 100644 --- a/core/continuum-core/src/cognition/serving_plan.rs +++ b/core/continuum-core/src/cognition/serving_plan.rs @@ -260,6 +260,29 @@ impl ModelFootprint { self.resident_bytes(served_window, lanes) .saturating_add(self.prefill_compute_reserve(served_window, lanes)) } + + /// The capacity-fabric [`LeaseRequest`](crate::capacity::LeaseRequest) for serving + /// this model at `served_window` with `demand_lanes` concurrent minds — the bridge + /// from the serving plan's MODEL-RESIDENCY view (weights + per-lane KV) to the grid's + /// CONCURRENCY-SPIKE view, so [`GridPlacementPolicy`](crate::capacity::grid) can place + /// overflow lanes onto peers (#180 grid spill / [[frontier-is-a-scaling-question-over-misfits-not-a-capability-question]]). + /// `want_concurrency` = the minds that want a lane; `spike_bytes` = ONE lane's transient + /// prefill compute buffer at this window (the term the 2026-07-14 OOM turned on), so a + /// peer is only offered a spill lane it can actually hold. NOTE: this sizes the + /// CONCURRENCY spike only; a peer must ALSO already hold this model resident — that + /// residency gate is the gossip-side half of the routing (owned with the grid snapshot), + /// NOT this pure mapping. + pub fn grid_lease_request( + &self, + served_window: u32, + demand_lanes: u32, + ) -> crate::capacity::LeaseRequest { + crate::capacity::LeaseRequest { + consumer: self.model_id.clone(), + want_concurrency: demand_lanes.max(1), + spike_bytes: self.prefill_compute_reserve(served_window, 1), + } + } } /// The serving decision for this host. @@ -801,6 +824,26 @@ mod tests { // more lanes fit. OOM-safe — window_for still bounds it, so a higher ceiling only raises // the cap toward the budget bound, never past it. This is the "stop setting it in stone // at launch" fix that opens the elastic-lease path. + // what this catches: the serving→grid bridge (#180 spill) — a model footprint maps to a + // capacity-fabric LeaseRequest whose want_concurrency is the demanded lanes and whose + // spike_bytes is ONE lane's transient prefill compute reserve at the served window (the + // 2026-07-14 OOM term), so GridPlacementPolicy offers a peer only a spill lane it can hold. + #[test] + fn grid_lease_request_maps_demand_and_the_prefill_spike() { + let devstral = fp("devstral-24b", 14, 112 * 1024, 131_072, 3); + let window = 16_384; + let lease = devstral.grid_lease_request(window, 3); + assert_eq!(lease.consumer, "devstral-24b"); + assert_eq!(lease.want_concurrency, 3, "want_concurrency = demanded lanes"); + assert_eq!( + lease.spike_bytes, + devstral.prefill_compute_reserve(window, 1), + "spike_bytes = ONE lane's prefill compute reserve at the served window" + ); + // Zero demand floors at one lane — never a degenerate 0-concurrency lease. + assert_eq!(devstral.grid_lease_request(window, 0).want_concurrency, 1); + } + #[test] fn demand_ceiling_is_elastic_grows_for_a_hard_task_shrinks_for_a_simple_one() { let host = HostBudget { usable_bytes: 48 * GB, perf_cores: 10 }; From e5267104d2ebe919e4cae16edea8664dbb320be3 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:30:20 -0500 Subject: [PATCH 13/19] =?UTF-8?q?feat(capacity):=20ModelResidencyView=20?= =?UTF-8?q?=E2=80=94=20the=20residency=20eligibility=20gate=20for=20grid?= =?UTF-8?q?=20overflow=20(governor=20consumer=20slice=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grid-overflow routing (spill a persona's generation to a peer) has TWO orthogonal gates that COMPOSE, never absorb each other (settled with BigMama 2026-07-27): (1) RESIDENCY = eligibility — does the peer already hold model M resident? If not, accepting the hop forces a cold full-weights load (seconds to minutes), which defeats overflowing for speed. So the fast path is eligible only for peers that already hold M. THIS module. (2) CONCURRENCY = right-now admission — does the peer have a free lane? capacity/grid (LeaseRequest, LocalFirstFitPolicy). Unchanged. The one crossing point between the two abstractions is ModelFootprint::grid_lease_request (serving demand -> LeaseRequest) — one bridge, not two half-bridges. Residency deliberately does NOT live on PeerCapacity (that would blur the concurrency abstraction with a residency fact); it lives here as ModelResidencyView, keyed on Uuid exactly like gossip's capacity ledger. The governor COMPOSES the two at the placement filter: residency_eligible() returns a SMALLER snapshot (local untouched — the overflowing node holds M by definition; peers filtered to those holding M), and the unchanged capacity policy places on the survivors and applies reachability itself. Two concerns, composed at exactly one point, neither absorbed. Latest-wins replace (not merge) so a model paged OUT stops being eligible on the next beacon — a merge would resurrect evicted models and route a hop to a peer that no longer holds it. 3 tests: eligibility filter, latest-wins replace, and the residency->capacity compose end-to-end (resident+reachable gets lanes; resident+unreachable reclaimed by place(); non-resident absent from placement). Zero blast radius — new file, no existing struct touched. Next slice: populate the view from a residency beacon (gossip wiring, coordinated with BigMama — piggyback CapacityOffer vs a separate slower-cadence stream). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/capacity/mod.rs | 1 + .../src/capacity/model_residency.rs | 217 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 core/continuum-core/src/capacity/model_residency.rs diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index c9fbbb0096..5ab9baaeed 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -31,6 +31,7 @@ pub mod expert_residency; pub mod gossip; pub mod grid; pub mod lease; +pub mod model_residency; pub mod moe_serving; pub mod placement; pub mod recursion_depth; diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs new file mode 100644 index 0000000000..22bd547b23 --- /dev/null +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -0,0 +1,217 @@ +//! Model residency across the grid — WHICH peer currently holds WHICH model resident. The +//! ELIGIBILITY half of grid-overflow routing; the CONCURRENCY half is [`super::grid`]. +//! +//! ## Why this is a separate abstraction (orthogonal, never absorbed) +//! +//! `serving_plan` reasons about MODEL RESIDENCY — can a node hold model M's weights + per-lane +//! KV at the served window. `capacity/grid` reasons about CONCURRENCY SPIKES — does a node have +//! a free lane RIGHT NOW ([`super::LeaseRequest`]). Settled with BigMama 2026-07-27: these are +//! **orthogonal and compose** — neither should absorb the other, the mapping is the only +//! crossing point ([`super::serving_plan`]'s `grid_lease_request` is that one bridge). So +//! residency does NOT belong on [`super::grid::PeerCapacity`] (that would blur the concurrency +//! abstraction with a residency fact); it lives here, and the governor **composes** the two at +//! the placement filter. +//! +//! ## Why residency gates grid overflow +//! +//! A grid-overflow hop routes a persona's generation to a peer. If that peer already holds M +//! resident, the hop is fast — it needs only a free lane (the concurrency check). If it merely +//! has free VRAM but NOT M, accepting the hop forces a cold full-weights load (seconds to +//! minutes for a large model) — which defeats the entire point of overflowing for speed. So the +//! fast overflow path is eligible only for peers that already hold M. A peer with unknown +//! residency (never beaconed) is NOT eligible for the fast path — conservative by construction, +//! same spirit as [`super::residency_detect`] never claiming a promotion is faster than it is. +//! +//! ## The compose point +//! +//! [`ModelResidencyView::residency_eligible`] takes a live [`GridSnapshot`] and returns a +//! SMALLER snapshot — local device untouched (the overflowing node holds M by definition; it's +//! the one serving it), peers filtered to those holding M. The unchanged capacity placement +//! policy ([`super::grid::LocalFirstFitPolicy`]) then runs on that smaller snapshot: it never +//! learns about residency, it just sees fewer peers. Reachability stays the policy's job — an +//! unreachable-but-resident peer survives this filter and is dropped downstream by `place()`, +//! keeping the two concerns cleanly separate. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::grid::GridSnapshot; +use crate::identity::PeerId; + +/// Per-peer set of model ids that peer currently holds resident, folded from residency beacons. +/// +/// Keyed on the peer's `Uuid` (via [`PeerId::as_uuid`]) — the same choice +/// [`super::gossip::GridCapacityLedger`] makes for capacity offers, so residency and capacity +/// index the grid identically. A peer absent from the map has UNKNOWN residency (never +/// beaconed), which [`Self::holds`] reports as `false`: not eligible for the fast overflow path. +#[derive(Debug, Clone, Default)] +pub struct ModelResidencyView { + by_peer: HashMap>, +} + +impl ModelResidencyView { + pub fn new() -> Self { + Self::default() + } + + /// Record (latest-wins) the full set of models a peer currently holds resident. Latest-wins + /// because residency is a live fact — a model paged out is no longer held, so a fresh beacon + /// REPLACES the peer's set rather than merging (a merge would resurrect evicted models). + pub fn set_resident(&mut self, peer: PeerId, models: I) + where + I: IntoIterator, + S: Into, + { + self.by_peer + .insert(peer.as_uuid(), models.into_iter().map(Into::into).collect()); + } + + /// Does this peer currently hold `model_id` resident? `false` for a peer that never beaconed + /// (unknown residency) — the conservative default that keeps the fast overflow path honest. + pub fn holds(&self, peer: &PeerId, model_id: &str) -> bool { + self.by_peer + .get(&peer.as_uuid()) + .is_some_and(|set| set.contains(model_id)) + } + + /// Number of peers with a known residency beacon — probe surface, mirrors + /// [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn known_peers(&self) -> usize { + self.by_peer.len() + } + + /// Compose this residency view with a live capacity snapshot for `model_id`: keep the local + /// device (the overflowing node holds M by definition) and keep only peers that hold M + /// resident. The returned snapshot feeds the UNCHANGED capacity placement policy — concurrency + /// logic never learns about residency, it just sees a shorter peer list. Reachability is NOT + /// applied here (that stays the policy's job downstream), so an unreachable-but-resident peer + /// survives this filter and is reclaimed by `place()`. Orthogonal, composed at exactly one + /// point, neither abstraction absorbing the other. + pub fn residency_eligible(&self, snapshot: &GridSnapshot, model_id: &str) -> GridSnapshot { + let mut eligible = snapshot.clone(); + eligible.peers.retain(|peer| self.holds(&peer.peer, model_id)); + eligible + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridPlacementPolicy, GridSnapshot, LocalFirstFitPolicy, PeerCapacity}; + use crate::capacity::{DeviceCapacity, LeaseRequest}; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + // what this catches: the residency ELIGIBILITY gate. Only peers that beaconed the model as + // resident survive the filter; a peer with plenty of free VRAM but NOT holding M is dropped + // (routing to it would force a cold full-weights load — the slow path the gate exists to + // avoid), and a peer that never beaconed at all (unknown residency) is dropped too. The local + // device is always kept — the overflowing node holds M by definition. + #[test] + fn only_peers_holding_the_model_are_eligible() { + let snap = GridSnapshot { + local: dev(2), + peers: vec![ + peer(1, 40, true), // holds qwen-coder + peer(2, 40, true), // holds something else, NOT qwen-coder + peer(3, 40, true), // never beaconed — unknown residency + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "embed-small"]); + view.set_resident(peer_id(2), ["llama-70b"]); + // peer 3 intentionally not recorded. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.local, snap.local, "local device is always kept — it holds M"); + assert_eq!(eligible.peers.len(), 1, "only the peer holding qwen-coder survives"); + assert_eq!(eligible.peers[0].peer.as_uuid(), Uuid::from_u128(1)); + } + + // what this catches: latest-wins REPLACE, not merge. A peer that paged qwen-coder OUT (its + // fresh beacon lists only what it still holds) must stop being eligible — a merge would + // resurrect the evicted model and route a hop to a peer that no longer has it. + #[test] + fn fresh_beacon_replaces_so_evicted_models_stop_being_eligible() { + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder", "llama-70b"]); + assert!(view.holds(&peer_id(1), "qwen-coder")); + + // Peer paged qwen-coder out; next beacon lists only llama-70b. + view.set_resident(peer_id(1), ["llama-70b"]); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "evicted model must not linger"); + assert!(view.holds(&peer_id(1), "llama-70b")); + } + + // what this catches: the COMPOSE contract end-to-end — residency filters WHO is eligible, + // then the unchanged capacity policy places lanes on the survivors and applies reachability + // itself. A resident+reachable peer gets lanes; a resident+UNREACHABLE peer survives the + // residency filter (reachability isn't residency's job) but is reclaimed by place(); a + // non-resident peer is absent from placement entirely. Two orthogonal gates, composed. + #[test] + fn residency_then_capacity_policy_compose() { + let snap = GridSnapshot { + local: dev(1), // no local room — force the spill onto peers + peers: vec![ + peer(1, 40, true), // resident + reachable → should get lanes + peer(2, 40, false), // resident + UNREACHABLE → reclaimed by place() + peer(3, 40, true), // reachable but NOT resident → filtered out before place() + ], + }; + let mut view = ModelResidencyView::new(); + view.set_resident(peer_id(1), ["qwen-coder"]); + view.set_resident(peer_id(2), ["qwen-coder"]); + // peer 3 holds nothing relevant. + + let eligible = view.residency_eligible(&snap, "qwen-coder"); + assert_eq!(eligible.peers.len(), 2, "peers 1 & 2 are resident; peer 3 filtered out"); + + // Small spike so many lanes fit per peer — we're testing WHO gets lanes, not how many. + let req = LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: 4, + spike_bytes: GB, + }; + let placement = LocalFirstFitPolicy { safety_margin_bytes: GB }.place(&eligible, &req); + + // The reachable resident peer carries the remote lanes; the unreachable one gets none. + let peer1_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(1)) + .map(|(_, n)| *n) + .sum(); + let peer2_lanes: u32 = placement + .remote + .iter() + .filter(|(p, _)| p.as_uuid() == Uuid::from_u128(2)) + .map(|(_, n)| *n) + .sum(); + let peer3_present = placement.remote.iter().any(|(p, _)| p.as_uuid() == Uuid::from_u128(3)); + + assert!(peer1_lanes > 0, "resident + reachable peer must carry overflow lanes"); + assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); + assert!(!peer3_present, "non-resident peer never reaches placement"); + } +} From 4084e3ef7f3e277885b34d506de60486e6b159e6 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:43:01 -0500 Subject: [PATCH 14/19] =?UTF-8?q?feat(capacity):=20ResidencyBeacon=20+=20R?= =?UTF-8?q?esidencyLedger=20=E2=80=94=20the=20receive/project=20half=20of?= =?UTF-8?q?=20the=20residency=20beacon=20(governor=20consumer=20slice=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The residency sibling of capacity/gossip's CapacityOffer/GridCapacityLedger, with the same identity + freshness discipline, a different (slower) cadence, and its own payload: - ResidencyBeacon (wire): the model ids a node holds resident + a sender timestamp. Rides its OWN grid_residency EphemeralCoalesced envelope — residency changes on model page-in/out (minute-scale), NOT the 10s capacity beat, so coupling them would either over-publish residency or under-refresh capacity. Peer identity is the WIRE's, never the payload's — a peer cannot beacon residency on another's behalf. - ResidencyLedger + global_residency_ledger(): folds heard beacons (latest-per-peer wins), projects a ModelResidencyView, evicts beacons silent past RESIDENCY_EVICTION_WINDOW_MS, excludes the node's own echo (local residency is its own serving truth). view() is the exact residency analogue of GridCapacityLedger::snapshot(). Eviction is GENEROUS (6× the capacity window) precisely because the two abstractions stay orthogonal: residency is sticky, and the COMPOSED capacity snapshot already gates reachability — so a residency reading never has to prove liveness itself (that would blur residency into concurrency). A long-silent peer falls back to UNKNOWN residency (not asserted-resident), keeping the fast overflow path honest. 3 more tests (6 total in the module): heard-beacon-projects + own-echo-excluded (loopback), stale-evict-then-fresh-restore, serde round-trip (camelCase wire, catches field drift). Next slice: the publish + inbound-fold wiring — a GridResidencyModule mirroring GridCapacityModule (build the beacon from the serving plan's resident set, broadcast grid_residency) + the inbound_attach fold into global_residency_ledger(). Then a node advertises its residency and residency_eligible() runs on live grid data. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/capacity/model_residency.rs | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs index 22bd547b23..2b24475511 100644 --- a/core/continuum-core/src/capacity/model_residency.rs +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -33,7 +33,10 @@ //! keeping the two concerns cleanly separate. use std::collections::{HashMap, HashSet}; +use std::sync::OnceLock; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::grid::GridSnapshot; @@ -95,6 +98,95 @@ impl ModelResidencyView { } } +/// One node's residency beacon — the wire payload advertising which models it currently holds +/// resident. Rides its OWN `grid_residency` `EphemeralCoalesced` envelope, separate from +/// [`super::gossip::CapacityOffer`]'s `grid_capacity`: residency changes on model page-in/out +/// (minute-scale), not the 10s capacity beat, so coupling them would either over-publish +/// residency or under-refresh capacity. Names + timestamp only; peer identity is the WIRE's +/// (the transcript event's authenticated peer id), never the payload's — same rule as +/// `CapacityOffer`, so a peer cannot beacon residency on another's behalf. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResidencyBeacon { + /// Model ids this node holds resident (warm) RIGHT NOW — its base plus any co-resident. + pub resident_models: Vec, + /// Sender clock when the reading was taken (ms since epoch). Displayed, not trusted: + /// freshness is judged by RECEIVER clock at hear-time, exactly like `CapacityOffer.at_ms`. + pub at_ms: u64, +} + +/// A heard beacon + the receiver-clock instant it arrived (the freshness anchor). +#[derive(Debug, Clone)] +struct HeardBeacon { + models: Vec, + heard_at_ms: u64, +} + +/// Beacons silent past this drop from the projected view entirely. GENEROUS versus capacity's +/// eviction because the two abstractions stay orthogonal: residency is STICKY (a model stays +/// resident across many capacity beats), and the COMPOSED capacity snapshot already gates +/// reachability — so a residency reading never has to prove liveness itself (that would blur +/// residency into concurrency). 6× the capacity eviction window: a peer whose residency we +/// haven't reheard in that long falls back to UNKNOWN (not asserted-resident), keeping the fast +/// overflow path honest rather than routing to a model a long-silent peer may have paged out. +pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; + +/// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the +/// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness +/// discipline, different (slower) cadence and payload. +#[derive(Default)] +pub struct ResidencyLedger { + heard: DashMap, +} + +/// The one process-global residency ledger — the resource it mirrors (this node's view of who +/// holds what across the grid) is process-global, same granularity argument as +/// [`super::gossip::global_ledger`]. +pub fn global_residency_ledger() -> &'static ResidencyLedger { + static LEDGER: OnceLock = OnceLock::new(); + LEDGER.get_or_init(ResidencyLedger::default) +} + +impl ResidencyLedger { + /// Fold one heard beacon in (latest per peer wins — residency is a live fact). `from_peer` + /// is the transcript event's transport identity, never payload-declared. Returns `true` when + /// this peer is NEW to the ledger — the probe-on-join surface; steady re-beacons stay silent. + pub fn hear(&self, from_peer: Uuid, beacon: ResidencyBeacon, heard_at_ms: u64) -> bool { + self.heard + .insert( + from_peer, + HeardBeacon { models: beacon.resident_models, heard_at_ms }, + ) + .is_none() + } + + /// Project the ledger into a [`ModelResidencyView`] the governor composes with the capacity + /// snapshot, evicting beacons silent past [`RESIDENCY_EVICTION_WINDOW_MS`] as it goes. + /// `own_peer` is excluded — the local node's residency is its OWN serving truth (it holds M + /// by definition when it overflows), not a round-tripped beacon. + pub fn view(&self, own_peer: Uuid, now_ms: u64) -> ModelResidencyView { + let mut view = ModelResidencyView::new(); + self.heard.retain(|peer, heard| { + let age = now_ms.saturating_sub(heard.heard_at_ms); + if age > RESIDENCY_EVICTION_WINDOW_MS { + return false; // silent too long — residency falls back to unknown + } + if *peer != own_peer { + view.by_peer + .insert(*peer, heard.models.iter().cloned().collect()); + } + true + }); + view + } + + /// Number of peers currently on the ledger (self included if echoed) — probe surface, + /// mirrors [`super::gossip::GridCapacityLedger::heard_count`]. + pub fn heard_count(&self) -> usize { + self.heard.len() + } +} + #[cfg(test)] mod tests { use super::*; @@ -214,4 +306,70 @@ mod tests { assert_eq!(peer2_lanes, 0, "resident but unreachable peer is reclaimed by place()"); assert!(!peer3_present, "non-resident peer never reaches placement"); } + + fn beacon(models: &[&str], at_ms: u64) -> ResidencyBeacon { + ResidencyBeacon { + resident_models: models.iter().map(|s| s.to_string()).collect(), + at_ms, + } + } + + // what this catches: the beacon RECEIVE→PROJECT pipeline — a heard beacon projects into a + // ModelResidencyView that holds() reflects, and the node's OWN echoed beacon is excluded + // (the local node's residency is its own serving truth, arriving live, not a round-trip). + // This is the residency sibling of gossip's loopback contract. + #[test] + fn heard_beacon_projects_and_own_echo_is_excluded() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + ledger.hear(me, beacon(&["qwen-coder"], 1_000), 1_000); // our own beacon, round-tripped + ledger.hear(other, beacon(&["qwen-coder", "llama-70b"], 1_000), 1_000); + + let view = ledger.view(me, 2_000); + assert!(!view.holds(&peer_id(1), "qwen-coder"), "own echo excluded from the peer view"); + assert!(view.holds(&peer_id(2), "qwen-coder"), "the real peer's residency projects"); + assert!(view.holds(&peer_id(2), "llama-70b")); + assert_eq!(view.known_peers(), 1, "only the genuine peer, not the echo"); + } + + // what this catches: eviction after long silence — a peer we haven't reheard past the + // (generous) residency window falls back to UNKNOWN residency, so the fast overflow path + // won't route to a model that long-silent peer may since have paged out. A fresh beacon + // brings it right back (grow is first-class). + #[test] + fn stale_beacon_evicts_then_a_fresh_one_restores() { + let ledger = ResidencyLedger::default(); + let me = Uuid::from_u128(1); + let peer = Uuid::from_u128(2); + ledger.hear(peer, beacon(&["qwen-coder"], 0), 0); + assert!(ledger.view(me, 1_000).holds(&peer_id(2), "qwen-coder")); + + // Silent past the residency eviction window: gone from the view (unknown, not asserted). + let t_evict = RESIDENCY_EVICTION_WINDOW_MS + 1; + assert!( + !ledger.view(me, t_evict).holds(&peer_id(2), "qwen-coder"), + "long-silent peer's residency falls back to unknown" + ); + + // It beacons again: instantly back. + ledger.hear(peer, beacon(&["qwen-coder"], t_evict), t_evict); + assert!( + ledger.view(me, t_evict + 1).holds(&peer_id(2), "qwen-coder"), + "a returning peer's residency is adopted on its first fresh beacon" + ); + } + + // what this catches: the wire payload survives serde round-trip byte-for-byte (camelCase + // like every realtime inline payload). If resident_models renamed or at_ms narrowed, heard + // residency would silently drift from published — the grid would gate overflow on models + // nobody actually beaconed. + #[test] + fn beacon_round_trips_through_json() { + let b = beacon(&["qwen-coder", "embed-small"], 42); + let json = serde_json::to_string(&b).expect("serialize"); + assert!(json.contains("residentModels"), "camelCase wire field: {json}"); + let back: ResidencyBeacon = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(back, b, "beacon must survive the wire byte-for-byte"); + } } From dc806d5699e0fa08d1534fbc0dcaf06d6b99d913 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:51:44 -0500 Subject: [PATCH 15/19] =?UTF-8?q?feat(capacity):=20GridResidencyModule=20+?= =?UTF-8?q?=20grid=5Fresidency=20envelope=20=E2=80=94=20residency=20beacon?= =?UTF-8?q?=20publish/fold=20wiring=20(governor=20consumer=20slice=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the residency-beacon loop end-to-end, mirroring the capacity gossip path (GridCapacityModule + inbound_attach fold) exactly: - GridResidencyModule (modules/grid_residency.rs): a Background ServiceModule that every RESIDENCY_PUBLISH_INTERVAL_MS reads the daemon's live serving plan (lock-free watch snapshot — the SAME source, no parallel probe) and broadcasts a ResidencyBeacon over airc as an EphemeralCoalesced grid_residency envelope. Today the resident set is [base_model_id]; a multi-model plan extends only current_beacon(). Honest silence when no plan is computed yet (nothing to advertise). Glass box speaks on change. - AircRealtimeSchema::GridResidency (airc/realtime.rs): the new schema variant, EphemeralCoalesced like GridCapacity. ts-rs binding regenerated (AircRealtimeSchema.ts). - inbound_attach fold: residency_beacon_from_envelope decoder + the else-if that folds a heard beacon into global_residency_ledger(), keyed on the WIRE's peer id — the orthogonal sibling of the capacity fold. Own echo lands here too (the single-node loopback proof). - Registered in ipc/mod.rs right after GridCapacityModule, fed serving_daemon.subscribe(). - RESIDENCY_PUBLISH_INTERVAL_MS (model_residency.rs) = eviction/12 — the same publish:eviction ratio capacity uses, on a slower beat (residency changes minute-scale). This completes the grid-overflow governor-consumer stack: a node now ADVERTISES which models it holds, every peer folds those beacons into a ModelResidencyView, and the governor composes it with the capacity snapshot (residency_eligible -> grid_lease_request -> place -> aircPeer hop). The live two-node routing smoke (BigMama's node up + serving) validates the cross-node generation — the milestone this stack was built for. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../continuum-core/src/airc/inbound_attach.rs | 41 +++++ core/continuum-core/src/airc/realtime.rs | 11 +- .../src/capacity/model_residency.rs | 7 + core/continuum-core/src/ipc/mod.rs | 8 + .../src/modules/grid_residency.rs | 166 ++++++++++++++++++ core/continuum-core/src/modules/mod.rs | 1 + .../typescript/airc/AircRealtimeSchema.ts | 2 +- 7 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 core/continuum-core/src/modules/grid_residency.rs diff --git a/core/continuum-core/src/airc/inbound_attach.rs b/core/continuum-core/src/airc/inbound_attach.rs index 061f5d5cd9..791685df7a 100644 --- a/core/continuum-core/src/airc/inbound_attach.rs +++ b/core/continuum-core/src/airc/inbound_attach.rs @@ -212,6 +212,31 @@ pub async fn publish_transcript_event( "first capacity offer heard from a grid peer", ); } + } else if let Some(beacon) = residency_beacon_from_envelope(&envelope) { + // Residency beacon (grid-overflow eligibility): fold the heard beacon into the + // process-global residency ledger, keyed on the WIRE's peer id — the orthogonal + // sibling of the capacity fold above. Our own echo lands here too (the loopback + // proof that publish→hear works before a second node exists). + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let model_count = beacon.resident_models.len(); + let is_new = crate::capacity::model_residency::global_residency_ledger().hear( + event.peer_id.as_uuid(), + beacon, + now_ms, + ); + if is_new { + crate::probe!( + class = "grid.residency.heard", + from_peer = %event.peer_id.as_uuid(), + model_count = model_count, + heard_peers = + crate::capacity::model_residency::global_residency_ledger().heard_count(), + "first residency beacon heard from a grid peer", + ); + } } else if let Some((name, payload)) = chat_posted_from_envelope(&envelope, event) { crate::probe!( class = "airc.chat.projected", @@ -284,6 +309,22 @@ fn capacity_offer_from_envelope( serde_json::from_value(payload.inline.clone()?).ok() } +/// Decode a `grid_residency` envelope's inline payload into a [`ResidencyBeacon`]. +/// Returns `None` for any other envelope — the residency sibling of +/// [`capacity_offer_from_envelope`], same honest schema gate. +fn residency_beacon_from_envelope( + envelope: &AircRealtimeEnvelope, +) -> Option { + let crate::airc::realtime::AircRealtimePayload::ExistingSchema { payload } = &envelope.payload + else { + return None; + }; + if payload.schema != crate::airc::realtime::AircRealtimeSchema::GridResidency { + return None; + } + serde_json::from_value(payload.inline.clone()?).ok() +} + /// Project a plain airc chat message into the THIN `chat:posted` bus /// payload the positron chat projection consumes (`AircChatPosted` in /// `ipc/positron_source.rs`). Returns `None` for any non-message event diff --git a/core/continuum-core/src/airc/realtime.rs b/core/continuum-core/src/airc/realtime.rs index 1d412b1a79..8606c64a30 100644 --- a/core/continuum-core/src/airc/realtime.rs +++ b/core/continuum-core/src/airc/realtime.rs @@ -53,6 +53,13 @@ pub enum AircRealtimeSchema { /// presence-of-compute. EphemeralCoalesced: latest wins, never replayed /// (a stale capacity reading is a lie). GridCapacity, + /// A node's residency beacon (`capacity::model_residency::ResidencyBeacon`) — + /// which models it holds resident, the grid-overflow ELIGIBILITY signal (a peer + /// is only a fast overflow target for a model it already holds). Orthogonal to + /// GridCapacity (concurrency): the governor composes the two. EphemeralCoalesced + /// like capacity, but published on a slower cadence — residency changes on model + /// page-in/out (minute-scale), not the 10s capacity beat. + GridResidency, } /// Handle to a payload already defined by a Continuum schema. @@ -458,7 +465,9 @@ impl AircRealtimePayload { | AircRealtimeSchema::LiveKitBridgeEvent => AircRealtimeDelivery::Control, // Capacity offers are presence-of-compute: latest wins, never // replayed — a stale reading must not outlive its freshness. - AircRealtimeSchema::GridCapacity => AircRealtimeDelivery::EphemeralCoalesced, + AircRealtimeSchema::GridCapacity | AircRealtimeSchema::GridResidency => { + AircRealtimeDelivery::EphemeralCoalesced + } _ => AircRealtimeDelivery::Durable, }, Self::Presence { event } => event.delivery(), diff --git a/core/continuum-core/src/capacity/model_residency.rs b/core/continuum-core/src/capacity/model_residency.rs index 2b24475511..96cccc0f71 100644 --- a/core/continuum-core/src/capacity/model_residency.rs +++ b/core/continuum-core/src/capacity/model_residency.rs @@ -131,6 +131,13 @@ struct HeardBeacon { /// overflow path honest rather than routing to a model a long-silent peer may have paged out. pub const RESIDENCY_EVICTION_WINDOW_MS: u64 = 6 * super::gossip::EVICTION_WINDOW_MS; +/// The residency beacon heartbeat — deliberately SLOWER than capacity's 10s +/// ([`super::gossip::PUBLISH_INTERVAL_MS`]) because residency changes on model page-in/out +/// (minute-scale), not the free-VRAM beat. 12 beats fit inside +/// [`RESIDENCY_EVICTION_WINDOW_MS`] — the same publish:eviction ratio capacity gossip uses, +/// so a couple of dropped beacons never evicts a still-resident peer. +pub const RESIDENCY_PUBLISH_INTERVAL_MS: u64 = RESIDENCY_EVICTION_WINDOW_MS / 12; + /// Process-global ledger of heard residency beacons, keyed by the WIRE's peer id — the /// residency sibling of [`super::gossip::GridCapacityLedger`], same identity + freshness /// discipline, different (slower) cadence and payload. diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index d314ecb1f9..f50889aa23 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1782,6 +1782,14 @@ pub fn start_server( resource_daemon.clone(), default_room, ))); + // Grid residency beacon (grid-overflow eligibility, slice 3): advertise which models + // this node holds resident on a slower cadence, folded by inbound_attach into + // capacity::model_residency::global_residency_ledger. Reads the SAME serving plan the + // daemon computes (watch snapshot, no parallel probe); orthogonal sibling of capacity. + runtime.register(Arc::new(crate::modules::grid_residency::GridResidencyModule::new( + serving_daemon.subscribe(), + default_room, + ))); let continuum_root = crate::modules::persona_instance_manager::resolve_continuum_root(); let daemon_socket_for_rag_inspect = daemon_socket.clone(); let registry = crate::persona::PersonaAircRuntimeRegistry::new(); diff --git a/core/continuum-core/src/modules/grid_residency.rs b/core/continuum-core/src/modules/grid_residency.rs new file mode 100644 index 0000000000..3f5fb81f0d --- /dev/null +++ b/core/continuum-core/src/modules/grid_residency.rs @@ -0,0 +1,166 @@ +//! GridResidencyModule — this node's residency-beacon gossip publisher (grid-overflow slice 3). +//! +//! The residency sibling of [`super::grid_capacity::GridCapacityModule`]. Every +//! [`RESIDENCY_PUBLISH_INTERVAL_MS`] tick it reads the SAME live serving plan the daemon +//! already computes (one source, no parallel probe) and broadcasts a [`ResidencyBeacon`] over +//! airc as an `EphemeralCoalesced` `grid_residency` realtime envelope: which models this node +//! holds resident — the grid-overflow ELIGIBILITY signal. The receive half lives in +//! [`crate::airc::inbound_attach`], which folds heard beacons (our own echo included — the +//! loopback proof) into [`crate::capacity::model_residency::global_residency_ledger`], whose +//! `view()` IS the `ModelResidencyView` the governor composes with the capacity snapshot. +//! +//! ## Why a SEPARATE module + slower cadence +//! +//! Residency and capacity are orthogonal (settled with BigMama 2026-07-27): capacity is +//! free-VRAM RIGHT NOW (10s beat, [`super::grid_capacity`]); residency is which models are warm +//! (minute-scale, changes only on page-in/out). Coupling them onto one envelope would either +//! over-publish residency or under-refresh capacity. Same module shape as the style guide +//! mandates — no new tokio task, the runtime tick drives it; the plan read is a lock-free +//! `watch` snapshot; the publish is one small envelope through the existing +//! `airc/realtime-publish` command surface (the same path capacity + chat ride). +//! +//! ## Today's local residency = the base model +//! +//! `ServingPlan` carries a single `base_model_id` + a `resident_models` COUNT (not a per-id +//! set), so today this node's honest resident set is `[base_model_id]`. When multi-model +//! residency lands (a per-id resident list on the plan), only [`Self::current_beacon`] changes +//! — the wire, ledger, and governor compose are already model-set-shaped. + +use std::any::Any; +use std::sync::Mutex; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::{json, Value}; +use tokio::sync::watch; + +use crate::capacity::model_residency::{ + global_residency_ledger, ResidencyBeacon, RESIDENCY_PUBLISH_INTERVAL_MS, +}; +use crate::cognition::serving_plan::ServingPlan; +use crate::runtime::{ + CommandExecutor, CommandResult, LateBound, ModuleConfig, ModulePriority, ServiceModule, +}; +use airc_core::RoomId; +use std::sync::Arc; + +pub struct GridResidencyModule { + /// Lock-free live snapshot of the daemon's serving plan — the SAME source the prefill + /// valve and serving control derive from (no parallel probe). + plan_rx: watch::Receiver>, + /// The node's discovered default room — where the grid rendezvous happens today. + room: RoomId, + executor_slot: Arc>, + /// Last published resident set — the glass box speaks on CHANGE, not on every beat (a + /// steady residency is silence). `None` = never published, so the first real plan speaks. + last_published: Mutex>>, +} + +impl GridResidencyModule { + pub fn new(plan_rx: watch::Receiver>, room: RoomId) -> Self { + Self { + plan_rx, + room, + executor_slot: Arc::new(LateBound::new("grid-residency::executor")), + last_published: Mutex::new(None), + } + } + + /// Build this node's residency beacon from the live serving plan. `None` when no plan is + /// computed yet (nothing resident to advertise) — honest silence, never a fabricated set. + /// Today the resident set is `[base_model_id]`; a multi-model plan extends only this line. + fn current_beacon(&self) -> Option { + let plan = self.plan_rx.borrow().clone()?; + Some(ResidencyBeacon { + resident_models: vec![plan.base_model_id], + at_ms: now_ms(), + }) + } +} + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[async_trait] +impl ServiceModule for GridResidencyModule { + fn config(&self) -> ModuleConfig { + ModuleConfig { + name: "grid-residency", + priority: ModulePriority::Background, + command_prefixes: &[], + event_subscriptions: &[], + needs_dedicated_thread: false, + max_concurrency: 0, + tick_interval: Some(Duration::from_millis(RESIDENCY_PUBLISH_INTERVAL_MS)), + } + } + + async fn initialize(&self, _ctx: &crate::runtime::ModuleContext) -> Result<(), String> { + Ok(()) + } + + async fn handle_command(&self, command: &str, _params: Value) -> Result { + Err(format!( + "grid-residency has no command surface — '{command}' (beacons publish on the module \ + tick; read the grid via capacity::model_residency::global_residency_ledger)" + )) + } + + async fn tick(&self) -> Result<(), String> { + // Boot ordering: a beat or two may fire before start_server installs the executor — + // transient, skip (the next beat publishes). Same guaranteed-installed path as capacity. + let Some(executor) = self.executor_slot.cloned() else { + return Ok(()); + }; + let Some(beacon) = self.current_beacon() else { + // No serving plan yet — nothing resident to advertise. Honest silence. + return Ok(()); + }; + + let envelope = json!({ + "eventId": uuid::Uuid::new_v4().to_string(), + "roomId": self.room.as_uuid().to_string(), + "sourceId": "grid-residency", + "createdAtMs": beacon.at_ms, + "delivery": "ephemeral_coalesced", + "payload": { + "kind": "existing_schema", + "payload": { + "schema": "grid_residency", + "inline": serde_json::to_value(&beacon) + .map_err(|e| format!("residency beacon encode failed: {e}"))?, + } + }, + }); + executor + .execute_json("airc/realtime-publish", json!({ "envelope": envelope })) + .await + .map_err(|e| format!("grid-residency beacon publish failed: {e}"))?; + + // Glass box: speak on change (resident set differs), silent on a steady residency. + let mut last = self.last_published.lock().map_err(|e| format!("residency lock: {e}"))?; + if last.as_deref() != Some(beacon.resident_models.as_slice()) { + crate::probe!( + class = "grid.residency.beacon", + models = ?beacon.resident_models, + heard_peers = global_residency_ledger().heard_count(), + "residency beacon published to the grid", + ); + *last = Some(beacon.resident_models); + } + Ok(()) + } + + fn install_executor(&self, executor: Arc) { + self.executor_slot.install(executor); + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/core/continuum-core/src/modules/mod.rs b/core/continuum-core/src/modules/mod.rs index f146dbe500..9323d8f772 100644 --- a/core/continuum-core/src/modules/mod.rs +++ b/core/continuum-core/src/modules/mod.rs @@ -46,6 +46,7 @@ pub mod gpu; pub mod grant_issuance; pub mod grid; pub mod grid_capacity; +pub mod grid_residency; pub mod health; pub mod hippocampus; pub mod inference_coordinator_module; diff --git a/protocol/typescript/airc/AircRealtimeSchema.ts b/protocol/typescript/airc/AircRealtimeSchema.ts index f5040ab98c..a8efd11857 100644 --- a/protocol/typescript/airc/AircRealtimeSchema.ts +++ b/protocol/typescript/airc/AircRealtimeSchema.ts @@ -3,4 +3,4 @@ /** * Existing Continuum schema carried by an AIRC realtime envelope. */ -export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity"; +export type AircRealtimeSchema = "jtag_message" | "event_bridge_payload" | "grid_frame" | "live_kit_bridge_command" | "live_kit_bridge_event" | "chat_transcript" | "grid_capacity" | "grid_residency"; From c5600bf6db37f0a0424d90338088bdd54b3f4fac Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:13:34 -0500 Subject: [PATCH 16/19] =?UTF-8?q?feat(capacity):=20route=5Fgrid=5Foverflow?= =?UTF-8?q?=20=E2=80=94=20remote-only=20overflow=20placement=20decision=20?= =?UTF-8?q?(governor=20consumer=20slice=204a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DECISION half of the driver, pure + fully unit-tested; only the thin EFFECT half (the actual Commands.execute("ai/generate", {aircPeer}) hop) needs a live peer and lands at the two-node smoke. Overflow lanes are BY DEFINITION the ones ServingPlan.grid_overflow_lanes said couldn't fit locally, so their placement is REMOTE-ONLY — it must never touch LocalFirstFitPolicy's local-first >=1 floor (that floor is the local persona's OWN guaranteed lane, orthogonal to spillover; re-cramming there is the exact thrash the honest overflow signal exists to avoid). Confirmed with BigMama 2026-07-27. Two orthogonal gates, composed (never absorbed): 1. RESIDENCY (ModelResidencyView::residency_eligible) — a peer is a fast overflow target only for a model it ALREADY holds (else a cold full-weights load defeats the point). 2. CONCURRENCY (reachability + lanes_that_fit misfit-parts) — among reachable eligible peers, most-free-first, each capped by its OWN budget for the prefill spike. Unplaced lanes (no eligible+reachable peer could take them) are SURFACED in OverflowRouting.unplaced for the caller to queue/degrade on — never silently dropped ([[fallbacks-are-illegal-fail-loud]]). 4 tests: remote-only + residency/reachability gating, unplaced-surfaced-not-dropped, zero-overflow no-op, most-free-first spill spread. This completes the pure governor-consumer decision path. Remaining (slice 4b, at the live two-node smoke): read grid_overflow_lanes from the live plan, build the lease via footprint.grid_lease_request, call route_grid_overflow, and execute the aircPeer hop per placed (peer, lanes) — the only piece needing a real peer to route to. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/capacity/grid_overflow.rs | 237 ++++++++++++++++++ core/continuum-core/src/capacity/mod.rs | 1 + 2 files changed, 238 insertions(+) create mode 100644 core/continuum-core/src/capacity/grid_overflow.rs diff --git a/core/continuum-core/src/capacity/grid_overflow.rs b/core/continuum-core/src/capacity/grid_overflow.rs new file mode 100644 index 0000000000..b34283d8d8 --- /dev/null +++ b/core/continuum-core/src/capacity/grid_overflow.rs @@ -0,0 +1,237 @@ +//! Grid-overflow routing — the DECISION half of the governor consumer: given a serving plan +//! that overflowed local capacity, decide which eligible peers take the overflow lanes. The +//! EFFECT half (the actual `Commands.execute("ai/generate", {aircPeer})` hop) is thin and lives +//! at the live seam; THIS is pure, deterministic, and fully unit-tested. +//! +//! ## Why overflow placement is REMOTE-ONLY (not [`super::grid::LocalFirstFitPolicy`]) +//! +//! [`super::grid::LocalFirstFitPolicy`] fills local first and floors local at `≥1` (a resident +//! model must be able to run one prefill). That floor is correct for a FRESH placement but +//! WRONG for overflow: overflow lanes are BY DEFINITION the ones that already could not fit +//! locally (`ServingPlan.grid_overflow_lanes = demand − local_lanes`). Placing them local-first +//! would re-cram the very lanes the planner just declared didn't fit — the thrash the honest +//! "over local capacity by N" signal exists to avoid. So overflow placement never touches local: +//! it spills ONLY to eligible remote peers. +//! +//! ## The two gates, composed (never absorbed) +//! +//! 1. RESIDENCY ([`ModelResidencyView::residency_eligible`]): a peer is a fast overflow target +//! only for a model it ALREADY holds — else the hop pays a cold full-weights load, defeating +//! the point. Filters the snapshot to peers holding the model. +//! 2. CONCURRENCY (the misfit-parts fit, [`super::lanes_that_fit`]): among residency-eligible + +//! REACHABLE peers, each takes at most what its OWN free budget fits for the prefill spike — +//! the same per-node-fit rule the single-device and grid policies run. +//! +//! Reachability is applied HERE (an unreachable-but-resident peer is a memory, not an offer), +//! composing cleanly with residency without either abstraction absorbing the other. +//! +//! ## Unplaced lanes are SURFACED, never dropped +//! +//! When no eligible peer can take a lane, it lands in [`OverflowRouting::unplaced`] — the honest +//! "the grid couldn't absorb N of your overflow" signal the caller queues or degrades on. Never +//! silently swallowed ([[fallbacks-are-illegal-fail-loud]]). + +use crate::identity::PeerId; + +use super::grid::GridSnapshot; +use super::lanes_that_fit; +use super::model_residency::ModelResidencyView; +use super::LeaseRequest; + +/// The routing decision for a plan's overflow lanes: where each lane lands (remote-only) plus +/// the honest count of lanes the reachable, residency-eligible grid could NOT absorb. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OverflowRouting { + /// Overflow lanes placed on named peers, most-free-first, each capped by its own fit. + pub remote: Vec<(PeerId, u32)>, + /// Overflow lanes no eligible+reachable peer could take — queue or degrade on these, + /// never drop them silently. + pub unplaced: u32, +} + +impl OverflowRouting { + /// Total overflow lanes actually placed on peers. + pub fn placed(&self) -> u32 { + self.remote.iter().map(|(_, n)| n).sum() + } +} + +/// Decide where a plan's overflow lanes run. `lease` is the demand→capacity projection the +/// serving side already built (`ModelFootprint::grid_lease_request(served_window, overflow_lanes)`): +/// `want_concurrency` = the overflow lane count, `spike_bytes` = the per-lane prefill transient. +/// `model_id` is what the overflowing node is serving (its `ServingPlan.base_model_id`) — the +/// residency key. REMOTE-ONLY by construction (see module docs): local is already saturated. +pub fn route_grid_overflow( + model_id: &str, + lease: &LeaseRequest, + residency: &ModelResidencyView, + snapshot: &GridSnapshot, + safety_margin_bytes: u64, +) -> OverflowRouting { + let want = lease.want_concurrency; + if want == 0 { + return OverflowRouting { remote: Vec::new(), unplaced: 0 }; + } + + // Gate 1 — residency: keep only peers that hold the model resident. + let eligible = residency.residency_eligible(snapshot, model_id); + + // Gate 2 — reachability + per-node fit: reachable eligible peers, most-free-first (fewest + // peers touched), each capped by its OWN budget for the prefill spike. + let mut reachable: Vec<_> = eligible.peers.iter().filter(|p| p.reachable).collect(); + reachable.sort_by(|a, b| { + b.capacity + .gpu_free_bytes_live + .cmp(&a.capacity.gpu_free_bytes_live) + }); + + let mut remaining = want; + let mut remote = Vec::new(); + for peer in reachable { + if remaining == 0 { + break; + } + let fit = lanes_that_fit( + peer.capacity.gpu_free_bytes_live, + safety_margin_bytes, + lease.spike_bytes, + ); + let take = fit.min(remaining); + if take > 0 { + remote.push((peer.peer, take)); + remaining -= take; + } + } + + // Whatever the reachable, residency-eligible grid couldn't absorb is surfaced, not dropped. + OverflowRouting { remote, unplaced: remaining } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capacity::grid::{GridSnapshot, PeerCapacity}; + use crate::capacity::DeviceCapacity; + use uuid::Uuid; + + const GB: u64 = 1024 * 1024 * 1024; + + fn peer_id(n: u128) -> PeerId { + PeerId::from_uuid(Uuid::from_u128(n)) + } + + fn dev(free_gb: u64) -> DeviceCapacity { + DeviceCapacity { + gpu_total_bytes: 80 * GB, + gpu_free_bytes_live: free_gb * GB, + system_ram_free_bytes: 64 * GB, + } + } + + fn peer(n: u128, free_gb: u64, reachable: bool) -> PeerCapacity { + PeerCapacity { + peer: peer_id(n), + capacity: dev(free_gb), + reachable, + } + } + + fn lease(want: u32, spike_gb: u64) -> LeaseRequest { + LeaseRequest { + consumer: "qwen-coder".into(), + want_concurrency: want, + spike_bytes: spike_gb * GB, + } + } + + fn view_holding(peers: &[(u128, &[&str])]) -> ModelResidencyView { + let mut v = ModelResidencyView::new(); + for (n, models) in peers { + v.set_resident(peer_id(*n), models.iter().map(|s| s.to_string())); + } + v + } + + // what this catches: overflow placement is REMOTE-ONLY and residency+reachability gated. + // Local is never assigned lanes (it's the saturated node that overflowed). Only a peer that + // holds the model AND is reachable takes lanes; a resident-but-unreachable peer and a + // reachable-but-not-resident peer both take nothing. + #[test] + fn overflow_routes_remote_only_to_reachable_resident_peers() { + let snap = GridSnapshot { + local: dev(1), // saturated — must never receive overflow lanes + peers: vec![ + peer(1, 40, true), // resident + reachable → takes lanes + peer(2, 40, false), // resident + UNREACHABLE → nothing + peer(3, 40, true), // reachable but NOT resident → nothing + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(2, 1), &residency, &snap, GB); + + assert_eq!(routing.remote.len(), 1, "only peer 1 is eligible + reachable"); + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1)); + assert_eq!(routing.placed(), 2, "both overflow lanes fit on peer 1"); + assert_eq!(routing.unplaced, 0); + } + + // what this catches: unplaced lanes are SURFACED, never dropped. When the eligible grid + // can't fit all overflow lanes (one small peer, a big per-lane spike), the shortfall is + // reported so the caller queues/degrades — the honest "grid couldn't absorb N" signal. + #[test] + fn lanes_the_grid_cannot_absorb_are_surfaced_not_dropped() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![peer(1, 10, true)], // ~10GB free, but each lane spikes 8GB + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + // want 3 lanes, 8GB spike each: only 1 fits on the 10GB peer (net of 1GB margin). + let routing = route_grid_overflow("qwen-coder", &lease(3, 8), &residency, &snap, GB); + + assert_eq!(routing.placed(), 1, "only one lane fits the peer's budget"); + assert_eq!(routing.unplaced, 2, "the other two are surfaced, not silently dropped"); + } + + // what this catches: no overflow (want == 0) is a clean no-op — nothing placed, nothing + // unplaced. The common case (demand fit locally, grid_overflow_lanes == 0) costs nothing. + #[test] + fn zero_overflow_is_a_clean_noop() { + let snap = GridSnapshot { + local: dev(20), + peers: vec![peer(1, 40, true)], + }; + let residency = view_holding(&[(1, &["qwen-coder"])]); + + let routing = route_grid_overflow("qwen-coder", &lease(0, 1), &residency, &snap, GB); + + assert!(routing.remote.is_empty()); + assert_eq!(routing.unplaced, 0); + assert_eq!(routing.placed(), 0); + } + + // what this catches: spill spreads across peers most-free-first, each capped by its OWN fit + // (the misfit-parts rule) — 12 aggregate GB across three 4GB peers can't run a 6-lane + // placement if no single peer fits it, but distinct small lanes DO spread. Two reachable + // resident peers each take their share until demand is met. + #[test] + fn spill_spreads_across_peers_most_free_first() { + let snap = GridSnapshot { + local: dev(1), + peers: vec![ + peer(1, 20, true), // more free → sorted first + peer(2, 12, true), + ], + }; + let residency = view_holding(&[(1, &["qwen-coder"]), (2, &["qwen-coder"])]); + + // 4 lanes, 8GB spike: peer1 (20-1 margin=19 → 2 lanes), peer2 (12-1=11 → 1 lane) = 3, + // one unplaced. Peer 1 (most free) is filled before peer 2. + let routing = route_grid_overflow("qwen-coder", &lease(4, 8), &residency, &snap, GB); + + assert_eq!(routing.remote[0].0.as_uuid(), Uuid::from_u128(1), "most-free peer first"); + assert_eq!(routing.placed() + routing.unplaced, 4, "every lane accounted for"); + assert!(routing.placed() >= 3, "peers absorb what their own budgets fit"); + } +} diff --git a/core/continuum-core/src/capacity/mod.rs b/core/continuum-core/src/capacity/mod.rs index 5ab9baaeed..8a4b977aad 100644 --- a/core/continuum-core/src/capacity/mod.rs +++ b/core/continuum-core/src/capacity/mod.rs @@ -30,6 +30,7 @@ pub mod expert_reconcile; pub mod expert_residency; pub mod gossip; pub mod grid; +pub mod grid_overflow; pub mod lease; pub mod model_residency; pub mod moe_serving; From 979d93dc4d38f81b47cf19130b4045a6f9640162 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:26:02 -0500 Subject: [PATCH 17/19] =?UTF-8?q?feat(persona):=20grid-overflow=20effector?= =?UTF-8?q?=20seam=20=E2=80=94=20materialize=5Fadapters=20overflow=5Fadapt?= =?UTF-8?q?er=5Ffor=20override=20(governor=20consumer=20slice=204b-i)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composition point where route_grid_overflow's decision becomes a real remote brain. materialize_adapters gains an `overflow_adapter_for(&profile, slot)` closure (same closure-DI shape as runtime_lookup / tool_executor_for): return Some(remote adapter) when the governor routed this persona off-box — her node is over capacity and a reachable peer holds her model — so her brain runs on that peer via AircRemoteInferenceAdapter; None → build the local adapter from the factory (the common case). This is the exact re-home seam the DeliberationModelBinding was designed for (its doc: re-home = "a new adapter / grid failover onto another node"). The remote adapter registers in the global provider registry by model_id just like the local one, so evaluate_response reaches it transparently — the persona doesn't know or care that her inference crosses the grid. host.rs passes `|_,_| None` for now (slice 4b-ii wires the live capacity + residency + airc closure at the ipc bootstrap, where the serving plan + airc handle live). Test overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory pins the contract: when the override supplies a slot's adapter, the local factory is NOT called for it (build_count == 1 of 2), both personas host, both warm. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/persona/host.rs | 4 + core/continuum-core/src/persona/supervisor.rs | 103 ++++++++++++++---- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index d42ad0df19..6731d7062d 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -328,6 +328,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, + // Grid-overflow effector closure (slice 4b-ii wires the live capacity + + // residency + airc context here). Until then every persona builds her + // local adapter — no off-box routing, the pre-effector behavior. + |_profile, _slot| None, ) .await; diff --git a/core/continuum-core/src/persona/supervisor.rs b/core/continuum-core/src/persona/supervisor.rs index 66471a2d4d..d55a966b1c 100644 --- a/core/continuum-core/src/persona/supervisor.rs +++ b/core/continuum-core/src/persona/supervisor.rs @@ -499,6 +499,14 @@ pub async fn materialize_adapters( uuid::Uuid, ) -> Option>, + // The grid-overflow EFFECTOR (governor consumer slice 4b). Given a persona's + // profile + slot, returns `Some(remote adapter)` when the governor routed her + // off-box — her node is over local capacity and a reachable peer holds her model + // (`capacity::grid_overflow::route_grid_overflow`) — so her brain runs on that + // peer via `AircRemoteInferenceAdapter`. `None` → build the local adapter from the + // factory (the common case; demand fit locally). Closure DI keeps the supervisor + // decoupled from the capacity fabric + airc handle, same shape as the lookups above. + overflow_adapter_for: impl Fn(&PersonaInferenceProfile, usize) -> Option>, ) -> Vec> { let mut out = Vec::with_capacity(plans.len()); for (slot_index, plan) in plans.into_iter().enumerate() { @@ -525,16 +533,22 @@ pub async fn materialize_adapters( continue; } }; - let adapter = match factory.build_adapter(&profile).await { - Ok(a) => a, - Err(message) => { - out.push(Err(SupervisorError::AdapterFactory { - slot_index, - role: plan.role, - message, - })); - continue; - } + // Grid-overflow effector: if the governor routed this persona off-box, her + // adapter is the airc-remote one (her brain runs on the peer that holds her + // model). Else build the local adapter from the factory — the common case. + let adapter = match overflow_adapter_for(&profile, slot_index) { + Some(remote) => remote, + None => match factory.build_adapter(&profile).await { + Ok(a) => a, + Err(message) => { + out.push(Err(SupervisorError::AdapterFactory { + slot_index, + role: plan.role, + message, + })); + continue; + } + }, }; // Warm the adapter's KV-cache / kernels BEFORE the persona // enters her service loop. Per [[init-once-handle-then-lease-zero-copy-refs]] @@ -1106,7 +1120,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); assert_eq!(factory.build_count(), 2); @@ -1153,7 +1167,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory called exactly once — for the Ok row only. @@ -1188,7 +1202,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::always_fails("simulated factory rejection"); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1211,7 +1225,7 @@ mod tests { #[tokio::test] async fn empty_plans_yields_empty_hosted() { let factory = ScriptedPersonaAdapterFactory::heuristic(); - let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(vec![], &factory, |_| None, |_| None, |_, _| None).await; assert!(hosted.is_empty()); assert_eq!(factory.build_count(), 0); } @@ -1240,7 +1254,7 @@ mod tests { let factory = ScriptedPersonaAdapterFactory::heuristic(); // `|_| None` here is the substrate-bug shape we're locking in: // the registry exists but doesn't contain this persona_id. - let hosted = materialize_adapters(plans, &factory, |_| None, |_| None).await; + let hosted = materialize_adapters(plans, &factory, |_| None, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); // Factory MUST NOT be called when the runtime lookup fails — @@ -1300,7 +1314,7 @@ mod tests { as Arc) } }; - let hosted = materialize_adapters(plans, &factory, lookup, |_| None).await; + let hosted = materialize_adapters(plans, &factory, lookup, |_| None, |_, _| None).await; assert_eq!(hosted.len(), 2); // Factory ran exactly once — for Paige, not Pax. @@ -1345,7 +1359,7 @@ mod tests { let (factory, counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; // Both slots materialize cleanly. assert_eq!(hosted.len(), 2); @@ -1359,6 +1373,56 @@ mod tests { ); } + // what this catches: the grid-overflow EFFECTOR seam — when the governor routes a + // persona off-box, `overflow_adapter_for` supplies her adapter (the airc-remote one, + // her brain on a peer) and the LOCAL factory is NOT called for that slot. Here slot 0 + // is overflow-routed (override returns a stand-in remote), slot 1 is local. Both host; + // the factory builds ONLY slot 1 (build_count == 1); both adapters still warm. This is + // the composition point where route_grid_overflow's decision becomes a real remote brain. + #[tokio::test] + async fn overflow_effector_supplies_remote_adapter_and_bypasses_the_local_factory() { + use crate::ai::heuristic_adapter::HeuristicInferenceAdapter; + let plans = vec![ + MaterializedPersonaPlan { + role: RoleId::Helper, + instance: fake_instance("Paige"), + profile: Ok(fake_profile("Paige", "model-a")), // slot 0 → overflow-routed + }, + MaterializedPersonaPlan { + role: RoleId::Coder, + instance: fake_instance("Pax"), + profile: Ok(fake_profile("Pax", "model-b")), // slot 1 → local + }, + ]; + + let (factory, _counts) = ScriptedPersonaAdapterFactory::heuristic_with_counters(); + let hosted = materialize_adapters( + plans, + &factory, + StubAircCitizen::fresh_lookup(), + |_| None, + // Overflow effector: slot 0 is routed off-box → supply a stand-in "remote" + // adapter; every other slot stays local (None). + |_profile, slot| { + if slot == 0 { + Some(Arc::new(HeuristicInferenceAdapter::new()) as Arc) + } else { + None + } + }, + ) + .await; + + assert_eq!(hosted.len(), 2); + assert!(hosted.iter().all(|r| r.is_ok()), "both personas host (one remote, one local)"); + assert_eq!( + factory.build_count(), + 1, + "the local factory builds ONLY the non-overflow slot — the overflow slot's \ + adapter came from the effector, her brain runs on the peer" + ); + } + /// Warmup failure surfaces as `SupervisorError::AdapterWarmup` — /// the persona does NOT reach hosted state. Per [[no-fallbacks-ever]] /// an adapter that refuses to warm gets a typed slot failure; @@ -1375,7 +1439,7 @@ mod tests { "simulated warmup failure", ); let hosted = - materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None).await; + materialize_adapters(plans, &factory, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None).await; assert_eq!(hosted.len(), 1); match &hosted[0] { @@ -1411,7 +1475,7 @@ mod tests { profile: Ok(fake_profile("Paige", "model-a")), }]; let hosted_ok = - materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None) + materialize_adapters(ok_plan, &factory_ok, StubAircCitizen::fresh_lookup(), |_| None, |_, _| None) .await; assert!(hosted_ok[0].is_ok(), "ok-warmup adapter materializes"); assert_eq!(ok_counts.warmups(), 1); @@ -1429,6 +1493,7 @@ mod tests { &factory_fail, StubAircCitizen::fresh_lookup(), |_| None, + |_, _| None, ) .await; assert!( From b7b526455103d89acc82a9fbb0c6f4ebad26b676 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 15:38:33 -0500 Subject: [PATCH 18/19] =?UTF-8?q?feat(persona):=20grid-overflow=20effector?= =?UTF-8?q?=20LIVE=20closure=20=E2=80=94=20a=20persona's=20brain=20routes?= =?UTF-8?q?=20off-box=20to=20a=20residency-eligible=20peer=20(governor=20c?= =?UTF-8?q?onsumer=20slice=204b-ii)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last mile. build_overflow_effector (persona/grid_overflow_effector.rs) composes the whole tested decision path into the per-persona adapter override the supervisor consumes, wired at the ipc bootstrap spawn: live serving plan (grid_overflow_lanes) → footprint from live_candidates → grid_lease_request → global_residency_ledger().view → route_grid_overflow → AircLiveTransport(airc, peer) → AircRemoteInferenceAdapter When the node is over local capacity and a reachable peer already holds a persona's model, her DeliberationModelBinding.adapter becomes the airc-remote one — her inference crosses the grid transparently (the re-home the binding was designed for), and she lives in the room as a peer hosted on another machine. That's "a competent peer so it's not just us there." DEFENSIVE by construction (safe to ship pre-smoke): returns None → local adapter on ANY uncertainty (airc not attached, no plan, no overflow, no matching footprint, no eligible reachable peer). Can only be a safe no-op or a correct off-box route — never a self-route (own peer excluded via airc.peer_id(), so its own residency-beacon loopback can't pick itself) and never a panic. The only unit-unprovable part is that the remote hop SUCCEEDS — the live two-node smoke validates that; a hop that can't warm surfaces as a loud AdapterWarmup slot failure, never a silent local downgrade ([[fallbacks-are-illegal-fail-loud]]). Plumbing: overflow_adapter_for threaded through spawn_all → materialize_adapters (4b-i seam); dedicated Arc clones of the airc-interceptor cell + serving daemon so the boot-spawn async-move capture doesn't strand the interceptor + reconcile task; live_candidates() → pub(crate). Completes the grid-overflow governor consumer end-to-end. Live validation + the cross-node generation run the moment BigMama's node is serving (model_id string-equality is the one thing to confirm live). All unit paths green (21 supervisor/overflow/residency tests). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/ipc/mod.rs | 26 +++- .../src/modules/serving_daemon.rs | 2 +- .../src/persona/grid_overflow_effector.rs | 115 ++++++++++++++++++ core/continuum-core/src/persona/host.rs | 17 ++- core/continuum-core/src/persona/mod.rs | 1 + 5 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 core/continuum-core/src/persona/grid_overflow_effector.rs diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index f50889aa23..c05b32e7e6 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -2108,6 +2108,15 @@ pub fn start_server( // below). Subscribe BEFORE the spawn so no plan edge is missed while // the task waits on the executor-ready oneshot. let mut serving_plan_rx = serving_daemon.subscribe(); + // A DEDICATED clone of the interceptor's airc-handle cell for the grid-overflow + // effector (slice 4b): the boot-spawn `async move` below captures by move, and the + // interceptor still needs the original `airc_interceptor_cell` further down — so the + // effector rides its own Arc clone of the SAME shared cell (both see the handle once + // attach_as fills it). + let overflow_airc_cell = airc_interceptor_cell.clone(); + // Same reason for the serving daemon: the reconcile task below still needs the + // original `serving_daemon` handle, so the effector rides its own clone. + let overflow_serving = serving_daemon.clone(); rt_handle.spawn(async move { // Wait for the IPC thread to deliver the WIRED executor (this both // gates ordering AND hands us the executor the personas' hands ride). @@ -2195,7 +2204,22 @@ pub fn start_server( } else { attempt += 1; let summary = supervisor - .spawn_all(&mut provider, Some(tool_executor.clone())) + .spawn_all( + &mut provider, + Some(tool_executor.clone()), + // Grid-overflow effector (slice 4b): the LIVE closure. Reads + // the serving plan's grid_overflow_lanes, filters residency- + // eligible reachable peers (from the beacon ledger), runs + // route_grid_overflow, and re-homes the persona's adapter to an + // AircRemoteInferenceAdapter over the interceptor's airc handle. + // DEFENSIVE: None on any uncertainty → local adapter (no + // regression); can only be a safe no-op or a correct off-box + // route (self excluded via airc.peer_id()). + crate::persona::grid_overflow_effector::build_overflow_effector( + overflow_airc_cell.clone(), + overflow_serving.clone(), + ), + ) .await; if summary.hosted > 0 { tracing::info!( diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index 452f91e4c3..90e8902ed2 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -560,7 +560,7 @@ impl ServingDaemonModule { /// one candidate, so the reconcile serves that model or (if it has dropped off /// disk) nothing. Suppress subtracts; pin intersects; the planner still owns /// the choice among whatever remains. - fn live_candidates(&self) -> Vec { + pub(crate) fn live_candidates(&self) -> Vec { let suppressed = self.suppressed.borrow(); let pinned = self.pinned.borrow(); servable_candidates(&self.catalog.snapshot(), &**suppressed, &pinned) diff --git a/core/continuum-core/src/persona/grid_overflow_effector.rs b/core/continuum-core/src/persona/grid_overflow_effector.rs new file mode 100644 index 0000000000..4acc4a7286 --- /dev/null +++ b/core/continuum-core/src/persona/grid_overflow_effector.rs @@ -0,0 +1,115 @@ +//! The grid-overflow EFFECTOR (governor consumer slice 4b-ii) — composes the tested +//! decision path into the live per-persona adapter override that +//! [`super::supervisor::materialize_adapters`] consumes. +//! +//! When the node is over local capacity (`ServingPlan.grid_overflow_lanes > 0`) and a +//! reachable peer already holds a persona's model, this routes HER BRAIN to that peer: her +//! `DeliberationModelBinding.adapter` becomes an [`AircRemoteInferenceAdapter`] over airc, so +//! her inference crosses the grid transparently — the exact re-home the binding was designed +//! for. The persona doesn't know or care that her model runs on another machine. +//! +//! ## Defensive by construction — safe to ship before the live smoke +//! +//! The closure returns `None` (→ the local factory adapter, zero behavior change) on ANY +//! uncertainty: airc not yet attached, no serving plan, no overflow, no matching footprint, or +//! no residency-eligible reachable peer. So it can only ever be a **safe no-op or a correct +//! remote route** — never a self-route (this node's own peer is excluded via `airc.peer_id()`, +//! so its own residency-beacon loopback can't select itself) and never a panic. The one thing +//! the unit path can't prove is that the remote hop SUCCEEDS — that is what the live two-node +//! smoke validates; a hop that can't warm surfaces as a loud per-slot `AdapterWarmup` failure +//! ([[fallbacks-are-illegal-fail-loud]]), never a silent local downgrade. + +use std::sync::Arc; + +use airc_lib::Airc; +use tokio::sync::OnceCell; + +use crate::ai::adapter::AIProviderAdapter; +use crate::capacity::gossip::global_ledger; +use crate::capacity::grid_overflow::route_grid_overflow; +use crate::capacity::model_residency::global_residency_ledger; +use crate::capacity::DeviceCapacity; +use crate::inference::airc_remote::adapter::AircRemoteInferenceAdapter; +use crate::inference::airc_remote::transport::AircLiveTransport; +use crate::modules::serving_daemon::ServingDaemonModule; +use crate::persona::inference_profile::PersonaInferenceProfile; + +/// Headroom kept free on a peer before it may accept an overflow lane — same 1 GiB spirit as +/// the single-device [`crate::capacity::grid::LocalFirstFitPolicy`] safety margin. +const OVERFLOW_SAFETY_MARGIN_BYTES: u64 = 1024 * 1024 * 1024; + +fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Build the effector closure `spawn_all` / `materialize_adapters` consume. Captures the +/// late-bound airc handle cell (shared with the interceptor) and the serving daemon (the live +/// plan + footprint source). See the module docs for the defensive contract. +pub fn build_overflow_effector( + airc_cell: Arc>>, + serving: Arc, +) -> impl Fn(&PersonaInferenceProfile, usize) -> Option> { + move |profile, _slot| { + // airc not attached yet → local (the boot window before attach_as fills the cell). + let airc = airc_cell.get()?.clone(); + // No plan, or demand fits locally → local. grid_overflow_lanes is the honest + // "over local capacity by N" signal; 0 means nothing to spill. + let plan = serving.compute_plan()?; + if plan.grid_overflow_lanes == 0 { + return None; + } + // The footprint for THIS persona's model — the lease's per-lane prefill spike. + // Absent (model not a live candidate) → local, never a fabricated footprint. + let footprint = serving + .live_candidates() + .into_iter() + .find(|f| f.model_id == profile.model_id)?; + let lease = + footprint.grid_lease_request(plan.served_context_window, plan.grid_overflow_lanes); + + // Own peer id from the airc handle → excludes self from the residency view + gossip + // snapshot (this node's own beacon loopback must never select itself as the target). + let own = airc.peer_id().as_uuid(); + let now = now_ms(); + let residency = global_residency_ledger().view(own, now); + // Overflow placement is REMOTE-ONLY, so local capacity is never read — a zeroed local + // is the honest input (route_grid_overflow only ever inspects the peer list). + let snapshot = global_ledger().snapshot( + own, + DeviceCapacity { + gpu_total_bytes: 0, + gpu_free_bytes_live: 0, + system_ram_free_bytes: 0, + }, + now, + ); + + let routing = route_grid_overflow( + &profile.model_id, + &lease, + &residency, + &snapshot, + OVERFLOW_SAFETY_MARGIN_BYTES, + ); + // First-cut assignment: this persona takes the first placed peer. No residency-eligible + // reachable peer with room → local (queue/degrade is the planner's, not a silent drop). + let (peer, _lanes) = routing.remote.first()?; + let peer_uuid = peer.as_uuid(); + + let transport = AircLiveTransport::new(airc, peer_uuid); + let adapter = AircRemoteInferenceAdapter::new(transport); + crate::probe!( + class = "grid.overflow.route", + persona = %profile.persona_name, + model = %profile.model_id, + peer = %peer_uuid, + overflow_lanes = plan.grid_overflow_lanes, + "routing persona brain OFF-BOX to a residency-eligible peer (grid overflow)", + ); + Some(Arc::new(adapter) as Arc) + } +} diff --git a/core/continuum-core/src/persona/host.rs b/core/continuum-core/src/persona/host.rs index 6731d7062d..c9ec318822 100644 --- a/core/continuum-core/src/persona/host.rs +++ b/core/continuum-core/src/persona/host.rs @@ -264,6 +264,15 @@ impl PersonaSpawnSupervisor { // persona's HANDS are built over it (identity-scoped), so the ACL gates // what they may do. `None` → personas spawn speak-only (no hands). tool_command_executor: Option>, + // The grid-overflow effector (governor consumer slice 4b): given a persona's + // profile + slot, `Some(remote adapter)` routes her brain to a peer that holds + // her model (the ipc bootstrap builds this from the live serving plan + residency + // ledger + airc handle); `None` → local adapter. Forwarded verbatim to + // `materialize_adapters`. `|_, _| None` is the pre-effector (all-local) behavior. + overflow_adapter_for: impl Fn( + &crate::persona::inference_profile::PersonaInferenceProfile, + usize, + ) -> Option>, ) -> BootSummary { let plans = match bootstrap_planned( &self.spawner, @@ -328,10 +337,10 @@ impl PersonaSpawnSupervisor { as Arc }) }, - // Grid-overflow effector closure (slice 4b-ii wires the live capacity + - // residency + airc context here). Until then every persona builds her - // local adapter — no off-box routing, the pre-effector behavior. - |_profile, _slot| None, + // Grid-overflow effector: the ipc bootstrap supplies the live decision + // (capacity + residency + airc). `|_, _| None` from a caller that doesn't + // route keeps the pre-effector all-local behavior. + overflow_adapter_for, ) .await; diff --git a/core/continuum-core/src/persona/mod.rs b/core/continuum-core/src/persona/mod.rs index b69f5930e4..008a907c3a 100644 --- a/core/continuum-core/src/persona/mod.rs +++ b/core/continuum-core/src/persona/mod.rs @@ -50,6 +50,7 @@ pub mod evaluator; pub mod focus; pub mod genome_paging; pub mod home; +pub mod grid_overflow_effector; pub mod host; pub mod hw_tier_descriptor; pub mod identity_provider; From f829ed6be12cd2b0a1fe5eda95aa8670b63f021e Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 16:14:40 -0500 Subject: [PATCH 19/19] =?UTF-8?q?fix(ipc):=20interceptor-attach=20must=20u?= =?UTF-8?q?se=20rt=5Fhandle.spawn,=20not=20bare=20tokio::spawn=20=E2=80=94?= =?UTF-8?q?=20the=20airc=20handle=20cell=20never=20filled=20(BigMama=20dia?= =?UTF-8?q?gnosed=20the=20panic)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At ipc/mod.rs the AircInterceptor bootstrap spawned its `Airc::attach_as` task with bare `tokio::spawn`. But this runs in `start_server` on the IPC thread, and the `rt_handle.enter()` guard (line ~1111) is SCOPED and has already dropped by here — so there is NO ambient tokio runtime and `tokio::spawn` panics "there is no reactor running, must be called from the context of a Tokio 1.x runtime". That panic killed the attach task, so the `OnceCell>` never filled, and EVERYTHING that reads it silently no-oped: the AircInterceptor's aircPeer command routing (send side) AND the grid-overflow effector (build_overflow_effector reads the same cell → always None → local, never routes). Fix: `rt_handle.spawn` (rt_handle is a start_server param, in scope; the SAME call 1498/2111 already use) — it targets the runtime by handle without needing ambient context. This is the sender-side unblock for the whole cross-node persona-serving path: without the handle, a node can never route an ai/generate to a peer. BigMama diagnosed the panic live 2026-07-27; this is the one-line fix on the continuum side. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/ipc/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/continuum-core/src/ipc/mod.rs b/core/continuum-core/src/ipc/mod.rs index c05b32e7e6..9e7e3422aa 100644 --- a/core/continuum-core/src/ipc/mod.rs +++ b/core/continuum-core/src/ipc/mod.rs @@ -1701,7 +1701,16 @@ pub fn start_server( if let Some((interceptor_daemon_socket, _room)) = interceptor_airc_deps { let cell = airc_interceptor_cell.clone(); let root = crate::modules::persona_instance_manager::resolve_continuum_root(); - tokio::spawn(async move { + // MUST be `rt_handle.spawn`, NOT bare `tokio::spawn`: this runs during + // start_server on the IPC thread, and the `rt_handle.enter()` guard (above) is + // scoped and has already dropped by here — so there is NO ambient runtime and + // `tokio::spawn` panics "there is no reactor running". That panic kills the + // attach task, the OnceCell never fills, and EVERYTHING that reads it silently + // no-ops: the AircInterceptor's aircPeer routing AND the grid-overflow effector + // (build_overflow_effector reads this same cell). rt_handle.spawn targets the + // runtime by handle without needing ambient context. (BigMama diagnosed the + // panic 2026-07-27; this is the one-line fix.) + rt_handle.spawn(async move { match airc_lib::Airc::attach_as(root, "continuum-airc-interceptor", interceptor_daemon_socket) .await {