From aa47f37b634eba399c63cd3a9de081a0c35d6916 Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:43:47 -0500 Subject: [PATCH 1/3] =?UTF-8?q?feat(serving):=20opt-in=20KV=20cache=20quan?= =?UTF-8?q?tization=20=E2=80=94=20q8=5F0=20halves=20KV,=20feeds=20the=20el?= =?UTF-8?q?astic=20window=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds SERVING_KV_CACHE_TYPE (config, default f16/off): when set to q8_0 (or q4_0) the llama-server lane runs --cache-type-k/v , cutting resident KV ~in half at near-lossless quality. That frees memory the elastic window (#234) can spend on a bigger context or more warm lanes — faster for multiple personas AND more room for hard coding, the same "faster + best code" pair. OFF by default and safe-by-construction: absent / f16 → byte-identical f16 launch (no behavior change), so this can't destabilize a backend whose build lacks Metal KV-quant kernels — enabling it is an explicit operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). Follow-up (noted in code): to have the PLAN grow the window on the freed memory rather than leave it as extra headroom, footprint_for must scale kv_per_token by the quant factor. This slice is the safe enablement; that fit-math coupling is the next step, and wants a live burst on a KV-quant-capable backend to validate quality + the speedup. continuum-core lib compiles clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/inference/llama_server.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index f3a82f8bc..7c622900d 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -2225,6 +2225,25 @@ impl LlamaServerControl for LlamaServerProcess { // surfaces the real defect: a RAG budget that overshot the served // window ([[fallbacks-are-illegal-fail-loud]]). .arg("--no-context-shift"); + // KV CACHE QUANTIZATION (#232, opt-in field-proven technique). f16 KV is the + // default; q8_0 is ~half the resident KV footprint at near-lossless quality, + // freeing memory the elastic window (#234) can spend on a BIGGER context or MORE + // warm lanes — faster for multiple personas AND more room for hard coding. + // OFF by default: not every backend/build ships Metal KV-quant kernels, so this + // is an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). + // Set SERVING_KV_CACHE_TYPE=q8_0 (or q4_0) to enable; absent / `f16` → byte-identical + // f16 behavior. NOTE: to have the plan actually GROW the window on the freed memory + // (not just leave it as extra headroom), the fit math must also scale kv_per_token — + // that footprint coupling is the follow-up; this slice is the safe enablement. + if let Some(kv_type) = crate::config_env::read("SERVING_KV_CACHE_TYPE") + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty() && s != "f16") + { + cmd.arg("--cache-type-k") + .arg(&kv_type) + .arg("--cache-type-v") + .arg(&kv_type); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render From 5ed66cd07fa9e0b5dd96a7163c7b0ee20b1c0f7b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 13:51:10 -0500 Subject: [PATCH 2/3] =?UTF-8?q?feat(serving):=20KV-quant=20fit=20coupling?= =?UTF-8?q?=20=E2=80=94=20the=20window=20GROWS=20into=20the=20freed=20KV?= =?UTF-8?q?=20memory=20(#232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the KV-quant feature. The launcher flag (prior commit) makes a lane run q8_0 KV; this makes the PLAN know it: footprint_for scales kv_per_token by the quant divisor, so the served window is sized against the KV the lane WILL actually hold, and the elastic window (#234) grows into the freed memory instead of leaving it idle. - kv_divisor_for (pure, env-free, unit-tested): f16/unset/unknown → 1 (no change), q8_0 → 2, q4_0/q4_1 → 3. CONSERVATIVE by design — under the ideal ~3.5x for q4 — so the plan can never over-grow the window past the real KV and OOM (over-reserve = smaller window = safe). - Applied only in the config-aware footprint_for; footprint_from_parts stays pure so its tests are env-independent. Same SERVING_KV_CACHE_TYPE key as the launcher — one config, two consumers (flag + fit rate), documented to stay in sync. Default (f16 / unset) → divisor 1 → byte-identical: this can't change serving on a box that doesn't opt in. Test pins the mapping + the safe-default. Wants a live burst on a KV-quant-capable backend to confirm quality + the actual window growth. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- .../src/modules/serving_daemon.rs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/core/continuum-core/src/modules/serving_daemon.rs b/core/continuum-core/src/modules/serving_daemon.rs index 8a46a5bc3..fb2c7d370 100644 --- a/core/continuum-core/src/modules/serving_daemon.rs +++ b/core/continuum-core/src/modules/serving_daemon.rs @@ -2663,12 +2663,38 @@ fn moe_host_cache_lease_inputs( pub fn footprint_for(model: &Model) -> Option { let path = crate::model_registry::artifacts::resolve_gguf_for_model(model)?; let weights_bytes = std::fs::metadata(&path).ok()?.len(); - footprint_from_parts( + let mut fp = footprint_from_parts( &model.id, weights_bytes, model.context_window, model.has(Capability::ToolUse), - ) + )?; + // KV CACHE QUANTIZATION (#232): a lane running quantized KV holds proportionally + // fewer bytes/token, so the plan can size a BIGGER window into the same budget — + // this is what turns the launcher's opt-in q8_0 flag into an actual window GROWTH. + // Divide the f16 rate by the quant factor; default (f16 / unset) → 1 → byte-identical. + // Keep the config key in sync with the launcher arg in inference/llama_server.rs — + // one SERVING_KV_CACHE_TYPE key, two consumers (launcher flag + this fit-math rate). + fp.kv_per_token = (fp.kv_per_token / kv_cache_quant_divisor()).max(1); + Some(fp) +} + +/// The resident-KV divisor implied by `SERVING_KV_CACHE_TYPE`, so the plan sizes the +/// served window against the KV the lane WILL actually hold, not the f16 default. (#232) +fn kv_cache_quant_divisor() -> u64 { + kv_divisor_for(crate::config_env::read("SERVING_KV_CACHE_TYPE").as_deref()) +} + +/// Pure KV-rate divisor for a cache-type string (testable without env). CONSERVATIVE by +/// design: q8_0 ≈ half of f16 → 2; q4_0/q4_1 ≈ a third → 3 (under the ideal ~3.5×, so the +/// plan never over-grows the window past the real KV and OOMs). Anything else / f16 → 1 +/// (no change). Over-reserve is a smaller window (safe); under-reserve is an OOM (fatal). +fn kv_divisor_for(cache_type: Option<&str>) -> u64 { + match cache_type.map(|s| s.trim().to_ascii_lowercase()).as_deref() { + Some("q8_0") => 2, + Some("q4_0") | Some("q4_1") => 3, + _ => 1, + } } /// Pure footprint estimate from the fields that drive it — split out from the @@ -3320,6 +3346,19 @@ mod tests { // what this catches: footprint estimate is honest about weights (passed // through), tool capability bumps the rank, KV is non-zero, and zero // weights → no footprint (we only offer what we can actually serve). + #[test] + fn kv_divisor_reflects_cache_type_conservatively() { + // what this catches: the #232 KV-quant fit-math coupling — the served window grows + // only when the lane actually runs quantized KV, and CONSERVATIVELY so the plan + // never over-grows past the real KV and OOMs. f16/unset/unknown must never scale. + assert_eq!(kv_divisor_for(None), 1, "unset never scales the window"); + assert_eq!(kv_divisor_for(Some("f16")), 1, "explicit f16 is the no-op default"); + assert_eq!(kv_divisor_for(Some("q8_0")), 2, "q8_0 ~ half of f16"); + assert_eq!(kv_divisor_for(Some(" Q8_0 ")), 2, "trimmed + case-insensitive"); + assert_eq!(kv_divisor_for(Some("q4_0")), 3, "q4_0 conservative, under the ideal ~3.5x"); + assert_eq!(kv_divisor_for(Some("garbage")), 1, "unknown type → no grow, never a bogus OOM"); + } + #[test] fn footprint_from_parts_is_footprint_aware() { let fp = footprint_from_parts("present", 3 * GB, 8192, true).unwrap(); From eac5800d60db72c9288b9b0d98db4a65ab87956b Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Mon, 27 Jul 2026 14:07:29 -0500 Subject: [PATCH 3/3] =?UTF-8?q?feat(serving):=20opt-in=20flash=20attention?= =?UTF-8?q?=20=E2=80=94=20faster=20prefill+decode,=20lower=20memory=20(#23?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fused attention kernel is faster on BOTH prefill and decode and lowers peak memory — directly attacking the prefill-bound turn latency (#139) and freeing room the elastic window (#234) can spend. SERVING_FLASH_ATTN=1|on|true adds --flash-attn to the lane; absent → llama.cpp default (no flag), byte-identical. OFF by default: Metal/backend flash-attn support + quality vary by build, so it's an operator opt-in, never a blind assumption ([[verify-real-device-numbers-not-a-clamp-premise]]). Composes with the KV-quant flag: enable both for the field-proven GLM-style speedup, then validate on a live burst. continuum-core lib compiles clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LoTjvf5j3Ez13g6k8mRkFo --- core/continuum-core/src/inference/llama_server.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index 7c622900d..8de207f60 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -2244,6 +2244,19 @@ impl LlamaServerControl for LlamaServerProcess { .arg("--cache-type-v") .arg(&kv_type); } + // FLASH ATTENTION (#232, opt-in field-proven technique). The fused attention kernel + // is faster on BOTH prefill and decode and lowers peak memory — directly attacking + // the prefill-bound turn latency (#139) and freeing room the elastic window (#234) + // can spend. OFF by default: Metal/backend flash-attn support + quality vary by build + // ([[verify-real-device-numbers-not-a-clamp-premise]]), so it's an operator opt-in, + // never a blind assumption. SERVING_FLASH_ATTN=1|on|true → enable; absent → llama.cpp + // default (no flag), byte-identical. + if crate::config_env::read("SERVING_FLASH_ATTN") + .map(|s| matches!(s.trim().to_ascii_lowercase().as_str(), "1" | "on" | "true" | "yes")) + .unwrap_or(false) + { + cmd.arg("--flash-attn"); + } // MULTIMODAL PROJECTOR (#106): a vision/audio-capable model needs its mmproj GGUF so // llama-server loads the vision (or audio) encoder and can tokenize image/audio content // parts. Present → the model actually SEES (the `ContentPart::Image` the persona render