diff --git a/core/continuum-core/src/inference/llama_server.rs b/core/continuum-core/src/inference/llama_server.rs index 8de207f60..64504e0e3 100644 --- a/core/continuum-core/src/inference/llama_server.rs +++ b/core/continuum-core/src/inference/llama_server.rs @@ -2283,6 +2283,28 @@ impl LlamaServerControl for LlamaServerProcess { mmproj_local_path) or drop the Vision capability so the row stops claiming sight." ); } + // NATIVE MTP SPECULATIVE DECODE (#440): if the model ships an `mtp-*.gguf` + // draft head beside its main GGUF (the ggml-org Qwen3.8 layout), load it as + // the spec-decode draft. MTP heads are trained WITH the model, so acceptance + // is high and there is no external draft model to fit: field-measured on + // Qwen3.8-27B (RTX 4090, 2026-08-15) decode went 40.7 → 60.1 t/s for ~0.1GB + // extra state. Artifact presence IS the capability signal (the mmproj + // pattern): no draft file → no flags → byte-identical serving. n-max 4 / + // p-min 0.7 are the upstream-recommended MTP operating point from that same + // field benchmark — per-model tuning, if ever needed, belongs on the Model + // row beside `sampling`, not here. + if let Some(draft) = + crate::model_registry::artifacts::resolve_mtp_draft_for_model(&target.model) + { + cmd.arg("--spec-type") + .arg("draft-mtp") + .arg("--spec-draft-model") + .arg(&draft) + .arg("--spec-draft-n-max") + .arg("4") + .arg("--spec-draft-p-min") + .arg("0.7"); + } // Device-fit resident-override (#29): source the RESIDENT (non-expert) // tensors from the precision-shrunk fit GGUF so the whole resident tier fits // VRAM offloaded to GPU, while this primary GGUF streams the experts. The diff --git a/core/continuum-core/src/model_registry/artifacts.rs b/core/continuum-core/src/model_registry/artifacts.rs index 98190df83..ff38ff6a7 100644 --- a/core/continuum-core/src/model_registry/artifacts.rs +++ b/core/continuum-core/src/model_registry/artifacts.rs @@ -79,6 +79,36 @@ fn find_mmproj_beside(dir: &Path) -> Option { }) } +/// Resolve a model's native-MTP speculative-decode draft head for serving +/// (`llama-server --spec-type draft-mtp --spec-draft-model `). +/// +/// Convention (ggml-org, e.g. Qwen3.8-27B): repos whose architecture bakes in +/// multi-token-prediction heads ship the head as a sibling `mtp--.gguf` +/// beside the main weights, so it lands in the same snapshot dir the normal GGUF +/// resolution finds. Artifact presence IS the capability signal — the exact pattern +/// [`resolve_mmproj_for_model`] established: no draft file → `None` → the spawn adds +/// no flags and serving is byte-identical to before this seam existed. +pub fn resolve_mtp_draft_for_model(model: &Model) -> Option { + let gguf = resolve_gguf_for_model(model)?; + find_mtp_draft_beside(gguf.parent()?) +} + +/// Find an MTP draft-head GGUF sitting in `dir` — an `mtp-*.gguf` sibling of the +/// model's GGUF. When several quants of the head are present, newest-mtime wins +/// (same tie-break as main-model candidate selection). +fn find_mtp_draft_beside(dir: &Path) -> Option { + let candidates: Vec = fs::read_dir(dir) + .ok()? + .flatten() + .filter_map(|entry| { + let path = entry.path(); + let name = path.file_name()?.to_str()?.to_ascii_lowercase(); + (name.starts_with("mtp-") && name.ends_with(".gguf")).then_some(path) + }) + .collect(); + pick_best_candidate(candidates) +} + /// Resolve a canonical model id to the HF safetensors repo id of its /// *trainable* form (`Model::hf_source`). The training lane (`mlx_lm.lora /// --model`) and the forge custodian's HF→PEFT→GGUF convert both need the @@ -463,15 +493,22 @@ fn is_gguf(path: &Path) -> bool { /// (#106): pulling Qwen2.5-VL wrote the mmproj AFTER the main weights, so /// mtime-newest candidate selection picked the projector as the model and /// every VL spawn died with "unsupported model architecture: 'clip'". -/// ONE predicate for every main-model collector; [`find_mmproj_beside`] -/// remains the mmproj-POSITIVE scan. +/// Also excludes `mtp-*.gguf` MTP draft heads (the ggml-org Qwen3.8 layout): +/// same failure shape — the draft downloads AFTER the main weights, so +/// mtime-newest would serve the 1.6GB head as the 27B model. Draft heads load +/// via `--spec-draft-model`, never `-m`. +/// ONE predicate for every main-model collector; [`find_mmproj_beside`] and +/// [`find_mtp_draft_beside`] remain the sidecar-POSITIVE scans. fn is_main_model_gguf(path: &Path) -> bool { if !is_gguf(path) { return false; } path.file_name() .and_then(|s| s.to_str()) - .is_some_and(|name| !name.to_ascii_lowercase().contains("mmproj")) + .is_some_and(|name| { + let name = name.to_ascii_lowercase(); + !name.contains("mmproj") && !name.starts_with("mtp-") + }) } fn home_dir_string() -> Option { @@ -612,6 +649,35 @@ mod tests { assert!(find_ggufs_under_snapshots(snap.path()).is_none()); } + // what this catches: #440 — the ggml-org Qwen3.8 snapshot ships main + `mtp-*.gguf` + // draft head + mmproj in ONE dir, and the draft downloads AFTER the main weights + // (live ordering 2026-08-15: main 02:48, mtp 02:49). Without the mtp exclusion, + // mtime-newest candidate selection serves the 1.6GB DRAFT HEAD as the 27B model — + // the exact #106 clip failure shape. The draft-POSITIVE scan + // (`find_mtp_draft_beside`) must still find it so the spawn can pass + // `--spec-type draft-mtp`. + #[test] + fn mtp_draft_head_never_wins_main_model_resolution_but_resolves_as_draft() { + let snap = tempfile::tempdir().unwrap(); + let model_dir = snap.path().join("snapshots").join("qwen38"); + std::fs::create_dir_all(&model_dir).unwrap(); + let main = model_dir.join("Qwen3.8-27B-Q4_K_M.gguf"); + write_empty_gguf(&main); + // Written SECOND → newer mtime, the exact live download ordering. + std::thread::sleep(std::time::Duration::from_millis(20)); + let draft = model_dir.join("mtp-Qwen3.8-27B-Q4_0.gguf"); + write_empty_gguf(&draft); + + let picked = find_ggufs_under_snapshots(snap.path()).expect("main model resolves"); + assert_eq!(picked, main, "draft head must not out-mtime the model"); + let found = find_mtp_draft_beside(&model_dir).expect("draft head still discoverable"); + assert_eq!(found, draft); + // A directory with no mtp sibling resolves no draft — the spawn adds no + // spec-decode flags and serving is byte-identical to pre-#440. + std::fs::remove_file(&draft).unwrap(); + assert!(find_mtp_draft_beside(&model_dir).is_none()); + } + // what this catches: tier-1 resolution — an explicitly declared projector resolves // (with `~` expansion + existence check) so the serving spawn passes `--mmproj` and // the model can SEE; a declared-but-absent projector with no GGUF-sibling either, diff --git a/core/continuum-core/src/model_registry/catalog.rs b/core/continuum-core/src/model_registry/catalog.rs index 4963cc73b..13e4b67da 100644 --- a/core/continuum-core/src/model_registry/catalog.rs +++ b/core/continuum-core/src/model_registry/catalog.rs @@ -518,6 +518,49 @@ pub fn models() -> Vec { stop_sequences: &["<|im_end|>"], ..ModelSpec::default() }), + // QWEN3.8-27B — the FRONTIER-TIER lane (Joel, 2026-08-15: "open models just got + // released that are better than opus and even fable"). Dense 27B, Arch::Qwen35 + // (the fork carries LLM_ARCH_QWEN35 + MTP draft spec-decode + the mmproj vision + // path for it). Published scores: SWE-bench Pro 61.7 vs Opus 4.6 Max 53.4, + // QwenSWEBench 79.0 vs 63.8 — a local model that beats the cloud flagship on + // agentic coding, on consumer hardware. Field-measured serving (RTX 4090, + // Q4_K_M-class): 40.7 t/s decode plain, 60.1 t/s with native MTP spec-decode + // (the `mtp-*.gguf` sibling this catalog's serving spawn now auto-loads, #440), + // 262k native context resident in 24GB with q4_0 KV. The ggml-org repo ships + // main + mtp draft + mmproj in ONE snapshot, so `models/pull` acquires all + // three and the sibling resolvers find them with zero per-machine paths. + // context_window is the MODEL's capability; the live served window comes from + // the adapter/live serve per #50. + model(ModelSpec { + id: "ggml-org/Qwen3.8-27B-GGUF", + name: "Qwen3.8-27B (frontier agentic coder + vision)", + provider: "llama-server", + arch: Arch::Qwen35, + context_window: 262_144, + max_output_tokens: 16_384, + // Conservative M5/Metal estimate for a dense 27B (Devstral 24B row carries + // 10.0); the 4090 numbers above don't transfer across backends. Corrected + // by live measurement, never by wish. + tokens_per_second: 10.0, + capabilities: &[ + Capability::TextGeneration, + Capability::Chat, + Capability::ToolUse, + Capability::Vision, + Capability::Streaming, + ], + gguf_hint: Some("huggingface.co/ggml-org/Qwen3.8-27B-GGUF"), + // Trainable HF safetensors base (verified live 2026-08-15: repo exists, + // pipeline image-text-to-text, arch qwen3_5) — what the genome forge + // trains LoRA against; the GGUF above is serving-only. + hf_source: Some("Qwen/Qwen3.8-27B"), + // Embedded template + --jinja (same pattern as Devstral/Hermes): the + // ggml-org GGUF carries Qwen3.5's own ChatML-with-tools template. + chat_template: None, + multi_party_strategy: MultiPartyChatStrategy::ProperChatMlSingleParty, + stop_sequences: &["<|im_end|>"], + ..ModelSpec::default() + }), // Hermes-3-Llama-3.1-8B — the OPPONENT, made first-class. A general (non-coder) model we // benchmark AGAINST; giving it a real catalog row lets it flow through OURS (base_model_id) // and opencode like any other model, so the head-to-head is model-through-harness fair, not