From abf6b4b181a187f53dbf434ff4b68e13081e58e4 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 13 Aug 2026 12:55:23 +0000 Subject: [PATCH 01/39] feat(vllm): add GPU memory utilization hint for OOM errors and improve documentation Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 274 ++++++++++++++++++++++++++++++++++-- crates/rocm-core/src/lib.rs | 12 ++ docs/vllm.md | 28 ++++ engines/vllm/src/lib.rs | 54 ++++++- 4 files changed, 353 insertions(+), 15 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 493f64d69..6557752da 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -5250,6 +5250,9 @@ fn serve(args: ServeArgs) -> Result<()> { host_gpu_summary.as_ref(), ) { println!(" {warning}"); + if engine_serves_vllm { + println!(" note: {}", rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT); + } } } if let Some(engine_recipe) = &resolve.engine_recipe { @@ -5336,6 +5339,7 @@ fn serve(args: ServeArgs) -> Result<()> { gpu_vram.as_deref(), gpu_memory_utilization_note.as_deref(), host_gpu_summary.as_ref(), + engine_serves_vllm, ); let summary = serve_summary::DeploymentSummary { engine: selected_engine.clone(), @@ -5373,6 +5377,7 @@ fn serve(args: ServeArgs) -> Result<()> { /// GPU/device warnings folded into the interactive deployment summary. Mirrors the /// inline warnings printed in the plain serve plan, in the same order. +#[allow(clippy::too_many_arguments)] fn collect_serve_notes( cpu_only: bool, gpu_selection: &GpuSelection, @@ -5381,6 +5386,7 @@ fn collect_serve_notes( gpu_vram: Option<&[GpuVramUsage]>, engine_flag_note: Option<&str>, host_gpu_summary: Option<&rocm_core::HostGpuSummary>, + engine_is_vllm: bool, ) -> Vec { let mut notes = Vec::new(); // An engine-scoped flag the selected engine cannot honor must be reported here @@ -5407,6 +5413,11 @@ fn collect_serve_notes( if let Some(warning) = serve_gpu_low_memory_warning(gpu_indices, gpu_vram, host_gpu_summary) { notes.push(warning); + // vLLM's total-VRAM reservation is what turns a busy card into an OOM; + // pair the generic warning with the concrete knob that avoids it. + if engine_is_vllm { + notes.push(rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT.to_owned()); + } } } notes @@ -18452,10 +18463,10 @@ const AUTO_FREE_VRAM_FRACTION: f64 = 0.90; /// Pick a GPU ordinal for `--gpu auto`. Prefers the lowest-numbered GPU that is /// idle (free VRAM at or above [`AUTO_FREE_VRAM_FRACTION`]) and not pinned by a /// running rocm-cli service; otherwise falls back to the non-busy GPU with the -/// most free VRAM, then to the first non-busy GPU. When the GPU count is unknown -/// (amd-smi unavailable or zero devices) it returns no selection rather than -/// assuming device 0 — the engine's device probe then pins the first present GPU -/// or fails fast under the GPU-required policy. +/// most free VRAM, then to the first non-busy GPU. When neither the amd-smi +/// device count nor the DRM sysfs VRAM fallback yields any device it returns no +/// selection rather than assuming device 0 — the engine's device probe then pins +/// the first present GPU or fails fast under the GPU-required policy. /// /// Selection reads service state without holding a lock, so two near-concurrent /// `--gpu auto` launches can race onto the same idle GPU. The VRAM-occupancy @@ -18478,12 +18489,20 @@ fn select_auto_gpu_index( busy: &[u32], vram: Option<&[GpuVramUsage]>, ) -> Vec { - let count = detected.unwrap_or(0); + // The amd-smi `list` device count is the primary source, but on a shared + // node without amd-smi it is `None` even though the DRM sysfs fallback probe + // may still have populated `vram`. Derive the count from those rows in that + // case so `--gpu auto` can rank GPUs by free VRAM instead of returning no + // selection and landing on a busy device 0. + let count = detected + .filter(|&count| count > 0) + .or_else(|| vram.map(<[GpuVramUsage]>::len).filter(|&count| count > 0)) + .unwrap_or(0); if count == 0 { - // No GPU count from amd-smi (unavailable, or genuinely zero devices). Do - // not assume device 0 exists: return no selection and let the engine's - // device probe pin the first present GPU or fail fast under the - // GPU-required policy (no GPU-0 fallback). + // No GPU count from amd-smi and no VRAM telemetry (both unavailable, or + // genuinely zero devices). Do not assume device 0 exists: return no + // selection and let the engine's device probe pin the first present GPU + // or fail fast under the GPU-required policy (no GPU-0 fallback). return Vec::new(); } let candidates = || (0..count as u32).filter(|index| !busy.contains(index)); @@ -18524,10 +18543,19 @@ fn select_auto_gpu_index( vec![0] } -/// Best-effort per-GPU VRAM occupancy via `amd-smi metric --json`. Returns -/// `None` when amd-smi is unavailable or its output cannot be parsed (callers -/// then fall back to service-state-only auto-selection). +/// Best-effort per-GPU VRAM occupancy. Prefers `amd-smi metric --json`; when +/// amd-smi is not installed (e.g. a shared multi-GPU node or container that ships +/// no rocm-smi/amd-smi) it falls back to the amdgpu DRM sysfs VRAM counters so +/// `--gpu auto` selection and the low-VRAM warning still have telemetry to work +/// with. Returns `None` only when neither source is readable (callers then fall +/// back to service-state-only auto-selection). fn gpu_vram_usage() -> Option> { + gpu_vram_usage_amd_smi().or_else(gpu_vram_usage_sysfs) +} + +/// Per-GPU VRAM occupancy via `amd-smi metric --json`. Returns `None` when +/// amd-smi is unavailable or its output cannot be parsed. +fn gpu_vram_usage_amd_smi() -> Option> { let binary = rocm_core::resolve_amd_smi_binary(); let output = ProcessCommand::new(&binary) .arg("metric") @@ -18545,6 +18573,94 @@ fn gpu_vram_usage() -> Option> { if rows.is_empty() { None } else { Some(rows) } } +/// Fallback per-GPU VRAM occupancy read straight from the amdgpu DRM sysfs +/// counters (`/sys/class/drm/card*/device/mem_info_vram_{total,used}`), used when +/// amd-smi is not installed. Linux-only; other platforms expose no such +/// interface and return `None`. +#[cfg(target_os = "linux")] +fn gpu_vram_usage_sysfs() -> Option> { + let rows = read_drm_vram_usage(Path::new("/sys/class/drm")); + if rows.is_empty() { None } else { Some(rows) } +} + +#[cfg(not(target_os = "linux"))] +fn gpu_vram_usage_sysfs() -> Option> { + None +} + +/// Read per-GPU VRAM occupancy from the amdgpu DRM sysfs tree rooted at +/// `drm_dir`. Enumerates primary `card` nodes (skipping connector sub-nodes +/// like `card0-DP-1`), keeps only AMD (`vendor == 0x1002`) devices exposing the +/// `mem_info_vram_total` counter, and assigns sequential 0-based ordinals in +/// ascending card order to mirror HIP device ordering on a standard single-host +/// install. Split from [`gpu_vram_usage_sysfs`] so it can be tested against a +/// planted sysfs layout. +#[cfg(any(target_os = "linux", test))] +fn read_drm_vram_usage(drm_dir: &Path) -> Vec { + const BYTES_PER_MIB: u64 = 1024 * 1024; + let Ok(entries) = fs::read_dir(drm_dir) else { + return Vec::new(); + }; + // Key by the numeric card suffix so ordinals are stable regardless of the + // filesystem-defined readdir order. + let mut cards: Vec<(u32, GpuVramUsage)> = Vec::new(); + for entry in entries.flatten() { + let card_path = entry.path(); + let Some(name) = card_path.file_name().and_then(|value| value.to_str()) else { + continue; + }; + if !name.starts_with("card") || name.contains('-') { + continue; + } + let Some(suffix) = name + .strip_prefix("card") + .and_then(|digits| digits.parse::().ok()) + else { + continue; + }; + let device_dir = card_path.join("device"); + if !drm_device_is_amd(&device_dir) { + continue; + } + let Some(total_bytes) = read_sysfs_u64(&device_dir.join("mem_info_vram_total")) else { + continue; + }; + let used_bytes = read_sysfs_u64(&device_dir.join("mem_info_vram_used")).unwrap_or(0); + cards.push(( + suffix, + GpuVramUsage { + // Placeholder ordinal; the final index is assigned after sorting. + index: 0, + used_mb: used_bytes / BYTES_PER_MIB, + total_mb: total_bytes / BYTES_PER_MIB, + }, + )); + } + cards.sort_by_key(|(suffix, _)| *suffix); + cards + .into_iter() + .enumerate() + .map(|(ordinal, (_, usage))| GpuVramUsage { + index: ordinal as u32, + ..usage + }) + .collect() +} + +/// Read a sysfs file whose entire contents are a single unsigned integer. +#[cfg(any(target_os = "linux", test))] +fn read_sysfs_u64(path: &Path) -> Option { + fs::read_to_string(path).ok()?.trim().parse::().ok() +} + +/// Whether a DRM `device` directory belongs to an AMD GPU (PCI `vendor` id +/// `0x1002`). +#[cfg(any(target_os = "linux", test))] +fn drm_device_is_amd(device_dir: &Path) -> bool { + fs::read_to_string(device_dir.join("vendor")) + .is_ok_and(|vendor| vendor.trim().eq_ignore_ascii_case("0x1002")) +} + /// Parse `amd-smi metric --json` output into per-GPU VRAM usage. Accepts both /// the `{"gpu_data": [...]}` envelope and a bare top-level array, mirroring the /// schema variance handled by the dashboard amd-smi collector. @@ -25309,13 +25425,23 @@ install therock"; None, Some(note), None, + false, ); assert!( notes.iter().any(|entry| entry == note), "the ignored-flag note must reach the summary: {notes:?}" ); - let quiet = collect_serve_notes(false, &GpuSelection::Auto, false, &[0], None, None, None); + let quiet = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + None, + None, + None, + false, + ); assert!( !quiet .iter() @@ -25491,6 +25617,57 @@ install therock"; ); } + #[test] + fn serve_notes_pair_low_vram_with_the_vllm_utilization_hint() { + // A busy discrete card that trips the low-VRAM warning: on vLLM the note + // must also carry the concrete `--gpu-memory-utilization` workaround, so + // the interactive summary tells the user how to avoid the OOM. + let busy = [vram(0, 182_000, 192_000)]; + let notes = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + Some(&busy), + None, + Some(&host_gpu("gfx1100")), + true, + ); + assert!( + notes.iter().any(|entry| entry.contains("has only")), + "the low-VRAM warning must be present: {notes:?}" + ); + assert!( + notes + .iter() + .any(|entry| entry.contains("--gpu-memory-utilization")), + "vLLM low-VRAM notes must hint the utilization workaround: {notes:?}" + ); + + // The same busy card on a non-vLLM engine keeps the warning but omits the + // vLLM-only knob, which that engine cannot honor. + let lemonade = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + Some(&busy), + None, + Some(&host_gpu("gfx1100")), + false, + ); + assert!( + lemonade.iter().any(|entry| entry.contains("has only")), + "the low-VRAM warning still fires for other engines: {lemonade:?}" + ); + assert!( + !lemonade + .iter() + .any(|entry| entry.contains("--gpu-memory-utilization")), + "non-vLLM engines must not be told to pass a vLLM-only flag: {lemonade:?}" + ); + } + #[test] fn engine_recipe_enables_tool_choice_reflects_flags() { assert!(!engine_recipe_enables_tool_choice(None)); @@ -25578,6 +25755,29 @@ install therock"; assert_eq!(select_auto_gpu_index(None, &[], None), Vec::::new()); } + #[test] + fn auto_selection_uses_vram_row_count_when_amd_smi_count_is_unknown() { + // A shared node without amd-smi: `detect_gpu_count()` is `None`, but the + // DRM sysfs fallback still populated per-GPU VRAM. Auto-selection must + // derive the device count from those rows and skip the busy GPU 0 rather + // than returning no selection (which would land the engine on GPU 0). + let usage = [ + vram(0, 182_000, 192_000), + vram(1, 1_000, 192_000), + vram(2, 500, 192_000), + ]; + assert_eq!( + select_auto_gpu_index(None, &[], Some(&usage)), + vec![1], + "with no amd-smi count, rank the sysfs VRAM rows and pick the first idle GPU" + ); + // Still nothing to go on when neither the count nor telemetry is present. + assert_eq!( + select_auto_gpu_index(None, &[], Some(&[])), + Vec::::new() + ); + } + #[test] fn validate_pinned_gpu_index_rejects_out_of_range() { // Index equal to or beyond the detected count is rejected. @@ -25611,6 +25811,54 @@ install therock"; assert!((rows[1].free_fraction().unwrap() - (142_000.0 / 192_000.0)).abs() < 1e-9); } + #[test] + fn read_drm_vram_usage_reads_amdgpu_cards_and_assigns_ordinals() { + // Plant a minimal `/sys/class/drm`-shaped tree: two AMD cards, a connector + // sub-node that must be skipped, and a non-AMD card that must be ignored. + // Read in reverse-numeric order to prove ordinals follow card index, not + // readdir order. + let root = std::env::temp_dir().join(format!("rocm-drm-vram-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let plant_amd = |card: &str, total: u64, used: u64| { + let device = root.join(card).join("device"); + fs::create_dir_all(&device).unwrap(); + fs::write(device.join("vendor"), "0x1002\n").unwrap(); + fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); + fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); + }; + // 192 GiB total; card1 mostly used, card0 mostly free (values in bytes). + plant_amd("card1", 206_158_430_208, 189_284_651_008); + plant_amd("card0", 206_158_430_208, 1_073_741_824); + // A connector sub-node under card0 — must be skipped, not parsed as a card. + fs::create_dir_all(root.join("card0-DP-1")).unwrap(); + // A non-AMD primary card — different vendor, must be ignored. + let intel = root.join("card2").join("device"); + fs::create_dir_all(&intel).unwrap(); + fs::write(intel.join("vendor"), "0x8086\n").unwrap(); + fs::write(intel.join("mem_info_vram_total"), "1000000\n").unwrap(); + + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); + + assert_eq!(rows.len(), 2, "only the two AMD cards count: {rows:?}"); + // Ordinals are assigned in ascending card order (card0 -> 0, card1 -> 1). + assert_eq!(rows[0].index, 0); + assert_eq!(rows[1].index, 1); + // 206_158_430_208 bytes / 1 MiB == 196_608 MiB. + assert_eq!(rows[0].total_mb, 196_608); + assert_eq!(rows[1].total_mb, 196_608); + // card0 is nearly idle; card1 is nearly full. + assert!(rows[0].free_fraction().unwrap() > AUTO_FREE_VRAM_FRACTION); + assert!(rows[1].free_fraction().unwrap() < AUTO_FREE_VRAM_FRACTION); + } + + #[test] + fn read_drm_vram_usage_is_empty_without_a_drm_tree() { + let missing = std::env::temp_dir().join("rocm-drm-vram-absent-98765"); + let _ = fs::remove_dir_all(&missing); + assert!(read_drm_vram_usage(&missing).is_empty()); + } + #[test] fn gpu_low_memory_warning_flags_busy_selected_gpu() { let usage = [vram(0, 182_000, 192_000), vram(1, 1_000, 192_000)]; diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index ee4f1eff8..5170a86c9 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -7390,6 +7390,18 @@ pub fn resolve_amd_smi_binary() -> OsString { resolve_amd_smi_binary_in_home(runtime_home_dir().as_deref()) } +/// The `--gpu-memory-utilization` workaround for a shared/busy GPU. +/// +/// Shared by both the `rocm` CLI (pre-launch low-VRAM note) and the vLLM engine +/// adapter (post-failure OOM hint) so the two surfaces never drift into +/// different wording for the same fix. vLLM reserves a fixed fraction of each +/// GPU's *total* VRAM by default (~0.9), independent of the model size or how +/// much is currently free, so on a shared or busy card that reservation +/// collides with memory already in use and the engine OOMs even a tiny model. +pub const VLLM_GPU_MEMORY_UTILIZATION_HINT: &str = "vLLM reserves ~90% of the GPU's total VRAM by default; on a shared or busy GPU this can \ + collide with memory already in use. Lower the reservation with `--gpu-memory-utilization \ + <0-1>` (e.g. 0.1 for a small model), or target a less-busy GPU with `--gpu `."; + /// Locate `amd-smi` inside the bin directories of the newest managed ROCm SDK /// runtime recorded in the registry. The binary ships with the TheRock wheel /// (under the SDK `bin_path` and/or the venv `install_root/bin`) and is not on diff --git a/docs/vllm.md b/docs/vllm.md index aef979e05..e1b58a285 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -150,6 +150,34 @@ Earlier releases pinned this to `0.80` to leave display/WSL headroom. That pin i gone, so an unchanged command now reserves vLLM's own (higher) default. Pass `--gpu-memory-utilization 0.8` to restore the previous reservation. +#### Shared or busy GPUs + +Because the reservation is a fraction of **total** VRAM, it ignores memory +already held by other workloads. On a shared multi-GPU node the default can +collide with in-use memory and the engine fails with `HIP out of memory` even for +a tiny model. rocm-cli helps in three ways: + +- **Auto-selection avoids busy cards.** `--gpu auto` ranks GPUs by free VRAM and + skips heavily-used ones. When `rocm-smi`/`amd-smi` are not installed it falls + back to the amdgpu DRM sysfs counters + (`/sys/class/drm/card*/device/mem_info_vram_{total,used}`), so selection still + works on stripped-down container images. +- **The serve summary warns on low free VRAM.** When the selected GPU is already + heavily used, `rocm serve` prints a note before launch and, for vLLM, points at + `--gpu-memory-utilization` as the fix. +- **OOM failures hint the workaround.** When a startup failure log shows an + out-of-memory error, the failure message suggests retrying with a smaller + reservation, e.g. `--gpu-memory-utilization 0.1`, or targeting a less-busy GPU + with `--gpu `. + +Explicitly, the workaround for an OOM on a shared card is: + +```bash +rocm serve --gpu-memory-utilization 0.1 +# optionally target a less-busy GPU +rocm serve --gpu 3 --gpu-memory-utilization 0.1 +``` + ### Tool calling The TUI chat tab attaches tool definitions to every chat request. vLLM rejects diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index b7f3ac138..c574b715d 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -2215,12 +2215,35 @@ fn startup_log_context(log_path: Option<&Path>) -> String { .and_then(|p| summarize_startup_log_tail(p, STARTUP_FAILURE_LOG_TAIL_LINES).ok()) .unwrap_or_default(); if summary.is_empty() { - String::new() + return String::new(); + } + let hint = oom_utilization_hint(&summary); + format!("\n\nLast {STARTUP_FAILURE_LOG_TAIL_LINES} lines of startup log:\n{summary}{hint}") +} + +/// Appends [`rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT`] when the startup-log +/// tail shows an out-of-memory failure, and nothing otherwise so healthy +/// failures are not cluttered with an irrelevant memory hint. Shares its wording +/// with the `rocm` CLI's pre-launch low-VRAM note so both surfaces point the +/// user at the same fix rather than drifting into different phrasing. +fn oom_utilization_hint(log_tail: &str) -> String { + if log_tail_shows_oom(log_tail) { + format!( + "\n\nDetected an out-of-memory failure. {}", + rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT + ) } else { - format!("\n\nLast {STARTUP_FAILURE_LOG_TAIL_LINES} lines of startup log:\n{summary}") + String::new() } } +/// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a +/// HIP allocation failure (e.g. `torch.OutOfMemoryError: HIP out of memory`). +fn log_tail_shows_oom(log_tail: &str) -> bool { + let lower = log_tail.to_ascii_lowercase(); + lower.contains("out of memory") || lower.contains("outofmemory") +} + /// Polls the vLLM endpoint until it reports the model is loaded, or times out. /// Uses a monotonic clock (`Instant`) so wall-clock adjustments cannot corrupt the /// timeout, and surfaces an early process exit immediately instead of waiting out @@ -2682,6 +2705,33 @@ mod tests { Ok(()) } + #[test] + fn oom_utilization_hint_fires_on_out_of_memory_log_tails() { + // The exact PyTorch/HIP signature from the field report. + let torch = "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB."; + assert!(log_tail_shows_oom(torch)); + let hint = oom_utilization_hint(torch); + assert!( + hint.contains("--gpu-memory-utilization"), + "an OOM tail must surface the utilization workaround: {hint}" + ); + assert!(hint.contains("--gpu ")); + + // Detection is case-insensitive and also matches the spaced phrasing. + assert!(log_tail_shows_oom("HIP OUT OF MEMORY")); + assert!(log_tail_shows_oom("RuntimeError: CUDA out of memory")); + } + + #[test] + fn oom_utilization_hint_stays_quiet_for_unrelated_failures() { + let unrelated = "ValueError: model architecture 'FooForCausalLM' is not supported"; + assert!(!log_tail_shows_oom(unrelated)); + assert!( + oom_utilization_hint(unrelated).is_empty(), + "non-OOM failures must not carry a memory hint" + ); + } + #[test] fn vllm_serve_args_omit_gpu_memory_utilization_without_override() { let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None); From 38257fec1f74c0adc423d030280aeebb8cb6cfac Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 21 Aug 2026 10:10:51 +0000 Subject: [PATCH 02/39] fix(vllm): validate --gpu against sysfs GPU count and harden DRM fallback Address review feedback on PR #251: - resolve_gpu_indices now validates an explicit --gpu against the DRM sysfs fallback's device count when amd-smi is unavailable, instead of only the amd-smi count. Previously an out-of-range index was silently accepted on a host with no amd-smi (sysfs works) and only failed later inside the engine. - read_drm_vram_usage no longer treats an unreadable mem_info_vram_used counter as 0 bytes used (which made the card look 100% free -- exactly what --gpu auto prefers first); it now skips that card instead. - read_drm_vram_usage withholds telemetry entirely when more than one AMD DRM card is present, since ascending card order is only guaranteed to match HIP's compute-topology ordinal on a single-GPU host (an APU passes the same vendor + mem_info_vram_total filter as a discrete GPU, so an APU+dGPU host could previously feed a diverged ordinal into HIP_VISIBLE_DEVICES). - docs/vllm.md: corrected the fallback description, which only ever probes amd-smi (never rocm-smi) before falling back to DRM sysfs. Extracted the count-fallback logic into a new effective_gpu_count helper shared by --gpu auto ranking and --gpu validation, and added unit tests for all of the above. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 160 ++++++++++++++++++++++++++++++++++-------- docs/vllm.md | 8 ++- 2 files changed, 137 insertions(+), 31 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6557752da..cbce5f230 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -18409,15 +18409,30 @@ fn resolve_gpu_indices( ) -> Result> { let detected = detect_gpu_count(); match selection { - GpuSelection::Index(index) => validate_pinned_gpu_index(*index, detected), + GpuSelection::Index(index) => { + validate_pinned_gpu_index(*index, effective_gpu_count(detected, vram)) + } GpuSelection::Auto => Ok(auto_select_gpu_indices(paths, detected, vram)), } } +/// The GPU count to validate `--gpu ` against and to rank `--gpu auto` +/// candidates over. Prefers the `amd-smi` device count; when `amd-smi` is +/// unavailable (`detected` is `None`) but the DRM sysfs VRAM fallback still +/// produced rows, its length stands in for the count so an out-of-range index +/// is caught here rather than accepted and left to fail later inside the +/// engine. `None` when neither source is available. +fn effective_gpu_count(detected: Option, vram: Option<&[GpuVramUsage]>) -> Option { + detected + .filter(|&count| count > 0) + .or_else(|| vram.map(<[GpuVramUsage]>::len).filter(|&count| count > 0)) +} + /// Validate an explicit `--gpu ` against the detected GPU count and /// return the single pinned ordinal. Errors when the index is out of range for -/// a known device count; an unknown count (amd-smi unavailable) is allowed -/// through so serving can still proceed where GPU probing is not possible. +/// a known device count; an unknown count (amd-smi unavailable and the DRM +/// sysfs fallback found no rows either) is allowed through so serving can +/// still proceed where GPU probing is not possible at all. fn validate_pinned_gpu_index(index: u32, detected: Option) -> Result> { if let Some(count) = detected && (index as usize) >= count @@ -18494,10 +18509,7 @@ fn select_auto_gpu_index( // may still have populated `vram`. Derive the count from those rows in that // case so `--gpu auto` can rank GPUs by free VRAM instead of returning no // selection and landing on a busy device 0. - let count = detected - .filter(|&count| count > 0) - .or_else(|| vram.map(<[GpuVramUsage]>::len).filter(|&count| count > 0)) - .unwrap_or(0); + let count = effective_gpu_count(detected, vram).unwrap_or(0); if count == 0 { // No GPU count from amd-smi and no VRAM telemetry (both unavailable, or // genuinely zero devices). Do not assume device 0 exists: return no @@ -18544,11 +18556,13 @@ fn select_auto_gpu_index( } /// Best-effort per-GPU VRAM occupancy. Prefers `amd-smi metric --json`; when -/// amd-smi is not installed (e.g. a shared multi-GPU node or container that ships -/// no rocm-smi/amd-smi) it falls back to the amdgpu DRM sysfs VRAM counters so -/// `--gpu auto` selection and the low-VRAM warning still have telemetry to work -/// with. Returns `None` only when neither source is readable (callers then fall -/// back to service-state-only auto-selection). +/// amd-smi is not installed (e.g. a shared node or container that ships no +/// amd-smi — this never tries `rocm-smi`) it falls back to the amdgpu DRM sysfs +/// VRAM counters so `--gpu auto` selection and the low-VRAM warning still have +/// telemetry to work with on a single-GPU host (see +/// [`read_drm_vram_usage`] for why the sysfs fallback withholds telemetry on a +/// multi-GPU one). Returns `None` only when neither source is readable +/// (callers then fall back to service-state-only auto-selection). fn gpu_vram_usage() -> Option> { gpu_vram_usage_amd_smi().or_else(gpu_vram_usage_sysfs) } @@ -18592,8 +18606,18 @@ fn gpu_vram_usage_sysfs() -> Option> { /// `drm_dir`. Enumerates primary `card` nodes (skipping connector sub-nodes /// like `card0-DP-1`), keeps only AMD (`vendor == 0x1002`) devices exposing the /// `mem_info_vram_total` counter, and assigns sequential 0-based ordinals in -/// ascending card order to mirror HIP device ordering on a standard single-host -/// install. Split from [`gpu_vram_usage_sysfs`] so it can be tested against a +/// ascending card order to mirror HIP device ordering on a standard single-GPU +/// host install. +/// +/// HIP ordinals come from the KFD *compute* topology, not DRM card numbers, and +/// the two enumerations are only guaranteed to agree when there is exactly one +/// AMD DRM card: an APU passes the same vendor + `mem_info_vram_total` filter as +/// a discrete GPU, so a host pairing an APU with a dGPU can order `card` +/// differently than HIP's node order, and that ordinal is fed to +/// `HIP_VISIBLE_DEVICES` downstream. Rather than guess, this returns no rows at +/// all when more than one AMD card is found — callers then fall back to +/// service-state-only auto-selection instead of trusting a possibly-wrong +/// ordinal. Split from [`gpu_vram_usage_sysfs`] so it can be tested against a /// planted sysfs layout. #[cfg(any(target_os = "linux", test))] fn read_drm_vram_usage(drm_dir: &Path) -> Vec { @@ -18625,7 +18649,12 @@ fn read_drm_vram_usage(drm_dir: &Path) -> Vec { let Some(total_bytes) = read_sysfs_u64(&device_dir.join("mem_info_vram_total")) else { continue; }; - let used_bytes = read_sysfs_u64(&device_dir.join("mem_info_vram_used")).unwrap_or(0); + // An unreadable `used` counter is unknown occupancy, not zero: treating + // it as `0` would make the card look 100% free, which is exactly what + // `--gpu auto` prefers first. Skip the card instead of guessing. + let Some(used_bytes) = read_sysfs_u64(&device_dir.join("mem_info_vram_used")) else { + continue; + }; cards.push(( suffix, GpuVramUsage { @@ -18636,6 +18665,12 @@ fn read_drm_vram_usage(drm_dir: &Path) -> Vec { }, )); } + // More than one AMD card: ascending card order is not guaranteed to match + // HIP's ordinal order (see doc comment above), so do not hand out ordinals + // that might feed the wrong device into `HIP_VISIBLE_DEVICES`. + if cards.len() > 1 { + return Vec::new(); + } cards.sort_by_key(|(suffix, _)| *suffix); cards .into_iter() @@ -25795,6 +25830,34 @@ install therock"; assert_eq!(validate_pinned_gpu_index(7, None).unwrap(), vec![7]); } + #[test] + fn effective_gpu_count_prefers_amd_smi_then_falls_back_to_vram_rows() { + let usage = [vram(0, 1_000, 192_000), vram(1, 1_000, 192_000)]; + // amd-smi's count wins when it is known, even over a differing row count. + assert_eq!(effective_gpu_count(Some(4), Some(&usage)), Some(4)); + // amd-smi unavailable (`None`): the sysfs VRAM fallback row count stands in. + assert_eq!(effective_gpu_count(None, Some(&usage)), Some(2)); + // Neither source available. + assert_eq!(effective_gpu_count(None, None), None); + assert_eq!(effective_gpu_count(None, Some(&[])), None); + } + + #[test] + fn resolve_gpu_indices_rejects_out_of_range_index_from_sysfs_fallback_count() { + // The exact environment this fixes: no amd-smi (`detected` is `None`), but + // the DRM sysfs fallback found one GPU. `--gpu 1` must be rejected here + // rather than accepted and left to fail later inside the engine. + let single_gpu = [vram(0, 1_000, 192_000)]; + let error = validate_pinned_gpu_index(1, effective_gpu_count(None, Some(&single_gpu))) + .expect_err("index 1 is out of range for a single detected GPU"); + assert!(error.to_string().contains("out of range")); + // The in-range index for that same host is still accepted. + assert_eq!( + validate_pinned_gpu_index(0, effective_gpu_count(None, Some(&single_gpu))).unwrap(), + vec![0] + ); + } + #[test] fn parse_gpu_vram_usage_reads_gpu_data_envelope() { let value = json!({ @@ -25812,11 +25875,9 @@ install therock"; } #[test] - fn read_drm_vram_usage_reads_amdgpu_cards_and_assigns_ordinals() { - // Plant a minimal `/sys/class/drm`-shaped tree: two AMD cards, a connector + fn read_drm_vram_usage_reads_a_single_amdgpu_card_and_assigns_ordinal_zero() { + // Plant a minimal `/sys/class/drm`-shaped tree: one AMD card, a connector // sub-node that must be skipped, and a non-AMD card that must be ignored. - // Read in reverse-numeric order to prove ordinals follow card index, not - // readdir order. let root = std::env::temp_dir().join(format!("rocm-drm-vram-{}", std::process::id())); let _ = fs::remove_dir_all(&root); let plant_amd = |card: &str, total: u64, used: u64| { @@ -25826,13 +25887,12 @@ install therock"; fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); }; - // 192 GiB total; card1 mostly used, card0 mostly free (values in bytes). - plant_amd("card1", 206_158_430_208, 189_284_651_008); + // 192 GiB total, mostly free (values in bytes). plant_amd("card0", 206_158_430_208, 1_073_741_824); // A connector sub-node under card0 — must be skipped, not parsed as a card. fs::create_dir_all(root.join("card0-DP-1")).unwrap(); // A non-AMD primary card — different vendor, must be ignored. - let intel = root.join("card2").join("device"); + let intel = root.join("card1").join("device"); fs::create_dir_all(&intel).unwrap(); fs::write(intel.join("vendor"), "0x8086\n").unwrap(); fs::write(intel.join("mem_info_vram_total"), "1000000\n").unwrap(); @@ -25840,16 +25900,60 @@ install therock"; let rows = read_drm_vram_usage(&root); let _ = fs::remove_dir_all(&root); - assert_eq!(rows.len(), 2, "only the two AMD cards count: {rows:?}"); - // Ordinals are assigned in ascending card order (card0 -> 0, card1 -> 1). + assert_eq!(rows.len(), 1, "only the one AMD card counts: {rows:?}"); assert_eq!(rows[0].index, 0); - assert_eq!(rows[1].index, 1); // 206_158_430_208 bytes / 1 MiB == 196_608 MiB. assert_eq!(rows[0].total_mb, 196_608); - assert_eq!(rows[1].total_mb, 196_608); - // card0 is nearly idle; card1 is nearly full. assert!(rows[0].free_fraction().unwrap() > AUTO_FREE_VRAM_FRACTION); - assert!(rows[1].free_fraction().unwrap() < AUTO_FREE_VRAM_FRACTION); + } + + #[test] + fn read_drm_vram_usage_withholds_telemetry_when_multiple_amd_cards_are_present() { + // Ascending `card` order is not guaranteed to match HIP's compute + // ordinal once more than one AMD card exists (e.g. an APU alongside a + // dGPU), so the fallback must not hand out ordinals it cannot vouch for. + let root = std::env::temp_dir().join(format!("rocm-drm-vram-multi-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let plant_amd = |card: &str, total: u64, used: u64| { + let device = root.join(card).join("device"); + fs::create_dir_all(&device).unwrap(); + fs::write(device.join("vendor"), "0x1002\n").unwrap(); + fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); + fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); + }; + plant_amd("card0", 206_158_430_208, 1_073_741_824); + plant_amd("card1", 206_158_430_208, 189_284_651_008); + + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); + + assert!( + rows.is_empty(), + "multi-card sysfs telemetry must be withheld: {rows:?}" + ); + } + + #[test] + fn read_drm_vram_usage_skips_a_card_with_an_unreadable_used_counter() { + // A card whose `mem_info_vram_used` cannot be read must not be treated as + // 0 bytes used (100% free) — that would make a broken counter look like + // the ideal `--gpu auto` pick. It should be skipped entirely instead. + let root = + std::env::temp_dir().join(format!("rocm-drm-vram-unreadable-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let device = root.join("card0").join("device"); + fs::create_dir_all(&device).unwrap(); + fs::write(device.join("vendor"), "0x1002\n").unwrap(); + fs::write(device.join("mem_info_vram_total"), "206158430208\n").unwrap(); + // No `mem_info_vram_used` file written at all. + + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); + + assert!( + rows.is_empty(), + "a card with no readable used counter must be skipped: {rows:?}" + ); } #[test] diff --git a/docs/vllm.md b/docs/vllm.md index e1b58a285..cd657bde6 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -158,10 +158,12 @@ collide with in-use memory and the engine fails with `HIP out of memory` even fo a tiny model. rocm-cli helps in three ways: - **Auto-selection avoids busy cards.** `--gpu auto` ranks GPUs by free VRAM and - skips heavily-used ones. When `rocm-smi`/`amd-smi` are not installed it falls - back to the amdgpu DRM sysfs counters + skips heavily-used ones. When `amd-smi` is not installed it falls back to the + amdgpu DRM sysfs counters (`/sys/class/drm/card*/device/mem_info_vram_{total,used}`), so selection still - works on stripped-down container images. + works on stripped-down container images with a single GPU. That fallback + withholds telemetry on a multi-GPU host, since its `card` numbering is not + guaranteed to match HIP's device ordinal there. - **The serve summary warns on low free VRAM.** When the selected GPU is already heavily used, `rocm serve` prints a note before launch and, for vLLM, points at `--gpu-memory-utilization` as the fix. From af059ae16afd3a6c37fae7cc49406a10588b6810 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 25 Aug 2026 12:20:31 +0300 Subject: [PATCH 03/39] fix(vllm): validate --gpu against amd-smi-independent count; harden DRM guard Round-2 review follow-ups on the DRM sysfs GPU-selection fallback. - Validate an explicit --gpu against the amd-smi list count or, when amd-smi is absent, the KFD/DRM authority behind usable_amd_gpu_indices() (pinned_index_bound) instead of the DRM VRAM-fallback row count. The row count withholds telemetry on multi-GPU hosts and shrinks on a transient counter-read failure, so borrowing it as a validation bound hard-rejected legitimate indices. VRAM rows stay scoped to --gpu auto ranking. - The multi-card ordinal-ambiguity guard now counts AMD cards *found* (before the telemetry filters) rather than surviving rows, so a second AMD card with an unreadable counter still trips the guard instead of mislabelling the survivor ordinal 0 (the APU+dGPU misattribution the guard exists to prevent). - drm_device_is_amd now matches on vendor id OR an amdgpu uevent DRIVER line, the same two-signal test as rocm_core::is_amdgpu_device, so the fallback probe and the count authority agree on what an AMD card is. - Drop the now-dead sort/enumerate tail in read_drm_vram_usage (at most one card survives the guard) and fix the stale doc comment. - Tests: exercise the real Index-arm delegate (resolve_pinned_gpu_index) and add a two-AMD-card-with-unreadable-counter regression; add pinned_index_bound coverage. - Docs: README and the assistant SKILL note the DRM sysfs VRAM fallback rather than presenting amd-smi as the only source. Signed-off-by: Roman Sirokov --- README.md | 3 +- apps/rocm/src/main.rs | 189 +++++++++++++++++++---------- skills/rocm-cli-assistant/SKILL.md | 3 +- 3 files changed, 130 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 26edffe22..b8e45aec6 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,8 @@ Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment before serving gated models. `--gpu` selects which AMD GPU the server runs on. `auto` (the default) probes -per-GPU VRAM with `amd-smi` and picks the lowest-numbered GPU that is idle and +per-GPU VRAM (via `amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` is +not installed) and picks the lowest-numbered GPU that is idle and not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory. Pass a single index (`--gpu 1`) to pin a specific device. The diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index cbce5f230..a00897769 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -18409,19 +18409,44 @@ fn resolve_gpu_indices( ) -> Result> { let detected = detect_gpu_count(); match selection { - GpuSelection::Index(index) => { - validate_pinned_gpu_index(*index, effective_gpu_count(detected, vram)) - } + GpuSelection::Index(index) => resolve_pinned_gpu_index(*index, detected), GpuSelection::Auto => Ok(auto_select_gpu_indices(paths, detected, vram)), } } -/// The GPU count to validate `--gpu ` against and to rank `--gpu auto` -/// candidates over. Prefers the `amd-smi` device count; when `amd-smi` is -/// unavailable (`detected` is `None`) but the DRM sysfs VRAM fallback still -/// produced rows, its length stands in for the count so an out-of-range index -/// is caught here rather than accepted and left to fail later inside the -/// engine. `None` when neither source is available. +/// Resolve an explicit `--gpu ` to its pinned ordinal — the `Index` arm +/// of [`resolve_gpu_indices`], split out so the validation path can be +/// unit-tested against an injected device count. +fn resolve_pinned_gpu_index(index: u32, detected: Option) -> Result> { + validate_pinned_gpu_index(index, pinned_index_bound(detected)) +} + +/// The device count to validate an explicit `--gpu ` against. Prefers the +/// `amd-smi` `list` count; when that is unavailable (`detected` is `None`) it +/// falls back to the amd-smi-independent KFD/DRM authority behind +/// [`rocm_core::usable_amd_gpu_indices`] — the same source the engine device +/// gate itself trusts. +/// +/// Deliberately does NOT borrow the DRM sysfs VRAM-fallback row count: that +/// probe withholds telemetry entirely on multi-GPU hosts and drops a card on a +/// transient counter-read failure, so using its length as a validation bound +/// would hard-reject legitimate `--gpu ` values on exactly those hosts. +/// VRAM rows stay scoped to `--gpu auto` ranking (see [`effective_gpu_count`]). +/// `None` when no count source is available, leaving the index unvalidated so +/// serving can still proceed where GPU probing is impossible. +fn pinned_index_bound(detected: Option) -> Option { + detected + .filter(|&count| count > 0) + .or_else(|| rocm_core::usable_amd_gpu_indices().map(|indices| indices.len())) + .filter(|&count| count > 0) +} + +/// The GPU count to rank `--gpu auto` candidates over. Prefers the `amd-smi` +/// device count; when `amd-smi` is unavailable (`detected` is `None`) but the +/// DRM sysfs VRAM fallback still produced rows, its length stands in so +/// auto-selection can still consider those devices. Used only for `--gpu auto` +/// ranking, never as a `--gpu ` validation bound — see +/// [`pinned_index_bound`]. `None` when neither source is available. fn effective_gpu_count(detected: Option, vram: Option<&[GpuVramUsage]>) -> Option { detected .filter(|&count| count > 0) @@ -18604,30 +18629,32 @@ fn gpu_vram_usage_sysfs() -> Option> { /// Read per-GPU VRAM occupancy from the amdgpu DRM sysfs tree rooted at /// `drm_dir`. Enumerates primary `card` nodes (skipping connector sub-nodes -/// like `card0-DP-1`), keeps only AMD (`vendor == 0x1002`) devices exposing the -/// `mem_info_vram_total` counter, and assigns sequential 0-based ordinals in -/// ascending card order to mirror HIP device ordering on a standard single-GPU -/// host install. +/// like `card0-DP-1`), keeps only AMD devices exposing readable +/// `mem_info_vram_{total,used}` counters, and assigns HIP ordinal 0 to the sole +/// AMD card to mirror HIP device ordering on a standard single-GPU host install. /// /// HIP ordinals come from the KFD *compute* topology, not DRM card numbers, and /// the two enumerations are only guaranteed to agree when there is exactly one -/// AMD DRM card: an APU passes the same vendor + `mem_info_vram_total` filter as -/// a discrete GPU, so a host pairing an APU with a dGPU can order `card` -/// differently than HIP's node order, and that ordinal is fed to -/// `HIP_VISIBLE_DEVICES` downstream. Rather than guess, this returns no rows at -/// all when more than one AMD card is found — callers then fall back to -/// service-state-only auto-selection instead of trusting a possibly-wrong -/// ordinal. Split from [`gpu_vram_usage_sysfs`] so it can be tested against a -/// planted sysfs layout. +/// AMD DRM card: an APU passes the same AMD filter as a discrete GPU, so a host +/// pairing an APU with a dGPU can order `card` differently than HIP's node +/// order, and that ordinal is fed to `HIP_VISIBLE_DEVICES` downstream. Rather +/// than guess, this returns no rows at all when more than one AMD card is +/// *found* — counted before the telemetry filters below, so a second AMD card +/// with an unreadable counter still trips the guard instead of leaving the +/// survivor mislabelled ordinal 0. Callers then fall back to service-state-only +/// auto-selection. Split from [`gpu_vram_usage_sysfs`] so it can be tested +/// against a planted sysfs layout. #[cfg(any(target_os = "linux", test))] fn read_drm_vram_usage(drm_dir: &Path) -> Vec { const BYTES_PER_MIB: u64 = 1024 * 1024; let Ok(entries) = fs::read_dir(drm_dir) else { return Vec::new(); }; - // Key by the numeric card suffix so ordinals are stable regardless of the - // filesystem-defined readdir order. - let mut cards: Vec<(u32, GpuVramUsage)> = Vec::new(); + // Every AMD card is counted here, *before* the telemetry filters below can + // drop it, so the multi-card ordinal-ambiguity guard fires on how many AMD + // cards exist rather than on how many produced readable counters. + let mut amd_cards_found = 0usize; + let mut cards: Vec = Vec::new(); for entry in entries.flatten() { let card_path = entry.path(); let Some(name) = card_path.file_name().and_then(|value| value.to_str()) else { @@ -18636,16 +18663,18 @@ fn read_drm_vram_usage(drm_dir: &Path) -> Vec { if !name.starts_with("card") || name.contains('-') { continue; } - let Some(suffix) = name + if name .strip_prefix("card") .and_then(|digits| digits.parse::().ok()) - else { + .is_none() + { continue; - }; + } let device_dir = card_path.join("device"); if !drm_device_is_amd(&device_dir) { continue; } + amd_cards_found += 1; let Some(total_bytes) = read_sysfs_u64(&device_dir.join("mem_info_vram_total")) else { continue; }; @@ -18655,31 +18684,21 @@ fn read_drm_vram_usage(drm_dir: &Path) -> Vec { let Some(used_bytes) = read_sysfs_u64(&device_dir.join("mem_info_vram_used")) else { continue; }; - cards.push(( - suffix, - GpuVramUsage { - // Placeholder ordinal; the final index is assigned after sorting. - index: 0, - used_mb: used_bytes / BYTES_PER_MIB, - total_mb: total_bytes / BYTES_PER_MIB, - }, - )); + cards.push(GpuVramUsage { + // The sole surviving AMD card is HIP ordinal 0 (guaranteed by the + // multi-card guard below). + index: 0, + used_mb: used_bytes / BYTES_PER_MIB, + total_mb: total_bytes / BYTES_PER_MIB, + }); } - // More than one AMD card: ascending card order is not guaranteed to match - // HIP's ordinal order (see doc comment above), so do not hand out ordinals - // that might feed the wrong device into `HIP_VISIBLE_DEVICES`. - if cards.len() > 1 { + // More than one AMD card found: ascending `card` order is not guaranteed + // to match HIP's ordinal order (see doc comment above), so do not hand out + // ordinals that might feed the wrong device into `HIP_VISIBLE_DEVICES`. + if amd_cards_found > 1 { return Vec::new(); } - cards.sort_by_key(|(suffix, _)| *suffix); cards - .into_iter() - .enumerate() - .map(|(ordinal, (_, usage))| GpuVramUsage { - index: ordinal as u32, - ..usage - }) - .collect() } /// Read a sysfs file whose entire contents are a single unsigned integer. @@ -18688,12 +18707,21 @@ fn read_sysfs_u64(path: &Path) -> Option { fs::read_to_string(path).ok()?.trim().parse::().ok() } -/// Whether a DRM `device` directory belongs to an AMD GPU (PCI `vendor` id -/// `0x1002`). +/// Whether a DRM `device` directory belongs to an AMD GPU. Matches on the PCI +/// `vendor` id (`0x1002`) or, when that is unreadable, an `amdgpu` `uevent` +/// `DRIVER=` line — the same two-signal test as `rocm_core::is_amdgpu_device`, +/// so this fallback probe and the KFD/DRM count authority agree on what counts +/// as an AMD card (a vendor-only test would under-count and let the multi-card +/// ordinal guard slip). #[cfg(any(target_os = "linux", test))] fn drm_device_is_amd(device_dir: &Path) -> bool { - fs::read_to_string(device_dir.join("vendor")) + if fs::read_to_string(device_dir.join("vendor")) .is_ok_and(|vendor| vendor.trim().eq_ignore_ascii_case("0x1002")) + { + return true; + } + fs::read_to_string(device_dir.join("uevent")) + .is_ok_and(|uevent| uevent.lines().any(|line| line.trim() == "DRIVER=amdgpu")) } /// Parse `amd-smi metric --json` output into per-GPU VRAM usage. Accepts both @@ -25843,19 +25871,23 @@ install therock"; } #[test] - fn resolve_gpu_indices_rejects_out_of_range_index_from_sysfs_fallback_count() { - // The exact environment this fixes: no amd-smi (`detected` is `None`), but - // the DRM sysfs fallback found one GPU. `--gpu 1` must be rejected here - // rather than accepted and left to fail later inside the engine. - let single_gpu = [vram(0, 1_000, 192_000)]; - let error = validate_pinned_gpu_index(1, effective_gpu_count(None, Some(&single_gpu))) - .expect_err("index 1 is out of range for a single detected GPU"); + fn resolve_pinned_gpu_index_validates_against_the_detected_count() { + // The `Index` arm of `resolve_gpu_indices` delegates here, so exercise it + // directly: an out-of-range index is rejected against a known count, and + // an in-range one pins exactly that ordinal. + let error = resolve_pinned_gpu_index(2, Some(1)) + .expect_err("index 2 is out of range for a single detected GPU"); assert!(error.to_string().contains("out of range")); - // The in-range index for that same host is still accepted. - assert_eq!( - validate_pinned_gpu_index(0, effective_gpu_count(None, Some(&single_gpu))).unwrap(), - vec![0] - ); + assert_eq!(resolve_pinned_gpu_index(0, Some(1)).unwrap(), vec![0]); + assert_eq!(resolve_pinned_gpu_index(3, Some(4)).unwrap(), vec![3]); + } + + #[test] + fn pinned_index_bound_prefers_the_amd_smi_count() { + // When amd-smi reports a device count, that is the validation bound; a + // zero count is treated as "no usable source" rather than a hard zero. + assert_eq!(pinned_index_bound(Some(4)), Some(4)); + assert_eq!(pinned_index_bound(Some(0)), None); } #[test] @@ -25933,6 +25965,37 @@ install therock"; ); } + #[test] + fn read_drm_vram_usage_withholds_telemetry_when_a_second_amd_card_has_no_counter() { + // The guard must count AMD cards *found*, not surviving rows: a genuine + // two-AMD-card host where the second card's `used` counter is unreadable + // would otherwise slip past a `rows.len() > 1` check and mislabel the + // survivor ordinal 0 — the exact APU+dGPU misattribution the guard exists + // to prevent. + let root = + std::env::temp_dir().join(format!("rocm-drm-vram-partial-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + // card0: fully readable. + let card0 = root.join("card0").join("device"); + fs::create_dir_all(&card0).unwrap(); + fs::write(card0.join("vendor"), "0x1002\n").unwrap(); + fs::write(card0.join("mem_info_vram_total"), "206158430208\n").unwrap(); + fs::write(card0.join("mem_info_vram_used"), "1073741824\n").unwrap(); + // card1: AMD, but its `used` counter cannot be read. + let card1 = root.join("card1").join("device"); + fs::create_dir_all(&card1).unwrap(); + fs::write(card1.join("vendor"), "0x1002\n").unwrap(); + fs::write(card1.join("mem_info_vram_total"), "206158430208\n").unwrap(); + + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); + + assert!( + rows.is_empty(), + "a second AMD card must trip the guard even without readable counters: {rows:?}" + ); + } + #[test] fn read_drm_vram_usage_skips_a_card_with_an_unreadable_used_counter() { // A card whose `mem_info_vram_used` cannot be read must not be treated as diff --git a/skills/rocm-cli-assistant/SKILL.md b/skills/rocm-cli-assistant/SKILL.md index 2396b35b8..0b54a2208 100644 --- a/skills/rocm-cli-assistant/SKILL.md +++ b/skills/rocm-cli-assistant/SKILL.md @@ -27,8 +27,9 @@ Use this skill when answering ROCm CLI local assistant questions. - vLLM and Lemonade are serving engines. - The built-in assistant is fixed to qwen served by Lemonade with GPU required. Do not switch the built-in assistant to vLLM. - Installing an engine and running a model server are different states. Answer each separately when the user asks. -- `rocm serve` accepts `--gpu auto|` to pick the AMD GPU. `auto` (default) prefers a GPU that looks idle from `amd-smi` VRAM telemetry and is not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory; a single index pins one GPU. Serving one model across multiple GPUs is not supported. Do not suggest CPU fallback when a GPU is busy or out of range. +- `rocm serve` accepts `--gpu auto|` to pick the AMD GPU. `auto` (default) prefers a GPU that looks idle from VRAM telemetry (`amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` is absent) and is not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory; a single index pins one GPU. Serving one model across multiple GPUs is not supported. Do not suggest CPU fallback when a GPU is busy or out of range. - Which engines actually run on the user's machine is stated in the host facts the CLI appends to this prompt. Answer from those facts; do not assume a platform. Never suggest CPU fallback. +- On native Windows, vLLM serving/install live checks are skipped; tell the user to use WSL/Linux for that ROCm GPU engine and do not suggest CPU fallback. ## ComfyUI From 956891f3d8fb2cb4439bdffac95d973ae443aa69 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 28 Aug 2026 11:12:12 +0300 Subject: [PATCH 04/39] feat(diagnose): add the vLLM out-of-memory failure mode to the catalog (EAI-8060) (#290) * feat(diagnose): add vLLM out-of-memory failure mode to the catalog (EAI-8060) Add a keyword-scored OOM signature to the rocm-core diagnosis catalog and a matching print-only remediation recipe. The error covers two distinct faults (a tenancy collision with vLLM's fixed ~90% VRAM reservation, versus a model that genuinely does not fit), so the wording stays conditional: it never prescribes lowering --gpu-memory-utilization as the unconditional answer, and the verify step avoids the tenancy knob. Because an Examination carries no per-GPU VRAM or tenancy fields, the match is keyword-only and the checker is gated to Linux (vLLM is Linux/WSL-only). - diagnose.rs: KEYWORDS_VLLM_OOM table + check_16_vllm_oom (linux-only) - fix.rs: print-only fix-16-vllm-oom recipe; catalog count 15 -> 16 and AUTO-set assertion strengthened - engines/vllm: serve OOM hint points users at 'rocm diagnose --symptom' - e2e-cucumber: conditional-remediation scenario + step defs and catalog contract updated Signed-off-by: Roman Sirokov * fix(diagnose): require a vLLM anchor before matching the OOM checker Review feedback on #290 flagged that KEYWORDS_VLLM_OOM never required anything vLLM-specific: 'torch.OutOfMemoryError: CUDA out of memory' scores 45+45=90 (high confidence) under the keyword table alone, so any ROCm PyTorch job's OOM would be misreported as a vLLM startup OOM with rocm-serve-only remediation. gpu_memory_utilization -- the one vLLM-specific token -- was weighted lowest and never required. check_16_vllm_oom now requires an explicit vLLM anchor (the word 'vllm', or one of its distinctive flags: gpu[-_]memory[-_]utilization, tensor[-_]parallel) before the keyword table is scored at all. Update the serve OOM hint's suggested --symptom text and the matching e2e-cucumber step to carry that anchor so the self-referential 'rocm serve' -> 'rocm diagnose' flow keeps working, and add a regression test for the reported false positive. Also fix a second issue from the same review: the diagnose() WSL branch dropped sub-threshold hits from matched entirely when nothing cleared MIN_SCORE_FOR_MATCH, which the DiagnoseReport::matched doc says should never happen. It now keeps whatever run_all_checks returns (empty only when no wsl-applicable checker fired at all) and adds a regression test. Signed-off-by: Roman Sirokov * diagnose: drop the non-existent multi-GPU sharding remedy and tighten the vLLM OOM anchor Round-2 review (EAI-8060): - Remove the `--tensor-parallel-size` / multi-GPU sharding remediation from both the fix catalog (fix-16-vllm-oom) and the diagnose summary: the flag does not exist and rocm-cli serves one model on a single GPU (docs/vllm.md). The 'model does not fit' branch now points only at a smaller/quantized model. - Drop `tensor[-_]parallel` from VLLM_ANCHOR_PATTERN: it is a Megatron/DeepSpeed term, so anchoring on it would misattribute those frameworks' OOMs to vLLM. The anchor is now just `vllm|gpu[-_]memory[-_]utilization`. - Route the user's *actual* failing log line into the `rocm diagnose --symptom` hint (vllm-anchored) instead of a canned literal. - Escape the dot in the `torch\.outofmemoryerror` keyword regex and let the `gpu[-_]memory[-_]utilization` keyword accept a hyphen like the anchor. - Tests: e2e reads high_confidence_threshold from the report instead of a hardcoded 75, locks out `--tensor-parallel-size`, and shares a find_vllm_oom helper; the sub-threshold unit test uses `.expect()` instead of a vacuous `if let`. - docs/vllm.md and the @requires-bare-metal doc note the `rocm diagnose --symptom` pointer and the WSL keyword-only exception. Signed-off-by: Roman Sirokov * style: rustfmt the new OOM --symptom routing test assertion Signed-off-by: Roman Sirokov --------- Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 249 +++++++++++++++++- crates/rocm-core/src/fix.rs | 34 ++- docs/vllm.md | 5 +- engines/vllm/src/lib.rs | 33 ++- tests/e2e-cucumber/features/diagnose.feature | 10 + tests/e2e-cucumber/src/expectation.rs | 13 +- .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 72 +++++ 7 files changed, 396 insertions(+), 20 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index ab8f0fe63..417c1aedb 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -288,6 +288,51 @@ const KEYWORDS_PAGE_FAULT: KeywordTable = &[ ("out_of_registers", 30, "compiler OUT_OF_REGISTERS"), ]; +// vLLM's startup OOM. The distinctive signature is a torch/HIP allocation +// failure while the engine is reserving KV-cache VRAM; a bare "out of memory" +// on its own is deliberately sub-threshold so this only claims the failure mode +// when the vLLM/HIP shape of the error is present. +// +// None of these tokens are vLLM-specific on their own: `torch.OutOfMemoryError: +// CUDA out of memory` is the identical shape any ROCm PyTorch job emits (ROCm's +// PyTorch build reports the CUDA-compat name), so this table alone cannot tell +// vLLM's OOM apart from an arbitrary training script hitting the same +// allocator error. `check_16_vllm_oom` therefore requires `VLLM_ANCHOR_PATTERN` +// to match before scoring this table at all. +const KEYWORDS_VLLM_OOM: KeywordTable = &[ + ( + r"torch\.outofmemoryerror", + 45, + "error mentions torch.OutOfMemoryError", + ), + ( + "hip out of memory", + 45, + "error mentions 'HIP out of memory'", + ), + ( + r"cuda out of memory", + 45, + "error mentions 'CUDA out of memory' (ROCm reports the CUDA-compat name)", + ), + ( + r"tried to allocate .*(?:gib|mib)", + 30, + "error names an allocation it could not satisfy", + ), + ( + r"\boutofmemory\b", + 30, + "error mentions standalone OutOfMemory", + ), + ("out of memory", 25, "error mentions 'out of memory'"), + ( + r"gpu[-_]memory[-_]utilization", + 20, + "log mentions gpu_memory_utilization (vLLM VRAM reservation)", + ), +]; + /// Score the strongest (top-2) keyword matches in `table` against `symptom`. fn keyword_score(symptom: &str, table: KeywordTable) -> (i32, Vec) { if symptom.is_empty() { @@ -1271,6 +1316,67 @@ fn check_15_msvc_redist(e: &Examination, symptom: &str) -> Diagnosis { ) } +/// A required anchor before [`KEYWORDS_VLLM_OOM`] is even scored: the word +/// "vllm" itself, or its distinctive VRAM-reservation flag. Without one of +/// these, a generic HIP/CUDA/PyTorch OOM string is not evidence of *this* +/// failure mode -- see the comment on [`KEYWORDS_VLLM_OOM`]. `tensor[-_]parallel` +/// is deliberately NOT an anchor: it is a Megatron/DeepSpeed term (rocm-cli does +/// not serve one model across GPUs), so anchoring on it would misattribute those +/// frameworks' OOMs to vLLM. +const VLLM_ANCHOR_PATTERN: &str = r"vllm|gpu[-_]memory[-_]utilization"; + +/// vLLM ran the GPU out of memory. Keyword-only: an [`Examination`] carries no +/// per-GPU VRAM or tenancy fields, so nothing structural can corroborate this — +/// the user must pass the error via `--symptom`, or arrive from the serve +/// failure note that points here. `e` is therefore unused. +fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { + if !symptom_matches(symptom, VLLM_ANCHOR_PATTERN) { + // No vLLM anchor: don't attribute a bare framework OOM to vLLM at all. + return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); + } + let (score, evidence) = keyword_score(symptom, KEYWORDS_VLLM_OOM); + if score <= 0 { + return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); + } + // Two different faults share this error, and the remediation splits on which + // one it is: on a shared/busy GPU vLLM's fixed ~90% reservation collides + // with memory already in use, so lowering the reservation is the fix; for a + // model that genuinely does not fit, lowering it only trades an earlier OOM + // for a later one, so the wording must not present the knob as the answer in + // that case. + let summary = format!( + "{} If instead the model genuinely does not fit in this GPU's VRAM, lowering the \ + reservation will not help — use a smaller or quantized model (rocm-cli serves one \ + model on a single GPU; it does not shard a model across GPUs).", + crate::VLLM_GPU_MEMORY_UTILIZATION_HINT + ); + let fix = Fix { + summary, + commands: vec![ + "# If the GPU is shared/busy (tenancy collision), lower vLLM's reservation:".to_owned(), + "rocm serve --gpu-memory-utilization 0.5".to_owned(), + "# ...or steer the server onto a less-busy device:".to_owned(), + "rocm serve --gpu ".to_owned(), + "# If the model genuinely does not fit, the reservation is not the problem:".to_owned(), + "# pick a smaller or quantized model (single-GPU serving only).".to_owned(), + ], + fix_id: "fix-16-vllm-oom".to_owned(), + auto_applicable: false, + verify: "rocm serve # re-run and watch for a clean startup".to_owned(), + notes: vec![ + "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it does not create room a too-large model needs.".to_owned(), + ], + ..Fix::default() + }; + finalize( + "fix-16-vllm-oom", + "vLLM ran the GPU out of memory at startup", + score, + evidence, + fix, + ) +} + /// A checker plus the OS families it applies to. type Checker = (fn(&Examination, &str) -> Diagnosis, &'static [&'static str]); @@ -1290,12 +1396,15 @@ const CHECKERS: &[Checker] = &[ (check_13_hip_sdk_missing, &["windows"]), (check_14_adrenalin_too_old, &["windows"]), (check_15_msvc_redist, &["windows"]), + (check_16_vllm_oom, &["linux", "wsl"]), ]; /// Run every applicable checker, drop zero-score results, sort by score /// descending (stable, so ties keep catalog order). fn run_all_checks(e: &Examination, symptom: &str) -> Vec { - let os_family = if e.os_family.is_empty() { + let os_family = if e.is_wsl { + "wsl" + } else if e.os_family.is_empty() { "linux" } else { e.os_family.as_str() @@ -1334,11 +1443,25 @@ pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { // it as out of scope (exit_code() == 2). Mirror that here: skip the // bare-metal Linux catalog entirely so we don't emit false positives like // fix-4-render-group / fix-5-amdgpu-load on a healthy WSL2 box. - let out_of_scope = wsl_out_of_scope_message(e); - let matched = if out_of_scope.is_some() { - Vec::new() + let (matched, out_of_scope) = if e.is_wsl { + // Most catalog checks inspect bare-metal Linux state that is irrelevant + // on WSL2. Keyword-only checks explicitly registered for `wsl` are safe + // to run there, however. `run_all_checks` already filters to just those + // (only fix-16-vllm-oom today), so an empty result here means no + // wsl-applicable checker fired at all -- that's the out-of-scope case. + // A *nonempty* result must be kept as-is, sub-threshold entries and all: + // `DiagnoseReport::matched`'s contract is every nonzero-score checker + // that fired, and dropping weak hits here would silently violate it (and + // would discard real signal -- e.g. a bare "out of memory" mention on + // WSL -- in favor of the generic out-of-scope routing message). + let wsl_matches = run_all_checks(e, symptom); + if wsl_matches.is_empty() { + (Vec::new(), wsl_out_of_scope_message(e)) + } else { + (wsl_matches, None) + } } else { - run_all_checks(e, symptom) + (run_all_checks(e, symptom), None) }; DiagnoseReport { has_match: any_cleared_threshold(&matched), @@ -1954,6 +2077,122 @@ mod tests { assert!(report.matched.iter().any(|d| d.id == "fix-11-iommu")); } + #[test] + fn vllm_oom_signature_is_a_high_confidence_match() { + // The distinctive vLLM startup OOM: torch/HIP allocation failure, with + // the vLLM anchor that tells this apart from an arbitrary PyTorch OOM. + let report = diagnose( + &linux_base(), + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + ); + let top = &report.matched[0]; + assert_eq!(top.id, "fix-16-vllm-oom"); + assert!(top.score >= HIGH_CONFIDENCE, "score was {}", top.score); + assert!(report.has_match()); + // The remediation must stay conditional: the reservation knob is right + // for a tenancy collision and wrong for a model that does not fit. + let fix = top.fix.as_ref().unwrap(); + assert!(!fix.auto_applicable, "OOM workaround must be print-only"); + assert!(fix.summary.contains("--gpu-memory-utilization")); + assert!( + fix.summary.contains("does not fit"), + "the 'genuinely does not fit' caveat must survive: {}", + fix.summary + ); + } + + #[test] + fn generic_pytorch_oom_without_a_vllm_signal_does_not_match() { + // The reviewed false positive: `torch.OutOfMemoryError: CUDA out of + // memory` scores 45+45=90 under the keyword table alone, but nothing + // about that string is vLLM-specific -- any ROCm PyTorch job emits the + // identical shape (ROCm's PyTorch build reports the CUDA-compat name). + // Without an explicit vLLM anchor, this failure mode must not fire. + let report = diagnose(&linux_base(), "torch.OutOfMemoryError: CUDA out of memory"); + assert!( + report.matched.iter().all(|d| d.id != "fix-16-vllm-oom"), + "a bare framework OOM with no vLLM signal must not be attributed to vLLM: {:?}", + report.matched + ); + assert!(!report.has_match()); + } + + #[test] + fn torch_oom_class_alone_is_not_a_vllm_match() { + // `outofmemory` is nested inside the exception class name. It must not + // count as a second, independent signal and turn one token into a + // high-confidence vLLM diagnosis for an arbitrary PyTorch workload. + let report = diagnose(&linux_base(), "vllm: torch.OutOfMemoryError"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the weak keyword signal remains visible"); + assert!(oom.score < MIN_SCORE_FOR_MATCH, "score was {}", oom.score); + assert!(!report.has_match()); + } + + #[test] + fn a_bare_out_of_memory_stays_below_the_match_threshold() { + // Without the vLLM/HIP shape, "out of memory" alone must not claim this + // failure mode -- it scores (the `vllm` anchor is present and "out of + // memory" is a keyword), but stays below MIN_SCORE_FOR_MATCH. + let report = diagnose(&linux_base(), "vllm: the process was killed: out of memory"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the weak `out of memory` keyword signal must remain visible in `matched`"); + assert!( + oom.score < MIN_SCORE_FOR_MATCH, + "a bare OOM must stay sub-threshold, got {}", + oom.score + ); + } + + #[test] + fn vllm_oom_is_not_available_on_native_windows() { + // vLLM is Linux/WSL-only (native Windows is unsupported), so the checker + // must not fire on a Windows examination even with the error text. + let e = Examination { + os_family: "windows".to_owned(), + ..Examination::default() + }; + let report = diagnose(&e, "vllm: torch.OutOfMemoryError: HIP out of memory."); + assert!(report.matched.iter().all(|d| d.id != "fix-16-vllm-oom")); + } + + #[test] + fn vllm_oom_keyword_diagnosis_is_available_on_wsl() { + let mut e = linux_base(); + e.is_wsl = true; + let report = diagnose( + &e, + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + ); + assert!(report.out_of_scope.is_none()); + assert!(report.has_match()); + assert_eq!(report.matched[0].id, "fix-16-vllm-oom"); + } + + #[test] + fn wsl_sub_threshold_vllm_signal_is_preserved_not_routed_out_of_scope() { + // A weak (sub-threshold) vLLM OOM signal on WSL must still surface in + // `matched` per the `DiagnoseReport::matched` contract -- it must not + // be silently dropped in favor of the out-of-scope routing message. + let mut e = linux_base(); + e.is_wsl = true; + let report = diagnose(&e, "vllm: out of memory"); + assert!(report.out_of_scope.is_none()); + assert!(!report.has_match()); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("a nonzero sub-threshold hit must stay in `matched`, not be dropped"); + assert!(oom.score < MIN_SCORE_FOR_MATCH, "score was {}", oom.score); + } + #[test] fn keyword_score_takes_top_two() { // INVALID_ISA: two hits (50 + 40) -> 90, not the sum of all. diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 695f86009..16cfccdda 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -364,6 +364,30 @@ const RECIPES: &[FixRecipe] = &[ applies_on: WINDOWS_ONLY, runner: None, }, + FixRecipe { + fix_id: "fix-16-vllm-oom", + title: "vLLM ran the GPU out of memory at startup", + rationale: "vLLM reserves a fixed fraction (~90%) of each GPU's TOTAL VRAM for its KV cache by default, regardless of the model size or how much is currently free. Two different faults surface the same OOM, and they need opposite responses. If the GPU is shared or already busy, that reservation collides with memory in use and lowering it (or moving to a less-busy GPU) is the fix. If the model genuinely does not fit in this GPU's VRAM, lowering the reservation only trades an earlier OOM for a later one -- use a smaller or quantized model instead (rocm-cli serves one model on a single GPU; it does not shard a model across GPUs).", + auto_applicable: false, + commands: &[ + "# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,", + "# or steer vLLM onto a less-busy device:", + "rocm serve --gpu-memory-utilization 0.5", + "rocm serve --gpu ", + "# Case 2 -- the model genuinely does not fit: the reservation is not the", + "# problem; pick a smaller or quantized model (single-GPU serving only).", + ], + needs_sudo: false, + needs_reboot: false, + needs_relogin: false, + verify: "rocm serve # re-run and watch for a clean startup", + notes: &[ + "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.", + "This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).", + ], + applies_on: LINUX_ONLY, + runner: None, + }, ]; fn find_recipe(fix_id: &str) -> Option<&'static FixRecipe> { @@ -459,7 +483,7 @@ pub fn apply(fix_id: &str, opts: &FixOptions) -> i32 { if looks_like_a_diagnosis_position(fix_id) { // `rocm diagnose` ranks findings `#1`, `#2`, and users reach for that // number here. It is a position in one report, not a name -- and it - // does not line up with the catalog's `fix-1 … fix-15` either, so a + // does not line up with the catalog's `fix-1 … fix-16` either, so a // bare "unknown id" left them with nothing to correct. eprintln!( "`{fix_id}` looks like a position in a `rocm diagnose` report, not a fix-id." @@ -1076,7 +1100,7 @@ mod tests { let count = ids.len(); ids.dedup(); assert_eq!(ids.len(), count, "duplicate fix-id in RECIPES"); - assert_eq!(count, 15, "expected 15 catalog entries"); + assert_eq!(count, 16, "expected 16 catalog entries"); } #[test] @@ -1107,6 +1131,12 @@ mod tests { "fix-9-igpu-dgpu" ] ); + // fix-16-vllm-oom carries a workaround the user must weigh (lowering the + // VRAM reservation is wrong for a model that genuinely does not fit), so + // it is deliberately PRINT-ONLY and must never join the AUTO set. + let oom = find_recipe("fix-16-vllm-oom").expect("OOM recipe is in the catalog"); + assert!(!oom.auto_applicable, "fix-16-vllm-oom must stay PRINT-ONLY"); + assert!(oom.runner.is_none(), "a PRINT-ONLY recipe has no runner"); } #[test] diff --git a/docs/vllm.md b/docs/vllm.md index cd657bde6..232cf7abd 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -170,7 +170,8 @@ a tiny model. rocm-cli helps in three ways: - **OOM failures hint the workaround.** When a startup failure log shows an out-of-memory error, the failure message suggests retrying with a smaller reservation, e.g. `--gpu-memory-utilization 0.1`, or targeting a less-busy GPU - with `--gpu `. + with `--gpu `, and points at `rocm diagnose --symptom ''` + for the full conditional remediation (busy GPU vs. a model that does not fit). Explicitly, the workaround for an OOM on a shared card is: @@ -178,6 +179,8 @@ Explicitly, the workaround for an OOM on a shared card is: rocm serve --gpu-memory-utilization 0.1 # optionally target a less-busy GPU rocm serve --gpu 3 --gpu-memory-utilization 0.1 +# for the full busy-GPU-vs-model-too-large breakdown, pass the error to diagnose +rocm diagnose --symptom 'vllm: torch.OutOfMemoryError: HIP out of memory' ``` ### Tool calling diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index c574b715d..a429a474c 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -2227,14 +2227,23 @@ fn startup_log_context(log_path: Option<&Path>) -> String { /// with the `rocm` CLI's pre-launch low-VRAM note so both surfaces point the /// user at the same fix rather than drifting into different phrasing. fn oom_utilization_hint(log_tail: &str) -> String { - if log_tail_shows_oom(log_tail) { - format!( - "\n\nDetected an out-of-memory failure. {}", - rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT - ) - } else { - String::new() + if !log_tail_shows_oom(log_tail) { + return String::new(); } + // Route the user's *actual* failing line into the `--symptom` example + // rather than a canned string, prefixed with a `vllm:` anchor so the + // diagnose checker attributes it to this failure mode. + let symptom_line = log_tail + .lines() + .rev() + .map(str::trim) + .find(|line| !line.is_empty() && log_tail_shows_oom(line)) + .unwrap_or("out of memory"); + format!( + "\n\nDetected an out-of-memory failure. {}\n\ + For conditional remediation, run `rocm diagnose --symptom 'vllm: {symptom_line}'`.", + rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT + ) } /// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a @@ -2716,6 +2725,16 @@ mod tests { "an OOM tail must surface the utilization workaround: {hint}" ); assert!(hint.contains("--gpu ")); + assert!( + hint.contains("rocm diagnose --symptom"), + "an OOM tail must route the user to the conditional catalog entry: {hint}" + ); + assert!( + hint.contains( + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB." + ), + "the user's real failing line must be routed into --symptom, not a canned string: {hint}" + ); // Detection is case-insensitive and also matches the spaced phrasing. assert!(log_tail_shows_oom("HIP OUT OF MEMORY")); diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index f49bcd8df..43dcbd588 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -140,3 +140,13 @@ Feature: Diagnosing failures and listing fixes When the user asks the CLI which fixes it offers Then every fix the catalog documents is listed And only the fixes the CLI can carry out itself are marked as such + + # vLLM runs on Linux and WSL, but not native Windows. This scenario is + # GPU-independent: it supplies the captured startup error as symptom text and + # proves the public diagnosis output preserves both branches of the remedy. + @id:diagnose-vllm-oom-is-conditional @requires-os:linux + Scenario: diagnose-13 - A vLLM startup OOM receives conditional remediation + Given a user whose vLLM server ran out of GPU memory + When the user asks the CLI to diagnose that symptom in machine-readable form + Then the diagnosis identifies the vLLM startup OOM + And the OOM remedy distinguishes a busy GPU from a model that does not fit diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 6001e9f55..2eaa60090 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -64,11 +64,14 @@ pub struct ScenarioDecl { /// in-tree amdgpu driver, so it does not hold under WSL2 — which uses /// `/dev/dxg` and the Windows host driver instead. /// - /// Two things stop short there. `rocm diagnose` skips its bare-metal catalog - /// *by design* rather than emitting Linux diagnoses that cannot apply, so a - /// scenario needing a catalog match has no premise. And `examine`'s probe - /// returns as soon as it recognises WSL2, before most of its steps run, so a - /// scenario asserting anything those steps populate has none either. + /// Two things stop short there. `rocm diagnose` skips its *bare-metal* + /// catalog *by design* rather than emitting Linux diagnoses that cannot + /// apply, so a scenario needing one of those bare-metal matches has no + /// premise. (Keyword-only checks explicitly registered for WSL — e.g. the + /// vLLM startup-OOM entry — do run there; those belong under + /// `@requires-os:linux`, not this tag.) And `examine`'s probe returns as + /// soon as it recognises WSL2, before most of its steps run, so a scenario + /// asserting anything those steps populate has none either. /// /// This is a SKIP and not an xfail row: the diagnose half is deliberate and /// separately unit-tested, and recording it as a known bug would tell every diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 74272ed19..87e25508d 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -57,6 +57,7 @@ const CATALOG_FIX_IDS: &[&str] = &[ "fix-13-hip-sdk-missing", "fix-14-adrenalin-too-old", "fix-15-msvc-redist", + "fix-16-vllm-oom", ]; /// The fixes the CLI carries out itself. Every other entry only prints a plan. @@ -117,6 +118,13 @@ async fn user_hit_unknown_failure(world: &mut E2eWorld) { world.model_name = Some("xyzzy totally unrelated gibberish".to_string()); } +#[given("a user whose vLLM server ran out of GPU memory")] +async fn user_hit_vllm_oom(world: &mut E2eWorld) { + world.model_name = Some( + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.".to_string(), + ); +} + #[given("a user who has chosen a known fix")] async fn user_chose_known_fix(world: &mut E2eWorld) { world.model_name = Some(PREVIEW_FIX_ID.to_string()); @@ -323,6 +331,70 @@ async fn assert_json_identifies_match(world: &mut E2eWorld) { ); } +/// Locate the `fix-16-vllm-oom` diagnosis in a parsed report's `matched` array, +/// panicking with the raw output if it is absent. Shared by the two `Then` +/// steps that assert on the vLLM OOM entry. +fn find_vllm_oom<'a>(report: &'a serde_json::Value, output: &str) -> &'a serde_json::Value { + report + .get("matched") + .and_then(serde_json::Value::as_array) + .and_then(|matches| { + matches.iter().find(|diagnosis| { + diagnosis.get("id").and_then(serde_json::Value::as_str) == Some("fix-16-vllm-oom") + }) + }) + .unwrap_or_else(|| panic!("expected fix-16-vllm-oom in diagnosis:\n{output}")) +} + +#[then("the diagnosis identifies the vLLM startup OOM")] +async fn assert_diagnosis_identifies_vllm_oom(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "diagnose should exit 0"); + let (report, output) = parsed_diagnosis(world); + let oom = find_vllm_oom(&report, &output); + let high_confidence = report + .get("high_confidence_threshold") + .and_then(serde_json::Value::as_i64) + .unwrap_or_else(|| panic!("report must publish its high_confidence_threshold:\n{output}")); + assert!( + oom.get("score") + .and_then(serde_json::Value::as_i64) + .is_some_and(|score| score >= high_confidence), + "the distinctive OOM signature should be high-confidence:\n{output}" + ); + assert_eq!( + oom.pointer("/fix/auto_applicable") + .and_then(serde_json::Value::as_bool), + Some(false), + "the OOM remedy must remain print-only:\n{output}" + ); +} + +#[then("the OOM remedy distinguishes a busy GPU from a model that does not fit")] +async fn assert_oom_remedy_is_conditional(world: &mut E2eWorld) { + let (report, output) = parsed_diagnosis(world); + let fix = find_vllm_oom(&report, &output) + .get("fix") + .unwrap_or_else(|| panic!("expected fix-16-vllm-oom remediation:\n{output}")); + let rendered = fix.to_string(); + assert!( + rendered.contains("shared/busy") + && rendered.contains("does not fit") + && rendered.contains("--gpu-memory-utilization") + && rendered.contains("smaller or quantized"), + "the remedy must retain both conditional branches:\n{output}" + ); + assert!( + !rendered.contains("--tensor-parallel-size"), + "the remedy must not prescribe the non-existent multi-GPU sharding flag:\n{output}" + ); + assert!( + !fix.get("verify") + .and_then(serde_json::Value::as_str) + .is_some_and(|verify| verify.contains("--gpu-memory-utilization")), + "verification must not unconditionally prescribe the tenancy workaround:\n{output}" + ); +} + /// Hold the report to its own arithmetic: `has_match` is true exactly when some /// cause cleared the threshold the report itself declares. Returns the ids that /// cleared it. From 235adb907bfb8412f8d4865e043c6cf3aa5f1bd7 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 28 Aug 2026 11:21:01 +0300 Subject: [PATCH 05/39] fix(serve): validate --gpu against the usable-device set, not its length (EAI-8058) When amd-smi is unavailable, `--gpu ` validation fell back to the *length* of usable_amd_gpu_indices(). That set holds absolute, unrenumbered ordinals after the visibility mask, so on a masked multi-GPU host (e.g. HIP_VISIBLE_DEVICES=2 on a 4-GPU box -> usable [2]) the length-1 bound rejected --gpu 2 (the sole usable device) and accepted a hidden --gpu 0. Replace the count bound with an injectable validate_pinned_gpu_index_against that prefers the amd-smi count and otherwise checks index membership in the usable set -- the same authority the engine device gate uses. The detected/usable seam makes both branches unit-testable without hardware. Also correct two docs/vllm.md inaccuracies (note timing; missing --engine vllm in the workaround examples). Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 138 ++++++++++++++++++++++++++++-------------- docs/vllm.md | 12 ++-- 2 files changed, 100 insertions(+), 50 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index a00897769..4daaa620e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -18416,29 +18416,13 @@ fn resolve_gpu_indices( /// Resolve an explicit `--gpu ` to its pinned ordinal — the `Index` arm /// of [`resolve_gpu_indices`], split out so the validation path can be -/// unit-tested against an injected device count. +/// unit-tested against injected GPU authorities. fn resolve_pinned_gpu_index(index: u32, detected: Option) -> Result> { - validate_pinned_gpu_index(index, pinned_index_bound(detected)) -} - -/// The device count to validate an explicit `--gpu ` against. Prefers the -/// `amd-smi` `list` count; when that is unavailable (`detected` is `None`) it -/// falls back to the amd-smi-independent KFD/DRM authority behind -/// [`rocm_core::usable_amd_gpu_indices`] — the same source the engine device -/// gate itself trusts. -/// -/// Deliberately does NOT borrow the DRM sysfs VRAM-fallback row count: that -/// probe withholds telemetry entirely on multi-GPU hosts and drops a card on a -/// transient counter-read failure, so using its length as a validation bound -/// would hard-reject legitimate `--gpu ` values on exactly those hosts. -/// VRAM rows stay scoped to `--gpu auto` ranking (see [`effective_gpu_count`]). -/// `None` when no count source is available, leaving the index unvalidated so -/// serving can still proceed where GPU probing is impossible. -fn pinned_index_bound(detected: Option) -> Option { - detected - .filter(|&count| count > 0) - .or_else(|| rocm_core::usable_amd_gpu_indices().map(|indices| indices.len())) - .filter(|&count| count > 0) + validate_pinned_gpu_index_against( + index, + detected, + rocm_core::usable_amd_gpu_indices().as_deref(), + ) } /// The GPU count to rank `--gpu auto` candidates over. Prefers the `amd-smi` @@ -18446,25 +18430,52 @@ fn pinned_index_bound(detected: Option) -> Option { /// DRM sysfs VRAM fallback still produced rows, its length stands in so /// auto-selection can still consider those devices. Used only for `--gpu auto` /// ranking, never as a `--gpu ` validation bound — see -/// [`pinned_index_bound`]. `None` when neither source is available. +/// [`validate_pinned_gpu_index_against`]. `None` when neither source is available. fn effective_gpu_count(detected: Option, vram: Option<&[GpuVramUsage]>) -> Option { detected .filter(|&count| count > 0) .or_else(|| vram.map(<[GpuVramUsage]>::len).filter(|&count| count > 0)) } -/// Validate an explicit `--gpu ` against the detected GPU count and -/// return the single pinned ordinal. Errors when the index is out of range for -/// a known device count; an unknown count (amd-smi unavailable and the DRM -/// sysfs fallback found no rows either) is allowed through so serving can -/// still proceed where GPU probing is not possible at all. -fn validate_pinned_gpu_index(index: u32, detected: Option) -> Result> { - if let Some(count) = detected - && (index as usize) >= count +/// Validate an explicit `--gpu ` against whichever GPU authority is +/// available — both injected so each branch is unit-testable — and return the +/// single pinned ordinal. +/// +/// Prefers the `amd-smi` `list` count (`detected`), a renumbered `0..count` +/// space, so an index at or beyond it is out of range. When amd-smi is +/// unavailable (`detected` is `None`) it falls back to *membership* in the +/// amd-smi-independent KFD/DRM usable set (`usable`) — the same authority the +/// engine device gate trusts. Those are absolute, unrenumbered ordinals (e.g. +/// `HIP_VISIBLE_DEVICES=2` on a 4-GPU host yields `[2]`), so a valid index must +/// be one of them, not merely below their count; validating against the length +/// would hard-reject the sole usable device on exactly that host. When neither +/// source is available the index is left unvalidated so serving can still +/// proceed where GPU probing is impossible; the engine device gate stays the +/// backstop. An empty `usable` set is not treated as authoritative here — the +/// caller's no-usable-GPU fail-fast owns that message — so it does not reject. +fn validate_pinned_gpu_index_against( + index: u32, + detected: Option, + usable: Option<&[u32]>, +) -> Result> { + if let Some(count) = detected.filter(|&count| count > 0) { + if (index as usize) >= count { + bail!( + "--gpu index {index} is out of range; {count} GPU(s) detected (valid indices 0..{})", + count - 1 + ); + } + } else if let Some(usable) = usable.filter(|indices| !indices.is_empty()) + && !usable.contains(&index) { bail!( - "--gpu index {index} is out of range; {count} GPU(s) detected (valid indices 0..{})", - count - 1 + "--gpu index {index} is not among the usable AMD GPU(s) [{}]; \ + it may be hidden by HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES", + usable + .iter() + .map(u32::to_string) + .collect::>() + .join(", ") ); } Ok(vec![index]) @@ -25843,19 +25854,29 @@ install therock"; #[test] fn validate_pinned_gpu_index_rejects_out_of_range() { - // Index equal to or beyond the detected count is rejected. - let error = validate_pinned_gpu_index(4, Some(4)).expect_err("index 4 is out of range"); + // Index equal to or beyond the detected amd-smi count is rejected. + let error = validate_pinned_gpu_index_against(4, Some(4), None) + .expect_err("index 4 is out of range"); assert!(error.to_string().contains("out of range")); - assert!(validate_pinned_gpu_index(9, Some(2)).is_err()); + assert!(validate_pinned_gpu_index_against(9, Some(2), None).is_err()); } #[test] fn validate_pinned_gpu_index_accepts_in_range_or_unknown_count() { // In-range index pins exactly that ordinal. - assert_eq!(validate_pinned_gpu_index(0, Some(1)).unwrap(), vec![0]); - assert_eq!(validate_pinned_gpu_index(3, Some(4)).unwrap(), vec![3]); - // Unknown count (amd-smi unavailable) is allowed through unvalidated. - assert_eq!(validate_pinned_gpu_index(7, None).unwrap(), vec![7]); + assert_eq!( + validate_pinned_gpu_index_against(0, Some(1), None).unwrap(), + vec![0] + ); + assert_eq!( + validate_pinned_gpu_index_against(3, Some(4), None).unwrap(), + vec![3] + ); + // No count and no usable set (GPU unprobeable): allowed through unvalidated. + assert_eq!( + validate_pinned_gpu_index_against(7, None, None).unwrap(), + vec![7] + ); } #[test] @@ -25883,11 +25904,38 @@ install therock"; } #[test] - fn pinned_index_bound_prefers_the_amd_smi_count() { - // When amd-smi reports a device count, that is the validation bound; a - // zero count is treated as "no usable source" rather than a hard zero. - assert_eq!(pinned_index_bound(Some(4)), Some(4)); - assert_eq!(pinned_index_bound(Some(0)), None); + fn validate_pinned_gpu_index_prefers_the_amd_smi_count() { + // When amd-smi reports a device count, that is the validation authority: + // an index within it passes and one at/beyond it is out of range, + // regardless of the usable set. + assert_eq!( + validate_pinned_gpu_index_against(3, Some(4), None).unwrap(), + vec![3] + ); + assert!(validate_pinned_gpu_index_against(3, Some(2), Some(&[3])).is_err()); + } + + #[test] + fn validate_pinned_gpu_index_falls_back_to_usable_set_membership() { + // amd-smi unavailable (`detected == None`): validation switches from a + // count bound to membership in the KFD/DRM usable set, which carries + // absolute, unrenumbered ordinals. On a 4-GPU host with + // `HIP_VISIBLE_DEVICES=2` the usable set is `[2]`, so `--gpu 2` — the + // sole usable device — must pass even though its ordinal exceeds the + // set's length, and a hidden device must be rejected. + assert_eq!( + validate_pinned_gpu_index_against(2, None, Some(&[2])).unwrap(), + vec![2] + ); + let error = validate_pinned_gpu_index_against(0, None, Some(&[2, 3])) + .expect_err("index 0 is hidden by the visibility mask"); + assert!(error.to_string().contains("not among the usable")); + // An empty usable set is not authoritative here (the caller's GPU + // fail-fast owns the "no usable GPU" message), so it does not reject. + assert_eq!( + validate_pinned_gpu_index_against(1, None, Some(&[])).unwrap(), + vec![1] + ); } #[test] diff --git a/docs/vllm.md b/docs/vllm.md index 232cf7abd..bd2236c43 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -165,8 +165,9 @@ a tiny model. rocm-cli helps in three ways: withholds telemetry on a multi-GPU host, since its `card` numbering is not guaranteed to match HIP's device ordinal there. - **The serve summary warns on low free VRAM.** When the selected GPU is already - heavily used, `rocm serve` prints a note before launch and, for vLLM, points at - `--gpu-memory-utilization` as the fix. + heavily used, `rocm serve` prints a note — before launch on the plain path, or + in the post-readiness summary in the default interactive mode — and, for vLLM, + points at `--gpu-memory-utilization` as the fix. - **OOM failures hint the workaround.** When a startup failure log shows an out-of-memory error, the failure message suggests retrying with a smaller reservation, e.g. `--gpu-memory-utilization 0.1`, or targeting a less-busy GPU @@ -176,9 +177,10 @@ a tiny model. rocm-cli helps in three ways: Explicitly, the workaround for an OOM on a shared card is: ```bash -rocm serve --gpu-memory-utilization 0.1 -# optionally target a less-busy GPU -rocm serve --gpu 3 --gpu-memory-utilization 0.1 +```bash +rocm serve --engine vllm --gpu-memory-utilization 0.1 +# optionally target a specific, less-busy GPU by index +rocm serve --engine vllm --gpu 1 --gpu-memory-utilization 0.1 # for the full busy-GPU-vs-model-too-large breakdown, pass the error to diagnose rocm diagnose --symptom 'vllm: torch.OutOfMemoryError: HIP out of memory' ``` From 77f26cd0096a6a91d8b0022d4eda6ae99a053529 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 28 Aug 2026 11:09:17 +0000 Subject: [PATCH 06/39] docs(vllm): remove duplicate code-fence opener in OOM workaround example A merge-conflict resolution left two consecutive ```bash fence openers before the shared-card OOM workaround example, so the block opened and immediately reopened on the literal text and the commands rendered outside a code block. Drop the duplicate opener so the example renders as a single fenced block. Signed-off-by: Roman Sirokov --- docs/vllm.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/vllm.md b/docs/vllm.md index bd2236c43..ecdb858ec 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -176,7 +176,6 @@ a tiny model. rocm-cli helps in three ways: Explicitly, the workaround for an OOM on a shared card is: -```bash ```bash rocm serve --engine vllm --gpu-memory-utilization 0.1 # optionally target a specific, less-busy GPU by index From e62ee2f162c0fe8a16cbad4032008f86cba2ab4b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 3 Sep 2026 08:30:04 +0000 Subject: [PATCH 07/39] =?UTF-8?q?serve:=20address=20OOM=20diagnosis=20revi?= =?UTF-8?q?ew=20=E2=80=94=20decouple=20engine=20hint=20from=20scorer,=20de?= =?UTF-8?q?dup=20keyword=20spans,=20anchor=20per-line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer follow-ups on the vLLM OOM diagnosis path: - Engine no longer hard-codes `vllm: ` into the printed `rocm diagnose --symptom`. It now consults rocm_core::vllm_oom_symptom_is_diagnosable and falls back to VLLM_OOM_CANONICAL_SYMPTOM when the user's actual line would not score, so the command it prints always reports a cause. The user's real line stays visible in the human-readable message. Adds a loop-invariant test that every emitted --symptom value is diagnosable. - keyword_score now de-duplicates overlapping regex spans before taking the top two, so a phrase nested inside a stronger one is not double-counted. The VLLM_OOM table splits allocator *messages* (score 50, clear alone) from the exception *class* name (45, sub-threshold). - vLLM OOM anchor must appear on the same line as an OOM token, not merely somewhere in a pasted log, closing a misattribution gap. - WSL out-of-scope routing is gated on has_match rather than a non-empty match list, so a sub-threshold signal is preserved and still routed out. - serve --gpu membership rejection now says "not available" and lists the usable set, matching the engine wording and the e2e contract. - --gpu auto ranks the actual reported VRAM row indices (which can be non-contiguous under a visibility mask) instead of a synthetic 0..count. - diagnosis Fix for shared fix ids (fix-15, fix-16) is cross-checked against the fix catalog so verify/notes/commands cannot silently diverge. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 53 ++++- crates/rocm-core/src/diagnose.rs | 337 +++++++++++++++++++++++++++---- crates/rocm-core/src/fix.rs | 19 ++ crates/rocm-core/src/lib.rs | 4 +- engines/vllm/src/lib.rs | 50 ++++- 5 files changed, 408 insertions(+), 55 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 4daaa620e..21f909177 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -18469,8 +18469,8 @@ fn validate_pinned_gpu_index_against( && !usable.contains(&index) { bail!( - "--gpu index {index} is not among the usable AMD GPU(s) [{}]; \ - it may be hidden by HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES", + "--gpu index {index} is not available; the usable AMD GPU(s) are [{}] \ + (others may be hidden by HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES)", usable .iter() .map(u32::to_string) @@ -18553,12 +18553,24 @@ fn select_auto_gpu_index( // or fail fast under the GPU-required policy (no GPU-0 fallback). return Vec::new(); } - let candidates = || (0..count as u32).filter(|index| !busy.contains(index)); + // Candidate ordinals to rank, lowest first. When VRAM telemetry is present, + // drive the scan from the *actual* row indices it reports rather than a + // synthetic `0..count` range: amd-smi metric (and the DRM sysfs fallback) + // can report non-contiguous ordinals when devices are hidden by a + // visibility mask, so `0..count` would look up rows that do not exist and + // miss the real ones. Without telemetry, fall back to the dense detected + // range so service-state-only selection still has candidates. + let mut candidate_indices: Vec = match vram { + Some(rows) => rows.iter().map(|row| row.index).collect(), + None => (0..count as u32).collect(), + }; + candidate_indices.sort_unstable(); + candidate_indices.retain(|index| !busy.contains(index)); let usage_for = |index: u32| vram.and_then(|rows| rows.iter().find(|row| row.index == index)); if vram.is_some() { // Pass 1: lowest-index idle GPU. - for index in candidates() { + for &index in &candidate_indices { if let Some(free) = usage_for(index).and_then(|usage| usage.free_fraction()) && free >= AUTO_FREE_VRAM_FRACTION { @@ -18568,9 +18580,9 @@ fn select_auto_gpu_index( // Pass 2: the non-busy GPU with the most free VRAM in absolute terms // (not free percentage, which can favor a smaller GPU on heterogeneous // VRAM systems). - if let Some(index) = candidates().max_by(|left, right| { - let left_free = usage_for(*left).map(|usage| usage.free_mb()); - let right_free = usage_for(*right).map(|usage| usage.free_mb()); + if let Some(&index) = candidate_indices.iter().max_by(|left, right| { + let left_free = usage_for(**left).map(|usage| usage.free_mb()); + let right_free = usage_for(**right).map(|usage| usage.free_mb()); left_free .cmp(&right_free) // Break ties toward the lowest index. @@ -18582,7 +18594,7 @@ fn select_auto_gpu_index( } // Pass 3: no VRAM telemetry; first GPU not pinned by a managed service. - if let Some(index) = candidates().next() { + if let Some(&index) = candidate_indices.first() { return vec![index]; } // Every detected GPU is pinned by a running service. We still return GPU 0 @@ -25852,6 +25864,28 @@ install therock"; ); } + #[test] + fn auto_selection_ranks_the_actual_reported_row_indices() { + // A visibility mask hides GPUs 0 and 1, so amd-smi metric reports only + // the surviving devices with their absolute, non-contiguous ordinals + // [2, 3] -- there is no row at index 0 or 1. Auto-selection must scan + // those real indices, not a synthetic `0..count` range (which would + // look up absent rows and fall through to a bogus GPU 0). + let usage = [vram(2, 182_000, 192_000), vram(3, 1_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(None, &[], Some(&usage)), + vec![3], + "should skip the busy GPU 2 and pick the idle GPU 3 by its real ordinal" + ); + // With GPU 3 also pinned by a managed service, pass 3 still returns a + // real reported ordinal (GPU 2), never a fabricated index 0. + assert_eq!( + select_auto_gpu_index(None, &[3], Some(&usage)), + vec![2], + "the sole non-busy reported GPU must be selected by its real ordinal" + ); + } + #[test] fn validate_pinned_gpu_index_rejects_out_of_range() { // Index equal to or beyond the detected amd-smi count is rejected. @@ -25929,7 +25963,8 @@ install therock"; ); let error = validate_pinned_gpu_index_against(0, None, Some(&[2, 3])) .expect_err("index 0 is hidden by the visibility mask"); - assert!(error.to_string().contains("not among the usable")); + assert!(error.to_string().contains("not available")); + assert!(error.to_string().contains("[2, 3]")); // An empty usable set is not authoritative here (the caller's GPU // fail-fast owns the "no usable GPU" message), so it does not reject. assert_eq!( diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 417c1aedb..58f56afb4 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -82,9 +82,12 @@ pub struct DiagnoseReport { pub min_score_for_match: i32, pub high_confidence_threshold: i32, pub route_when_no_match: Route, - /// Set when the host is out of scope for this catalog (e.g. WSL2). When - /// present, `matched` is empty — the catalog is deliberately not run, to - /// avoid emitting bare-metal-Linux diagnoses that don't apply. + /// Set when the host is out of scope for this catalog (e.g. WSL2). Present + /// whenever no checker cleared [`MIN_SCORE_FOR_MATCH`] on such a host; the + /// bare-metal Linux catalog is deliberately not run, so this carries the + /// routing note instead. `matched` may still hold sub-threshold entries + /// (e.g. a weak `out of memory` mention) alongside it — read + /// [`DiagnoseReport::has_match`] to tell a real cause from a weak signal. #[serde(default)] pub out_of_scope: Option, } @@ -297,24 +300,49 @@ const KEYWORDS_PAGE_FAULT: KeywordTable = &[ // CUDA out of memory` is the identical shape any ROCm PyTorch job emits (ROCm's // PyTorch build reports the CUDA-compat name), so this table alone cannot tell // vLLM's OOM apart from an arbitrary training script hitting the same -// allocator error. `check_16_vllm_oom` therefore requires `VLLM_ANCHOR_PATTERN` -// to match before scoring this table at all. +// allocator error. `check_16_vllm_oom` therefore requires an anchored OOM line +// (`VLLM_ANCHOR_PATTERN` on the same line as an OOM token) before scoring at all. +// +// Weighting splits the *allocator message* from the *exception class name*: +// - The HIP allocator's own messages (`HIP out of memory`, the +// `hipErrorOutOfMemory` status) and the CUDA-compat message ROCm's PyTorch +// prints clear `MIN_SCORE_FOR_MATCH` on their own — with the required +// anchor already establishing this is vLLM's line, one of those messages is +// a genuine OOM signal. +// - The bare exception *class* (`torch.OutOfMemoryError`, with or without the +// `.cuda.` module path) and a bare `out of memory` stay sub-threshold: any +// PyTorch job emits the class name, so it must corroborate rather than +// carry the verdict alone (`torch_oom_class_alone_is_not_a_vllm_match`). +// `keyword_score` de-duplicates overlapping spans, so `HIP out of memory` counts +// once (as the 50-point message), not also as the nested 25-point `out of +// memory` — one phrase yields one evidence bullet. const KEYWORDS_VLLM_OOM: KeywordTable = &[ - ( - r"torch\.outofmemoryerror", - 45, - "error mentions torch.OutOfMemoryError", - ), ( "hip out of memory", - 45, + 50, "error mentions 'HIP out of memory'", ), + ( + // ROCm's HIP runtime OOM status code, e.g. `RuntimeError: + // hipErrorOutOfMemory`. `\boutofmemory\b` does not match inside it (no + // word boundary after `hipError`), so it needs its own entry. + r"hiperroroutofmemory", + 50, + "error mentions hipErrorOutOfMemory", + ), ( r"cuda out of memory", - 45, + 50, "error mentions 'CUDA out of memory' (ROCm reports the CUDA-compat name)", ), + ( + // The exception class name, under either `torch.OutOfMemoryError` or the + // `.cuda.` module path ROCm's PyTorch reports it under. Sub-threshold + // alone: it is the class every PyTorch OOM raises, not vLLM's signature. + r"torch\.(?:cuda\.)?outofmemoryerror", + 45, + "error mentions torch.OutOfMemoryError", + ), ( r"tried to allocate .*(?:gib|mib)", 30, @@ -334,15 +362,21 @@ const KEYWORDS_VLLM_OOM: KeywordTable = &[ ]; /// Score the strongest (top-2) keyword matches in `table` against `symptom`. +/// +/// Matches whose spans overlap are collapsed to the strongest one, so a single +/// phrase cannot masquerade as two independent signals (e.g. `HIP out of memory` +/// must not also count the `out of memory` nested inside it). fn keyword_score(symptom: &str, table: KeywordTable) -> (i32, Vec) { if symptom.is_empty() { return (0, Vec::new()); } let sym = symptom.to_lowercase(); - let mut hits: Vec<(i32, &'static str)> = Vec::new(); + let mut hits: Vec<(i32, &'static str, usize, usize)> = Vec::new(); for (pattern, weight, label) in table { - if Regex::new(pattern).is_ok_and(|re| re.is_match(&sym)) { - hits.push((*weight, label)); + if let Ok(re) = Regex::new(pattern) + && let Some(m) = re.find(&sym) + { + hits.push((*weight, label, m.start(), m.end())); } } if hits.is_empty() { @@ -350,9 +384,18 @@ fn keyword_score(symptom: &str, table: KeywordTable) -> (i32, Vec) { } // Mirror diagnose.py's `hits.sort(reverse=True)`: weight desc, then label desc. hits.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| b.1.cmp(a.1))); - hits.truncate(2); - let score = hits.iter().map(|(w, _)| *w).sum(); - let labels = hits.iter().map(|(_, l)| (*l).to_owned()).collect(); + // Keep the strongest hit for any overlapping span: the same substring must + // not contribute two evidence bullets (nor two weights toward the score). + let mut kept: Vec<(i32, &'static str, usize, usize)> = Vec::new(); + for hit in hits { + let overlaps = kept.iter().any(|k| hit.2 < k.3 && k.2 < hit.3); + if !overlaps { + kept.push(hit); + } + } + kept.truncate(2); + let score = kept.iter().map(|(w, ..)| *w).sum(); + let labels = kept.iter().map(|(_, l, ..)| (*l).to_owned()).collect(); (score, labels) } @@ -1304,7 +1347,7 @@ fn check_15_msvc_redist(e: &Examination, symptom: &str) -> Diagnosis { fix_id: "fix-15-msvc-redist".to_owned(), auto_applicable: false, verify: "where vcruntime140.dll && where vcruntime140_1.dll".to_owned(), - notes: vec!["If installing the redistributable still leaves a missing-DLL error, the failing DLL is probably amdhip64_X.dll itself; that points at fix-13-hip-sdk-missing (the HIP SDK install) rather than this fix.".to_owned()], + notes: vec!["If installing the redistributable still leaves a missing-DLL error, the failing DLL is probably amdhip64_X.dll itself; that points at fix-13-hip-sdk-missing rather than this fix.".to_owned()], ..Fix::default() }; finalize( @@ -1325,13 +1368,60 @@ fn check_15_msvc_redist(e: &Examination, symptom: &str) -> Diagnosis { /// frameworks' OOMs to vLLM. const VLLM_ANCHOR_PATTERN: &str = r"vllm|gpu[-_]memory[-_]utilization"; +/// A canonical vLLM-OOM `--symptom` string guaranteed to clear +/// [`MIN_SCORE_FOR_MATCH`]. +/// +/// It carries two independent signals (the torch exception class *and* the HIP +/// allocator message) on one anchored line, so it scores even after overlap +/// de-duplication. The vLLM engine falls back to this when a user's actual +/// failing line would not itself be diagnosable, so the `rocm diagnose` command +/// it prints always reports a cause. +pub const VLLM_OOM_CANONICAL_SYMPTOM: &str = "vllm: torch.OutOfMemoryError: HIP out of memory"; + +/// Whether `symptom` carries the vLLM OOM anchor (`vllm` or +/// `gpu_memory_utilization`) on the *same line* as an OOM token from +/// [`KEYWORDS_VLLM_OOM`]. Co-occurrence anywhere in a pasted log is too loose -- +/// a stray `vllm` mention re-enables the very misattribution the anchor exists +/// to prevent (a `llama.cpp` OOM in a paste that also names vLLM). The engine's +/// emitted `vllm: ` satisfies this by construction. +fn vllm_oom_line_is_anchored(symptom: &str) -> bool { + let anchor = Regex::new(VLLM_ANCHOR_PATTERN); + let Ok(anchor) = anchor else { return false }; + symptom.lines().any(|line| { + let low = line.to_lowercase(); + anchor.is_match(&low) + && KEYWORDS_VLLM_OOM + .iter() + .any(|(pattern, _, _)| Regex::new(pattern).is_ok_and(|re| re.is_match(&low))) + }) +} + +/// Whether `symptom` clears [`MIN_SCORE_FOR_MATCH`] for the vLLM-OOM failure +/// mode. +/// +/// The vLLM engine calls this to decide whether the user's actual failing line +/// is worth routing into a `--symptom` example, or whether to fall back to +/// [`VLLM_OOM_CANONICAL_SYMPTOM`] so the printed command is guaranteed to report +/// a cause rather than reading as "the tool checked and found nothing". +#[must_use] +pub fn vllm_oom_symptom_is_diagnosable(symptom: &str) -> bool { + let e = Examination { + os_family: "linux".to_owned(), + ..Examination::default() + }; + check_16_vllm_oom(&e, symptom).score >= MIN_SCORE_FOR_MATCH +} + /// vLLM ran the GPU out of memory. Keyword-only: an [`Examination`] carries no /// per-GPU VRAM or tenancy fields, so nothing structural can corroborate this — /// the user must pass the error via `--symptom`, or arrive from the serve /// failure note that points here. `e` is therefore unused. fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { - if !symptom_matches(symptom, VLLM_ANCHOR_PATTERN) { - // No vLLM anchor: don't attribute a bare framework OOM to vLLM at all. + if !vllm_oom_line_is_anchored(symptom) { + // No vLLM anchor on an OOM line: don't attribute a bare framework OOM to + // vLLM at all. Requiring the anchor on the same line as the OOM token + // (not merely somewhere in the paste) keeps a stray `vllm` mention from + // re-enabling the misattribution the anchor exists to prevent. return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); } let (score, evidence) = keyword_score(symptom, KEYWORDS_VLLM_OOM); @@ -1362,9 +1452,10 @@ fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { ], fix_id: "fix-16-vllm-oom".to_owned(), auto_applicable: false, - verify: "rocm serve # re-run and watch for a clean startup".to_owned(), + verify: "rocm serve # re-run and watch for a clean startup".to_owned(), notes: vec![ - "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it does not create room a too-large model needs.".to_owned(), + "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.".to_owned(), + "This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).".to_owned(), ], ..Fix::default() }; @@ -1440,25 +1531,28 @@ fn route_when_no_match(e: &Examination) -> Route { pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { // WSL2 is a distinct platform (it uses /dev/dxg + the Windows host driver, // not the in-tree amdgpu module or /dev/kfd), and `examine` already treats - // it as out of scope (exit_code() == 2). Mirror that here: skip the - // bare-metal Linux catalog entirely so we don't emit false positives like + // it as out of scope (exit_code() == 2) -- unconditionally, independent of + // the symptom. Keep `out_of_scope` a host property here too: skip the + // bare-metal Linux catalog so we don't emit false positives like // fix-4-render-group / fix-5-amdgpu-load on a healthy WSL2 box. let (matched, out_of_scope) = if e.is_wsl { // Most catalog checks inspect bare-metal Linux state that is irrelevant // on WSL2. Keyword-only checks explicitly registered for `wsl` are safe // to run there, however. `run_all_checks` already filters to just those - // (only fix-16-vllm-oom today), so an empty result here means no - // wsl-applicable checker fired at all -- that's the out-of-scope case. - // A *nonempty* result must be kept as-is, sub-threshold entries and all: - // `DiagnoseReport::matched`'s contract is every nonzero-score checker - // that fired, and dropping weak hits here would silently violate it (and - // would discard real signal -- e.g. a bare "out of memory" mention on - // WSL -- in favor of the generic out-of-scope routing message). + // (only fix-16-vllm-oom today). + // + // Suppression of the routing note is gated on `has_match` (a checker + // *cleared* MIN_SCORE_FOR_MATCH), NOT on `matched` being non-empty. A + // sub-threshold hit -- e.g. a score-25 `out of memory` mention -- must + // not bury `WSL_ROUTE_OUT_NOTE`: the user whose real problem is a + // missing WSL driver still needs the routing note even if their paste + // happens to contain a weak OOM keyword. The weak hit is preserved in + // `matched` (the contract) regardless; only the routing message is gated. let wsl_matches = run_all_checks(e, symptom); - if wsl_matches.is_empty() { - (Vec::new(), wsl_out_of_scope_message(e)) - } else { + if any_cleared_threshold(&wsl_matches) { (wsl_matches, None) + } else { + (wsl_matches, wsl_out_of_scope_message(e)) } } else { (run_all_checks(e, symptom), None) @@ -2176,15 +2270,21 @@ mod tests { } #[test] - fn wsl_sub_threshold_vllm_signal_is_preserved_not_routed_out_of_scope() { + fn wsl_sub_threshold_vllm_signal_is_preserved_but_still_routed_out_of_scope() { // A weak (sub-threshold) vLLM OOM signal on WSL must still surface in - // `matched` per the `DiagnoseReport::matched` contract -- it must not - // be silently dropped in favor of the out-of-scope routing message. + // `matched` per the `DiagnoseReport::matched` contract -- but it must + // NOT suppress the out-of-scope routing note. Suppression is gated on + // `has_match` (score >= MIN_SCORE_FOR_MATCH), not on `matched` being + // non-empty, so a score-25 mention no longer buries the WSL routing note + // for a user whose real problem is a missing WSL driver. let mut e = linux_base(); e.is_wsl = true; let report = diagnose(&e, "vllm: out of memory"); - assert!(report.out_of_scope.is_none()); assert!(!report.has_match()); + assert!( + report.out_of_scope.is_some(), + "a sub-threshold hit must not suppress the WSL routing note" + ); let oom = report .matched .iter() @@ -2193,6 +2293,165 @@ mod tests { assert!(oom.score < MIN_SCORE_FOR_MATCH, "score was {}", oom.score); } + #[test] + fn a_genuine_vllm_oom_match_on_wsl_is_surfaced_not_routed_out_of_scope() { + // The other side of the has_match gate: a real, at-or-above-threshold + // vLLM OOM on WSL surfaces the diagnosis instead of the routing note. + let mut e = linux_base(); + e.is_wsl = true; + let report = diagnose(&e, VLLM_OOM_CANONICAL_SYMPTOM); + assert!(report.has_match()); + assert!(report.out_of_scope.is_none()); + assert_eq!(report.matched[0].id, "fix-16-vllm-oom"); + } + + #[test] + fn hip_error_out_of_memory_status_is_a_match() { + // Regression: `RuntimeError: hipErrorOutOfMemory` previously scored 0 + // (`\boutofmemory\b` has no word boundary after `hipError`), so the + // engine printed a diagnose command that reported nothing. + let report = diagnose(&linux_base(), "vllm: RuntimeError: hipErrorOutOfMemory"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("hipErrorOutOfMemory must now be recognized"); + assert!( + oom.score >= MIN_SCORE_FOR_MATCH, + "hipErrorOutOfMemory should clear the threshold, got {}", + oom.score + ); + } + + #[test] + fn torch_cuda_module_path_oom_class_is_recognized() { + // Regression: ROCm's PyTorch reports the class under the `.cuda.` module + // path (`torch.cuda.OutOfMemoryError`); the class pattern must match it. + // It is still the (sub-threshold) class name, not a stronger message. + let report = diagnose(&linux_base(), "vllm: torch.cuda.OutOfMemoryError"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the torch.cuda.* class name must be recognized as evidence"); + assert!(oom.score > 0, "score was {}", oom.score); + assert!( + oom.score < MIN_SCORE_FOR_MATCH, + "the bare class name must stay sub-threshold, got {}", + oom.score + ); + } + + #[test] + fn hip_out_of_memory_yields_a_single_evidence_line() { + // Overlap guard: "HIP out of memory" must not be counted both as the + // 50-point HIP message and the 25-point `out of memory` nested inside + // it. One phrase, one evidence bullet. + let report = diagnose(&linux_base(), "vllm: HIP out of memory"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("HIP out of memory must match"); + let fix = oom.fix.as_ref().unwrap(); + let evidence_lines = oom + .evidence + .iter() + .filter(|e| e.to_lowercase().contains("out of memory")) + .count(); + assert_eq!( + evidence_lines, 1, + "one phrase must yield one evidence line, got {:?}", + oom.evidence + ); + assert_eq!(oom.score, 50, "the HIP message carries the match alone"); + assert!(fix.summary.contains("--gpu-memory-utilization")); + } + + #[test] + fn vllm_anchor_must_be_on_the_same_line_as_the_oom_token() { + // Co-occurrence anywhere is too loose: a paste that mentions vLLM on one + // line and an unrelated framework's OOM on another must not attribute + // the OOM to vLLM. + let report = diagnose( + &linux_base(), + "I installed vllm last week.\nToday my llama.cpp run died: CUDA out of memory", + ); + assert!( + report.matched.iter().all(|d| d.id != "fix-16-vllm-oom"), + "an OOM on a different line from the vLLM mention must not match: {:?}", + report.matched + ); + // The engine's emitted `vllm: ` keeps them on one line and + // still matches. + let anchored = diagnose(&linux_base(), "vllm: CUDA out of memory"); + assert!(anchored.matched.iter().any(|d| d.id == "fix-16-vllm-oom")); + } + + #[test] + fn canonical_vllm_oom_symptom_is_diagnosable() { + // The engine's fallback symptom must always clear the threshold, so the + // `rocm diagnose` command it prints never reports nothing. + assert!(vllm_oom_symptom_is_diagnosable(VLLM_OOM_CANONICAL_SYMPTOM)); + // And a bare framework mention the engine would reject is not diagnosable. + assert!(!vllm_oom_symptom_is_diagnosable("vllm: out of memory")); + } + + #[test] + fn diagnosis_remediation_matches_the_fix_catalog_for_shared_fix_ids() { + // fix-15 and fix-16 live in two places: the `rocm fix` catalog + // (crate::fix::RECIPES) and the Fix these checks embed in a Diagnosis. + // The prose *framing* legitimately differs by surface (a catalog + // rationale vs a diagnosis summary), but the actionable remediation must + // not silently diverge -- the verify command, the notes, and the + // executable command lines have to agree. This is the guard that they + // do; it caught a two-vs-three-space `verify` and a reworded note. + let vllm_oom = check_16_vllm_oom( + &Examination { + os_family: "linux".to_owned(), + ..Examination::default() + }, + VLLM_OOM_CANONICAL_SYMPTOM, + ); + let msvc = check_15_msvc_redist( + &Examination { + os_family: "windows".to_owned(), + msvc_redist_present: Some(false), + ..Examination::default() + }, + "ImportError: DLL load failed: vcruntime140_1.dll not found", + ); + for diag in [vllm_oom, msvc] { + assert!( + diag.score >= MIN_SCORE_FOR_MATCH, + "test setup must yield a real match for {}", + diag.id + ); + let fix = diag.fix.expect("a matched diagnosis must carry a Fix"); + let (verify, notes, run_commands) = + crate::fix::recipe_verify_notes_and_run_commands(&fix.fix_id) + .expect("a shared fix_id must exist in the recipe catalog"); + assert_eq!(fix.verify, verify, "verify diverged for {}", fix.fix_id); + assert_eq!( + fix.notes, + notes.iter().map(|s| (*s).to_owned()).collect::>(), + "notes diverged for {}", + fix.fix_id + ); + let emitted_run = fix + .commands + .iter() + .map(String::as_str) + .filter(|line| !line.trim_start().starts_with('#')) + .collect::>(); + assert_eq!( + emitted_run, run_commands, + "executable command lines diverged for {}", + fix.fix_id + ); + } + } + #[test] fn keyword_score_takes_top_two() { // INVALID_ISA: two hits (50 + 40) -> 90, not the sum of all. diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 16cfccdda..296b288d6 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -394,6 +394,25 @@ fn find_recipe(fix_id: &str) -> Option<&'static FixRecipe> { RECIPES.iter().find(|r| r.fix_id == fix_id) } +/// Test-only view of a recipe's actionable remediation: its `verify` command, +/// its `notes`, and its *executable* command lines (comment/prose lines +/// stripped). Lets the diagnosis catalog cross-check that the Fix it emits for a +/// shared `fix_id` has not silently diverged from this recipe. +#[cfg(test)] +pub(crate) fn recipe_verify_notes_and_run_commands( + fix_id: &str, +) -> Option<(&'static str, &'static [&'static str], Vec<&'static str>)> { + find_recipe(fix_id).map(|recipe| { + let run_commands = recipe + .commands + .iter() + .copied() + .filter(|line| !line.trim_start().starts_with('#')) + .collect::>(); + (recipe.verify, recipe.notes, run_commands) + }) +} + const fn current_os() -> &'static str { if runtime_is_windows() { "windows" diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 5170a86c9..c80d4309a 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -36,8 +36,8 @@ pub mod proc_lifecycle; pub mod runtime; pub mod uv; pub use diagnose::{ - DiagnoseReport, Diagnosis, Fix, diagnose as run_diagnose, - render_report_text as render_diagnose_text, + DiagnoseReport, Diagnosis, Fix, VLLM_OOM_CANONICAL_SYMPTOM, diagnose as run_diagnose, + render_report_text as render_diagnose_text, vllm_oom_symptom_is_diagnosable, }; pub use disk_space::{ SpaceCheck, available_space_for_path, check_space_for_path, ensure_space_for, diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index a429a474c..ae67a8dac 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -2230,18 +2230,28 @@ fn oom_utilization_hint(log_tail: &str) -> String { if !log_tail_shows_oom(log_tail) { return String::new(); } - // Route the user's *actual* failing line into the `--symptom` example - // rather than a canned string, prefixed with a `vllm:` anchor so the - // diagnose checker attributes it to this failure mode. + // Route the user's *actual* failing line into the `--symptom` example when + // the diagnose checker would actually score it; otherwise fall back to the + // canonical symptom so the printed command always reports a cause. The + // detector here is a coarse substring scan that accepts lines the scorer + // rates sub-threshold (e.g. a bare "... out of memory"), so without this + // fallback the diagnose command could report nothing -- which reads as "the + // tool checked and there's no known cause", worse than not printing it. let symptom_line = log_tail .lines() .rev() .map(str::trim) .find(|line| !line.is_empty() && log_tail_shows_oom(line)) .unwrap_or("out of memory"); + let candidate = format!("vllm: {symptom_line}"); + let symptom = if rocm_core::vllm_oom_symptom_is_diagnosable(&candidate) { + candidate + } else { + rocm_core::VLLM_OOM_CANONICAL_SYMPTOM.to_owned() + }; format!( - "\n\nDetected an out-of-memory failure. {}\n\ - For conditional remediation, run `rocm diagnose --symptom 'vllm: {symptom_line}'`.", + "\n\nDetected an out-of-memory failure ({symptom_line}). {}\n\ + For conditional remediation, run `rocm diagnose --symptom '{symptom}'`.", rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT ) } @@ -2741,6 +2751,36 @@ mod tests { assert!(log_tail_shows_oom("RuntimeError: CUDA out of memory")); } + #[test] + fn every_emitted_oom_symptom_is_diagnosable() { + // Closes the loop between the two layers: the engine prints + // `rocm diagnose --symptom ''`, so whatever it emits must + // actually score for the diagnose checker -- including for lines the + // coarse substring detector accepts but the scorer rates sub-threshold + // on their own (those fall back to the canonical symptom). + let accepted_lines = [ + "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + "RuntimeError: hipErrorOutOfMemory", + "torch.cuda.OutOfMemoryError", + "HIP error: out of memory", + "the process was killed: out of memory", + "CUDA out of memory", + ]; + for line in accepted_lines { + assert!(log_tail_shows_oom(line), "detector must accept: {line}"); + let hint = oom_utilization_hint(line); + let symptom = hint + .split("--symptom '") + .nth(1) + .and_then(|rest| rest.split('\'').next()) + .expect("hint must carry a --symptom value"); + assert!( + rocm_core::vllm_oom_symptom_is_diagnosable(symptom), + "emitted symptom must be diagnosable, got {symptom:?} for line {line:?}" + ); + } + } + #[test] fn oom_utilization_hint_stays_quiet_for_unrelated_failures() { let unrelated = "ValueError: model architecture 'FooForCausalLM' is not supported"; From 5702b556088d7d0e3f558fcbfa5b13c5126e2197 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 8 Sep 2026 12:18:23 +0000 Subject: [PATCH 08/39] test(vllm): cover serve-plan low-VRAM OOM guidance on the GPU lane (EAI-8058) The serve plan's pre-launch low-VRAM warning and its vLLM `--gpu-memory-utilization` note had unit coverage but no end-to-end scenario, so the user-observable behavior on a real serve went unverified. The plan note is reached only when a *selected* GPU is below the free-VRAM bar, which the GPU lane's genuinely-free cards never trip. Add an e2e-test-hooks-gated seam in gpu_vram_usage() (ROCM_E2E_FORCE_LOW_VRAM=) that reports a single near-full synthetic device, making the warning deterministic on the non-APU vLLM lane without touching the device the engine actually launches on. Add serve-19 (@requires-gpu @requires-engine:vllm @requires-os:linux) asserting the plan warns about low VRAM and names the memory-utilization knob, with its step definitions and a unit test for the seam. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 72 ++++++++++++++++++ .../features/model_serving.feature | 16 ++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 76 +++++++++++++++++++ 3 files changed, 164 insertions(+) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 21f909177..92380bf8b 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -18612,9 +18612,43 @@ fn select_auto_gpu_index( /// multi-GPU one). Returns `None` only when neither source is readable /// (callers then fall back to service-state-only auto-selection). fn gpu_vram_usage() -> Option> { + #[cfg(feature = "e2e-test-hooks")] + if let Some(forced) = simulated_low_vram_usage() { + return Some(forced); + } gpu_vram_usage_amd_smi().or_else(gpu_vram_usage_sysfs) } +/// E2E hook: when `ROCM_E2E_FORCE_LOW_VRAM` names a GPU ordinal, report that GPU +/// as almost entirely occupied, overriding the real telemetry. +/// +/// The serve-plan low-VRAM OOM warning (and, for vLLM, its +/// `--gpu-memory-utilization` note) only fires when a *selected* GPU is below +/// [`AUTO_FREE_VRAM_FRACTION`] free. The GPU lane's real cards are comfortably +/// free, so without this seam the branch is unreachable in E2E and the note has +/// no live coverage. Synthesizing one near-full single-GPU reading makes that +/// path deterministic on the vLLM GPU lane (a non-APU host, so +/// [`vram_capacity_is_meaningful`] still holds) without touching the device the +/// engine actually launches on. Compiled only into the `e2e-test-hooks` binary; +/// production builds never see it. +#[cfg(feature = "e2e-test-hooks")] +fn simulated_low_vram_usage() -> Option> { + let index: u32 = std::env::var("ROCM_E2E_FORCE_LOW_VRAM") + .ok()? + .trim() + .parse() + .ok()?; + // A single 24 GiB device with ~0.5 GiB free is well below the 0.90 free + // threshold, so `gpu_low_memory_warning` fires; one GPU keeps + // `vram_capacity_is_meaningful` honest on the non-APU vLLM lane. + let total_mb = 24 * 1024; + Some(vec![GpuVramUsage { + index, + used_mb: total_mb - 512, + total_mb, + }]) +} + /// Per-GPU VRAM occupancy via `amd-smi metric --json`. Returns `None` when /// amd-smi is unavailable or its output cannot be parsed. fn gpu_vram_usage_amd_smi() -> Option> { @@ -26353,6 +26387,44 @@ install therock"; } } + /// The `ROCM_E2E_FORCE_LOW_VRAM` hook must synthesize a reading that actually + /// trips the serve-plan low-VRAM warning on a non-APU (vLLM-lane) host, and + /// stay inert when the var is unset. This is what the `@requires-gpu` + /// `serve-vllm-low-vram-oom-guidance` scenario relies on to fire the note + /// deterministically on cards that are really free. + #[cfg(feature = "e2e-test-hooks")] + #[test] + fn forced_low_vram_hook_trips_the_serve_plan_warning() { + let mut env = ScopedTestEnv::new(); + + // Unset: the hook contributes nothing and real telemetry is consulted. + env.clear("ROCM_E2E_FORCE_LOW_VRAM"); + assert!( + simulated_low_vram_usage().is_none(), + "no override without the env var" + ); + + // Set to ordinal 0: a single near-full device that the serve-plan wrapper + // reports on a discrete (non-APU) host but that stays honest there. + env.set("ROCM_E2E_FORCE_LOW_VRAM", "0"); + let forced = simulated_low_vram_usage().expect("override present when the var is set"); + assert_eq!( + forced.len(), + 1, + "single synthetic GPU keeps the APU guard honest" + ); + assert_eq!(forced[0].index, 0); + assert!( + forced[0] + .free_fraction() + .is_some_and(|f| f < AUTO_FREE_VRAM_FRACTION), + "the synthetic reading must be below the free-VRAM bar so the warning fires" + ); + let warning = serve_gpu_low_memory_warning(&[0], Some(&forced), Some(&host_gpu("gfx1100"))) + .expect("a near-full discrete card warrants the serve-plan warning"); + assert!(warning.contains("GPU 0")); + } + #[test] fn driver_plan_ubuntu_2404_uses_official_dkms_commands() { let _env = ScopedTestEnv::with_amd_overrides_cleared(); diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 84563b3ec..e33ca52b3 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -187,3 +187,19 @@ Feature: Model serving When the user serves a model with Lemonade Then serving stops after one automatic retry And the user is told how to reinstall Lemonade and retry serving + + # Pre-launch OOM guidance (EAI-8058). When the GPU a serve is pinned to is + # nearly out of VRAM, the plan must warn before launch and — for vLLM — name the + # `--gpu-memory-utilization` knob that avoids the OOM, closing the loop with the + # `diagnose`/`fix` catalog. The GPU lane's real cards are comfortably free, so the + # near-full reading is injected via `ROCM_E2E_FORCE_LOW_VRAM` in debug/test builds: + # a single synthetic device on this non-APU host keeps the warning honest, while + # the engine still launches against the real, free device. Runs on the vLLM GPU + # lane; the mock lane has no GPU to pin, and an APU's shared-memory carveout would + # legitimately withhold the warning. + @id:serve-vllm-low-vram-oom-guidance @requires-gpu @requires-engine:vllm @requires-os:linux + Scenario: serve-19 - A vLLM serve plan on a nearly-full GPU points at the memory knob + Given the selected GPU is reported nearly out of VRAM + When the user previews a vLLM serve plan pinned to that GPU + Then the serve plan warns the GPU is low on VRAM + And the serve plan explains how to lower vLLM's memory reservation diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9db8e959e..d03af837d 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -820,6 +820,82 @@ async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await; } +/// Arm the debug/test VRAM hook so the *selected* GPU is reported nearly full. +/// The GPU lane's real cards are comfortably free, so the serve-plan low-VRAM +/// OOM warning is otherwise unreachable in E2E; this makes it deterministic +/// without touching the device the engine actually launches on. Consumed by the +/// following serve via `run_rocm_with_scenario_env`. +#[given("the selected GPU is reported nearly out of VRAM")] +async fn selected_gpu_nearly_out_of_vram(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_E2E_FORCE_LOW_VRAM", "0".into())); +} + +/// Preview a vLLM serve plan pinned to the nearly-full GPU. `--managed` +/// backgrounds the supervisor (teardown stops it), so this returns once the plan +/// has printed and the service has launched — the plan text is what the scenario +/// inspects. The engine still launches against the real, free device. +#[when("the user previews a vLLM serve plan pinned to that GPU")] +async fn user_previews_vllm_low_vram_serve_plan(world: &mut E2eWorld) { + ensure_serve_port_free().await; + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "serve", + "Qwen/Qwen2.5-0.5B-Instruct", + "--engine", + "vllm", + "--gpu", + "0", + "--managed", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// The serve must have actually launched, not just printed a plausible plan: a +/// non-zero rc after a good plan-print would otherwise go undetected since this +/// scenario only inspects the plan lines (mirrors `assert_vllm_default`). +#[then("the serve plan warns the GPU is low on VRAM")] +async fn assert_serve_plan_low_vram_warning(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no serve output"); + let rc = world.cli_rc.expect("no serve rc recorded"); + assert!( + rc == 0, + "{}", + e2e_cucumber::cli_failure_report( + &[ + "serve", + "", + "--engine", + "vllm", + "--gpu", + "0", + "--managed", + ], + rc, + output, + world.cli_stderr.as_deref().unwrap_or(""), + ) + ); + assert!( + output.contains("serving may fail on VRAM"), + "expected a low-VRAM serve-plan warning, got:\n{output}" + ); +} + +#[then("the serve plan explains how to lower vLLM's memory reservation")] +async fn assert_serve_plan_names_memory_knob(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no serve output"); + assert!( + output.contains("--gpu-memory-utilization"), + "expected the vLLM memory-utilization note in the serve plan, got:\n{output}" + ); +} + /// Serve under the GPU-required default (no `--device`), pinning the engine to the /// host's effective serve engine so the serve reaches GPU enforcement rather than /// tripping on engine selection. From 8fcbcfa21df003594e1d29886def693dd3ad15c5 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 9 Sep 2026 16:23:56 +0300 Subject: [PATCH 09/39] test(e2e): give the low-VRAM serve scenario its runtime precondition (EAI-8058) serve-19 (@id:serve-vllm-low-vram-oom-guidance) is the only real-serve scenario on the GPU lane that does not open with `a managed runtime is active`. It runs `rocm serve ... --managed` in a fresh isolated root, so the GPU-required device policy refuses before a plan is ever built: Error: device_policy: gpu_required; no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id/--env-id That is the single unexpected failure in the E2E tests (GPU) lane on this branch (the other five are declared xfail), so the scenario added to cover the pre-launch OOM note never actually exercised it. Add the same precondition serve-05/06/07 use; the `ROCM_E2E_FORCE_LOW_VRAM` Given stays second so the runtime setup's own `rocm` calls do not drain command_env. Also correct auto_selection_uses_vram_row_count_when_amd_smi_count_is_unknown, which attributed a three-row fixture to the DRM sysfs fallback. That fallback withholds telemetry once a second AMD card is found and so emits at most one row; a multi-row `detected == None` state is amd-smi `metric` answering while `list` does not. Relabel it and add the single-row case the sysfs fallback can actually reach. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 22 ++++++++++++++----- .../features/model_serving.feature | 8 +++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 92380bf8b..397749cf1 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -25877,10 +25877,14 @@ install therock"; #[test] fn auto_selection_uses_vram_row_count_when_amd_smi_count_is_unknown() { - // A shared node without amd-smi: `detect_gpu_count()` is `None`, but the - // DRM sysfs fallback still populated per-GPU VRAM. Auto-selection must - // derive the device count from those rows and skip the busy GPU 0 rather - // than returning no selection (which would land the engine on GPU 0). + // `detect_gpu_count()` is `None` while VRAM telemetry is present. With + // *several* rows that is the amd-smi shape of the state -- `metric` + // answered while `list` did not -- not the DRM sysfs fallback, which + // withholds telemetry entirely once a second AMD card is found and so + // can never produce more than one row (`read_drm_vram_usage`). Either + // way auto-selection must derive the count from the rows and skip the + // busy GPU 0 rather than returning no selection (which would land the + // engine on GPU 0). let usage = [ vram(0, 182_000, 192_000), vram(1, 1_000, 192_000), @@ -25889,7 +25893,15 @@ install therock"; assert_eq!( select_auto_gpu_index(None, &[], Some(&usage)), vec![1], - "with no amd-smi count, rank the sysfs VRAM rows and pick the first idle GPU" + "with no amd-smi count, rank the reported VRAM rows and pick the first idle GPU" + ); + // The state the DRM sysfs fallback can actually reach: exactly one row. + // It is still ranked (and still selected) rather than discarded for + // being a single device. + assert_eq!( + select_auto_gpu_index(None, &[], Some(&[vram(0, 1_000, 192_000)])), + vec![0], + "the single row the sysfs fallback can emit must still be selectable" ); // Still nothing to go on when neither the count nor telemetry is present. assert_eq!( diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index e33ca52b3..15b24bf33 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -196,10 +196,14 @@ Feature: Model serving # a single synthetic device on this non-APU host keeps the warning honest, while # the engine still launches against the real, free device. Runs on the vLLM GPU # lane; the mock lane has no GPU to pin, and an APU's shared-memory carveout would - # legitimately withhold the warning. + # legitimately withhold the warning. `--managed` serves under the GPU-required + # device policy, which refuses before it ever builds a plan unless a ROCm + # runtime is active — hence the same `a managed runtime is active` precondition + # every other real-serve scenario on this lane opens with. @id:serve-vllm-low-vram-oom-guidance @requires-gpu @requires-engine:vllm @requires-os:linux Scenario: serve-19 - A vLLM serve plan on a nearly-full GPU points at the memory knob - Given the selected GPU is reported nearly out of VRAM + Given a managed runtime is active + And the selected GPU is reported nearly out of VRAM When the user previews a vLLM serve plan pinned to that GPU Then the serve plan warns the GPU is low on VRAM And the serve plan explains how to lower vLLM's memory reservation From 2b83d441471fb8fffde3998758a46d1f31d7fe28 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 10 Sep 2026 10:30:50 +0300 Subject: [PATCH 10/39] fix(diagnose): scope keyword overlap de-duplication to the vLLM OOM table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overlapping-span de-duplication added for `KEYWORDS_VLLM_OOM` was applied by `keyword_score` to all 16 keyword tables. Several older tables pair a greedy `.*` pattern with a second, genuinely independent keyword, and the greedy match span swallows it: `api-ms-win-crt-.*\.dll` runs to the last `.dll` on the line and covers an independent `msvcp140.dll`, `dkms .*failed` covers an independent `dpkg: error`, `amdhip64.*not found` covers `could not find hip`, and `404.*repo\.radeon\.com` covers `unable to locate package rocm`. Collapsing can only lower a score, so those pairs lost their second signal and dropped below MIN_SCORE_FOR_MATCH or HIGH_CONFIDENCE. On a Windows host with no PATH probe result, the symptom "api-ms-win-crt-runtime-l1-1-0.dll missing and msvcp140.dll not found" scored 35 instead of 65 and fix-15-msvc-redist disappeared from the report entirely. Keep the default "every matching entry is an independent signal" scoring for the catalog and route only `KEYWORDS_VLLM_OOM` — whose overlaps are true nestings of literal phrases rather than wildcard artifacts — through the collapsing variant. Pin the four pre-change scores plus the user-visible fix-15 outcome so the dedup cannot be re-generalized. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 128 +++++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 8 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 58f56afb4..b78541489 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -313,9 +313,11 @@ const KEYWORDS_PAGE_FAULT: KeywordTable = &[ // `.cuda.` module path) and a bare `out of memory` stay sub-threshold: any // PyTorch job emits the class name, so it must corroborate rather than // carry the verdict alone (`torch_oom_class_alone_is_not_a_vllm_match`). -// `keyword_score` de-duplicates overlapping spans, so `HIP out of memory` counts -// once (as the 50-point message), not also as the nested 25-point `out of -// memory` — one phrase yields one evidence bullet. +// This table alone is scored through `keyword_score_collapsing_overlaps`, which +// de-duplicates overlapping spans, so `HIP out of memory` counts once (as the +// 50-point message), not also as the nested 25-point `out of memory` — one +// phrase yields one evidence bullet. Every other table keeps the catalog's +// default "each entry is an independent signal" scoring. const KEYWORDS_VLLM_OOM: KeywordTable = &[ ( "hip out of memory", @@ -363,10 +365,38 @@ const KEYWORDS_VLLM_OOM: KeywordTable = &[ /// Score the strongest (top-2) keyword matches in `table` against `symptom`. /// -/// Matches whose spans overlap are collapsed to the strongest one, so a single -/// phrase cannot masquerade as two independent signals (e.g. `HIP out of memory` -/// must not also count the `out of memory` nested inside it). +/// Every entry that matches counts as an independent signal. This is the +/// behaviour every table in the catalog except [`KEYWORDS_VLLM_OOM`] is tuned +/// for -- see [`keyword_score_collapsing_overlaps`] for why that one differs and +/// why the difference must not be generalized. fn keyword_score(symptom: &str, table: KeywordTable) -> (i32, Vec) { + keyword_score_impl(symptom, table, false) +} + +/// Like [`keyword_score`], but collapses matches whose spans overlap to the +/// strongest one, so a single phrase cannot masquerade as two independent +/// signals (e.g. `HIP out of memory` must not also count the `out of memory` +/// nested inside it). +/// +/// This is **deliberately not** applied to the rest of the catalog. Several +/// older tables pair a greedy `.*` pattern with a second, genuinely independent +/// keyword, and the greedy span swallows it: `api-ms-win-crt-.*\.dll` runs to +/// the last `.dll` on the line and covers an independent `msvcp140.dll`, +/// `dkms .*failed` covers an independent `dpkg: error`, and so on. Since +/// collapsing can only lower a score, applying it there would push real +/// diagnoses below [`MIN_SCORE_FOR_MATCH`] and make them vanish +/// (`overlap_dedup_does_not_demote_other_catalog_keyword_tables` pins this). +/// [`KEYWORDS_VLLM_OOM`] is safe because its overlaps are true nestings of +/// literal phrases, not artifacts of a greedy wildcard. +fn keyword_score_collapsing_overlaps(symptom: &str, table: KeywordTable) -> (i32, Vec) { + keyword_score_impl(symptom, table, true) +} + +fn keyword_score_impl( + symptom: &str, + table: KeywordTable, + collapse_overlaps: bool, +) -> (i32, Vec) { if symptom.is_empty() { return (0, Vec::new()); } @@ -388,7 +418,7 @@ fn keyword_score(symptom: &str, table: KeywordTable) -> (i32, Vec) { // not contribute two evidence bullets (nor two weights toward the score). let mut kept: Vec<(i32, &'static str, usize, usize)> = Vec::new(); for hit in hits { - let overlaps = kept.iter().any(|k| hit.2 < k.3 && k.2 < hit.3); + let overlaps = collapse_overlaps && kept.iter().any(|k| hit.2 < k.3 && k.2 < hit.3); if !overlaps { kept.push(hit); } @@ -1424,7 +1454,7 @@ fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { // re-enabling the misattribution the anchor exists to prevent. return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); } - let (score, evidence) = keyword_score(symptom, KEYWORDS_VLLM_OOM); + let (score, evidence) = keyword_score_collapsing_overlaps(symptom, KEYWORDS_VLLM_OOM); if score <= 0 { return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); } @@ -2368,6 +2398,88 @@ mod tests { assert!(fix.summary.contains("--gpu-memory-utilization")); } + #[test] + fn overlap_dedup_does_not_demote_other_catalog_keyword_tables() { + // The vLLM-OOM table needs overlapping spans collapsed ("out of memory" + // nested inside "HIP out of memory"). Other tables must NOT get that + // treatment: several of them pair a greedy `.*` pattern with a second, + // genuinely independent keyword, and the greedy span swallows the + // independent one. Collapsing there silently loses real diagnoses, + // because a keyword score can only fall. + // + // Each case below pins the score these tables produced before the + // de-duplication was introduced. + + // `api-ms-win-crt-.*\.dll` (35) runs to the *last* `.dll` on the line, + // covering the independent `msvcp140.dll` (30). Two distinct missing + // DLLs, two signals -- 65 clears MIN_SCORE_FOR_MATCH, 35 does not. + let (score, ev) = keyword_score( + "api-ms-win-crt-runtime-l1-1-0.dll missing and msvcp140.dll not found", + KEYWORDS_MSVC_REDIST, + ); + assert_eq!( + score, 65, + "two distinct missing DLLs are two signals: {ev:?}" + ); + assert!(score >= MIN_SCORE_FOR_MATCH); + + // `dkms .*failed` (45) spans the whole line, covering `dpkg: error` (25). + let (score, ev) = keyword_score( + "dkms status: dpkg: error processing amdgpu-dkms, build failed", + KEYWORDS_DPKG_BROKEN, + ); + assert_eq!( + score, 70, + "a DKMS failure and a dpkg error are two signals: {ev:?}" + ); + assert!(score >= MIN_SCORE_FOR_MATCH); + + // `amdhip64.*not found` (50) spans past `could not find hip` (40): + // 90 is HIGH confidence, 50 is only a bare match. + let (score, ev) = keyword_score( + "amdhip64.dll: could not find hip runtime, not found", + KEYWORDS_HIP_SDK_MISSING, + ); + assert_eq!(score, 90, "both HIP-SDK signals must count: {ev:?}"); + assert!(score >= HIGH_CONFIDENCE); + + // `404.*repo\.radeon\.com` (50) runs to the *last* repo.radeon.com on + // the line, covering `unable to locate package rocm` (35). + let (score, ev) = keyword_score( + "e: failed to fetch https://repo.radeon.com/rocm/apt/jammy/release 404 not found, \ + unable to locate package rocm, retrying https://repo.radeon.com/rocm/apt", + KEYWORDS_REPO_BROKEN, + ); + assert_eq!(score, 85, "both apt-repo signals must count: {ev:?}"); + assert!(score >= HIGH_CONFIDENCE); + } + + #[test] + fn windows_missing_dll_pair_still_reaches_the_msvc_redist_fix() { + // The user-visible half of the guard above: with no PATH probe result + // (`msvc_redist_present == None`) the keyword score alone decides, so a + // demoted table makes fix-15 vanish from the report entirely. + let e = Examination { + os_family: "windows".to_owned(), + ..Examination::default() + }; + let report = diagnose( + &e, + "api-ms-win-crt-runtime-l1-1-0.dll missing and msvcp140.dll not found", + ); + let msvc = report + .matched + .iter() + .find(|d| d.id == "fix-15-msvc-redist") + .expect("the MSVC redistributable fix must still be reported"); + assert!( + msvc.score >= MIN_SCORE_FOR_MATCH, + "score was {}", + msvc.score + ); + assert!(report.has_match()); + } + #[test] fn vllm_anchor_must_be_on_the_same_line_as_the_oom_token() { // Co-occurrence anywhere is too loose: a paste that mentions vLLM on one From 724ef1ca87af7badb161f02eaf8601bf67c7178f Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 10 Sep 2026 13:33:30 +0300 Subject: [PATCH 11/39] fix(diagnose): score only the vLLM-anchored lines, not the whole symptom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same-line anchor was a boolean gate: `vllm_oom_line_is_anchored` returned true if any one line carried the anchor together with an OOM token, and `check_16_vllm_oom` then scored the *entire* symptom. Regex matching ignores line boundaries, so keyword hits on unanchored lines still counted at full weight, which is exactly the misattribution the anchor exists to prevent. A paste of vllm config: gpu_memory_utilization=0.9 llama.cpp: torch.OutOfMemoryError: CUDA out of memory passed the gate on the benign config echo, then scored `cuda out of memory` (50) plus `torch.outofmemoryerror` (45) off the llama.cpp line — 95, above HIGH_CONFIDENCE — and reported another framework's failure as a confident "vLLM ran the GPU out of memory". Replace the gate with `vllm_anchored_lines`, which returns the anchored lines joined by newlines, and score that. The joins are by `\n` and the table's patterns are literals or `.*` (which does not cross a newline), so dropping unanchored lines cannot fabricate a match across the seam. The same-line requirement now falls out of the scoring itself: an anchored line with no OOM token contributes nothing. `gpu[-_]memory[-_]utilization` stays an anchor despite also being a 20-point scoring entry: with only anchored lines scored, a bare flag echo (a config dump, or rocm-cli's own low-VRAM hint) is worth 20 — far below MIN_SCORE_FOR_MATCH — so it can surface as a weak signal but can never carry a verdict, nor lend its anchor to an OOM elsewhere in the paste. Scoring is otherwise unchanged: only `check_16_vllm_oom` is touched, so no other catalog table moves. Refs: EAI-8058 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 161 ++++++++++++++++++++++++++----- 1 file changed, 137 insertions(+), 24 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index b78541489..593c51cf0 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -300,8 +300,9 @@ const KEYWORDS_PAGE_FAULT: KeywordTable = &[ // CUDA out of memory` is the identical shape any ROCm PyTorch job emits (ROCm's // PyTorch build reports the CUDA-compat name), so this table alone cannot tell // vLLM's OOM apart from an arbitrary training script hitting the same -// allocator error. `check_16_vllm_oom` therefore requires an anchored OOM line -// (`VLLM_ANCHOR_PATTERN` on the same line as an OOM token) before scoring at all. +// allocator error. `check_16_vllm_oom` therefore scores *only* the lines +// carrying `VLLM_ANCHOR_PATTERN` (see `vllm_anchored_lines`), never the whole +// pasted log: an OOM on an unanchored line contributes nothing at all. // // Weighting splits the *allocator message* from the *exception class name*: // - The HIP allocator's own messages (`HIP out of memory`, the @@ -1396,6 +1397,16 @@ fn check_15_msvc_redist(e: &Examination, symptom: &str) -> Diagnosis { /// is deliberately NOT an anchor: it is a Megatron/DeepSpeed term (rocm-cli does /// not serve one model across GPUs), so anchoring on it would misattribute those /// frameworks' OOMs to vLLM. +/// +/// `gpu[-_]memory[-_]utilization` is also a (20-point) entry in +/// [`KEYWORDS_VLLM_OOM`], so a line that only echoes the flag -- a config dump, +/// or a paste of rocm-cli's own low-VRAM hint, which prints +/// `--gpu-memory-utilization` -- is both anchored and self-scoring. That is +/// harmless *because* only anchored lines are scored: such a line is worth 20, +/// far below [`MIN_SCORE_FOR_MATCH`], so it can surface as a weak signal in +/// `matched` but can never carry a verdict, and it can no longer lend its anchor +/// to an OOM elsewhere in the paste. It stays an anchor because the flag is +/// vLLM-specific, and because a real vLLM OOM often names it on the failing line. const VLLM_ANCHOR_PATTERN: &str = r"vllm|gpu[-_]memory[-_]utilization"; /// A canonical vLLM-OOM `--symptom` string guaranteed to clear @@ -1408,22 +1419,37 @@ const VLLM_ANCHOR_PATTERN: &str = r"vllm|gpu[-_]memory[-_]utilization"; /// it prints always reports a cause. pub const VLLM_OOM_CANONICAL_SYMPTOM: &str = "vllm: torch.OutOfMemoryError: HIP out of memory"; -/// Whether `symptom` carries the vLLM OOM anchor (`vllm` or -/// `gpu_memory_utilization`) on the *same line* as an OOM token from -/// [`KEYWORDS_VLLM_OOM`]. Co-occurrence anywhere in a pasted log is too loose -- -/// a stray `vllm` mention re-enables the very misattribution the anchor exists -/// to prevent (a `llama.cpp` OOM in a paste that also names vLLM). The engine's -/// emitted `vllm: ` satisfies this by construction. -fn vllm_oom_line_is_anchored(symptom: &str) -> bool { - let anchor = Regex::new(VLLM_ANCHOR_PATTERN); - let Ok(anchor) = anchor else { return false }; - symptom.lines().any(|line| { - let low = line.to_lowercase(); - anchor.is_match(&low) - && KEYWORDS_VLLM_OOM - .iter() - .any(|(pattern, _, _)| Regex::new(pattern).is_ok_and(|re| re.is_match(&low))) - }) +/// The lines of `symptom` that carry the vLLM OOM anchor (`vllm` or +/// `gpu_memory_utilization`), joined by newlines — the only text +/// [`check_16_vllm_oom`] is allowed to score. +/// +/// Co-occurrence anywhere in a pasted log is too loose -- a stray `vllm` mention +/// re-enables the very misattribution the anchor exists to prevent (a +/// `llama.cpp` OOM in a paste that also names vLLM). Returning the anchored +/// lines rather than a yes/no gate is what makes that true: regex matching +/// ignores line boundaries, so scoring the whole symptom behind a boolean gate +/// still counted keyword hits from *unanchored* lines at full weight, and one +/// benign `gpu_memory_utilization` config echo was enough to hand another +/// framework's OOM a HIGH_CONFIDENCE vLLM verdict +/// (`only_the_anchored_lines_are_scored_not_the_whole_paste`). +/// +/// The join is by `\n` so that no pattern can straddle two lines: the regexes +/// here are literals or use `.*`, which does not match a newline. Dropping the +/// unanchored lines therefore cannot fabricate a match across the seam. +/// +/// The same-line "anchor + OOM token" requirement falls out of this: an anchored +/// line with no OOM token contributes nothing, so the score is 0 and the checker +/// returns no diagnosis. The engine's emitted `vllm: ` keeps the +/// anchor and the error on one line by construction. +fn vllm_anchored_lines(symptom: &str) -> String { + let Ok(anchor) = Regex::new(VLLM_ANCHOR_PATTERN) else { + return String::new(); + }; + symptom + .lines() + .filter(|line| anchor.is_match(&line.to_lowercase())) + .collect::>() + .join("\n") } /// Whether `symptom` clears [`MIN_SCORE_FOR_MATCH`] for the vLLM-OOM failure @@ -1447,14 +1473,17 @@ pub fn vllm_oom_symptom_is_diagnosable(symptom: &str) -> bool { /// the user must pass the error via `--symptom`, or arrive from the serve /// failure note that points here. `e` is therefore unused. fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { - if !vllm_oom_line_is_anchored(symptom) { - // No vLLM anchor on an OOM line: don't attribute a bare framework OOM to - // vLLM at all. Requiring the anchor on the same line as the OOM token - // (not merely somewhere in the paste) keeps a stray `vllm` mention from - // re-enabling the misattribution the anchor exists to prevent. + // Score the anchored lines only, never the whole paste. The anchor is the + // *only* thing separating this failure mode from any other framework's + // PyTorch/HIP OOM, so a boolean gate over the whole symptom is not enough: + // keyword hits from unanchored lines would still count at full weight. Any + // future change here must keep the scored text restricted to anchored lines. + let anchored = vllm_anchored_lines(symptom); + if anchored.is_empty() { + // No vLLM anchor anywhere: don't attribute a bare framework OOM to vLLM. return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); } - let (score, evidence) = keyword_score_collapsing_overlaps(symptom, KEYWORDS_VLLM_OOM); + let (score, evidence) = keyword_score_collapsing_overlaps(&anchored, KEYWORDS_VLLM_OOM); if score <= 0 { return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); } @@ -2500,6 +2529,90 @@ mod tests { assert!(anchored.matched.iter().any(|d| d.id == "fix-16-vllm-oom")); } + #[test] + fn only_the_anchored_lines_are_scored_not_the_whole_paste() { + // Regression: the anchor was a boolean *gate*, after which the entire + // symptom was scored. Regex matching ignores line boundaries, so a + // benign vLLM config echo on one line unlocked full-weight scoring of + // another framework's OOM on a different line: `cuda out of memory` + // (50) + `torch.outofmemoryerror` (45) do not overlap, so both survived + // collapsing for 95 -- above HIGH_CONFIDENCE -- and the user got a + // confident "vLLM ran the GPU out of memory" verdict for a llama.cpp + // failure. Only the anchored lines may be scored. + let report = diagnose( + &linux_base(), + "vllm config: gpu_memory_utilization=0.9\n\ + llama.cpp: torch.OutOfMemoryError: CUDA out of memory", + ); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the anchored config line is still a (weak) vLLM signal"); + assert!( + oom.score < MIN_SCORE_FOR_MATCH, + "an unanchored framework's OOM must not score for vLLM, got {} from {:?}", + oom.score, + oom.evidence + ); + assert!( + oom.evidence + .iter() + .all(|e| !e.to_lowercase().contains("out of memory")), + "evidence must come from the anchored line only: {:?}", + oom.evidence + ); + + // Same shape with the anchored line carrying no OOM token at all: the + // unanchored OOM must not be borrowed to reach a verdict either. + let report = diagnose( + &linux_base(), + "starting vllm serve on gpu 0\n\ + llama.cpp: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 7.21 GiB.", + ); + assert!( + report.matched.iter().all(|d| d.id != "fix-16-vllm-oom"), + "an OOM on an unanchored line must not be attributed to vLLM: {:?}", + report.matched + ); + + // The control: once the OOM is on the anchored line, it still matches at + // full strength, so the scoping did not simply disable the checker. + let report = diagnose( + &linux_base(), + "vllm config: gpu_memory_utilization=0.9\n\ + vllm: torch.OutOfMemoryError: CUDA out of memory", + ); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("an anchored OOM line must still match"); + assert!(oom.score >= HIGH_CONFIDENCE, "score was {}", oom.score); + } + + #[test] + fn rocm_cli_s_own_low_vram_hint_is_never_a_verdict_on_its_own() { + // `gpu[-_]memory[-_]utilization` is both an anchor and a scoring entry, + // so a line that merely echoes the flag anchors itself. rocm-cli's own + // low-VRAM hint prints `--gpu-memory-utilization`, and users paste it + // back in. That must stay a weak signal, never a diagnosis: worth 20, + // far below MIN_SCORE_FOR_MATCH, with nothing else on the line to + // corroborate it. + let report = diagnose(&linux_base(), crate::VLLM_GPU_MEMORY_UTILIZATION_HINT); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the flag mention stays visible as a weak signal"); + assert!( + oom.score < MIN_SCORE_FOR_MATCH, + "a bare flag echo must not diagnose an OOM, got {} from {:?}", + oom.score, + oom.evidence + ); + } + #[test] fn canonical_vllm_oom_symptom_is_diagnosable() { // The engine's fallback symptom must always clear the threshold, so the From 2c3a84e05935036d1e488f6fdacee3eda2bf2e52 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 11 Sep 2026 10:32:06 +0300 Subject: [PATCH 12/39] docs(vllm): align the OOM reservation example with the fix recipe The catalog recipe and the --gpu-memory-utilization parse error both use 0.5; the docs said 0.1 for the same shared-GPU case. A 0.1 budget is below the weights of most models people serve, so it trades one startup failure for another. Signed-off-by: Roman Sirokov --- docs/vllm.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/vllm.md b/docs/vllm.md index ecdb858ec..a741fdcf3 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -170,16 +170,16 @@ a tiny model. rocm-cli helps in three ways: points at `--gpu-memory-utilization` as the fix. - **OOM failures hint the workaround.** When a startup failure log shows an out-of-memory error, the failure message suggests retrying with a smaller - reservation, e.g. `--gpu-memory-utilization 0.1`, or targeting a less-busy GPU + reservation, e.g. `--gpu-memory-utilization 0.5`, or targeting a less-busy GPU with `--gpu `, and points at `rocm diagnose --symptom ''` for the full conditional remediation (busy GPU vs. a model that does not fit). Explicitly, the workaround for an OOM on a shared card is: ```bash -rocm serve --engine vllm --gpu-memory-utilization 0.1 +rocm serve --engine vllm --gpu-memory-utilization 0.5 # optionally target a specific, less-busy GPU by index -rocm serve --engine vllm --gpu 1 --gpu-memory-utilization 0.1 +rocm serve --engine vllm --gpu 1 --gpu-memory-utilization 0.5 # for the full busy-GPU-vs-model-too-large breakdown, pass the error to diagnose rocm diagnose --symptom 'vllm: torch.OutOfMemoryError: HIP out of memory' ``` From 38f3d7b5acc1ccc18c02cec7ac028b5e484d5c2b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 11 Sep 2026 11:09:31 +0300 Subject: [PATCH 13/39] test(e2e): make the low-VRAM serve scenario assert the new guidance (EAI-8058) The scenario's warning step matched "serving may fail on VRAM", which predates this change, and its sibling matched only the bare flag name --- which a recipe echo of a user-supplied --gpu-memory-utilization would also satisfy. Neither fragment is specific to the pre-launch hint this scenario exists to prove. Require the warning to name the pinned GPU and the note to carry vLLM's total-VRAM reservation rationale, and pin both literals in the always-run unit test so a rewording fails on every lane rather than only on the GPU lane that can observe the scenario. Also record on serve-16 why the masked, non-contiguous usable-set case behind the --gpu membership rule has no scenario of its own: it is reachable only where amd-smi is absent, so a scenario would assert against an injected seam rather than the product. It stays unit-covered. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 18 +++++++++++++++ .../features/model_serving.feature | 11 ++++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 22 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index cd7a6d35d..c061dbfe1 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -26179,6 +26179,24 @@ install therock"; "vLLM low-VRAM notes must hint the utilization workaround: {notes:?}" ); + // The literal fragments the GPU-lane scenario + // `@id:serve-vllm-low-vram-oom-guidance` matches on. That scenario runs + // only where a real card exists, so pin the wording here too: a rename + // then fails on every lane rather than silently on the one lane that can + // observe it. `GPU 0 has only` is what makes the warning provably about + // the *pinned* device; `vLLM reserves ~90%` is what distinguishes the + // pre-launch hint from any other line that merely names the flag. + assert!( + notes.iter().any(|entry| entry.contains("GPU 0 has only")), + "the warning must name the selected GPU: {notes:?}" + ); + assert!( + notes + .iter() + .any(|entry| entry.contains("vLLM reserves ~90%")), + "the hint must explain vLLM's total-VRAM reservation: {notes:?}" + ); + // The same busy card on a non-vLLM engine keeps the warning but omits the // vLLM-only knob, which that engine cannot honor. let lemonade = collect_serve_notes( diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 15b24bf33..2cc2e1cd4 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -160,6 +160,17 @@ Feature: Model serving # fallback). Runs on GPU hardware: on a no-GPU host the GPU-required pre-flight # refuses ("no usable AMD GPU") before the index is ever validated, so the # index-specific rejection can only be observed where a real device is present. + # + # This is the live coverage for `--gpu` validation under either authority: the + # amd-smi device count, or — when amd-smi cannot count — membership in the + # KFD/DRM usable set (EAI-8058). Which one answers depends on the lane, so the + # step accepts either rejection wording. The *masked* sub-case that motivated + # the membership rule (a visibility mask leaving a non-contiguous usable set, + # e.g. `HIP_VISIBLE_DEVICES=2` on a 4-GPU host, where `--gpu 2` must pass and + # `--gpu 0` must be refused) has no scenario of its own: it is reachable only + # where amd-smi is *absent*, and faking that absence would assert against the + # seam rather than the product. It is unit-covered by + # `validate_pinned_gpu_index_falls_back_to_usable_set_membership`. @id:serve-absent-gpu-index-rejected @requires-gpu @requires-os:linux Scenario: serve-16 - Serving pinned to a GPU that does not exist is refused When the user serves a model pinned to a GPU index that does not exist diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d03af837d..a8edb1fc6 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -859,6 +859,13 @@ async fn user_previews_vllm_low_vram_serve_plan(world: &mut E2eWorld) { /// The serve must have actually launched, not just printed a plausible plan: a /// non-zero rc after a good plan-print would otherwise go undetected since this /// scenario only inspects the plan lines (mirrors `assert_vllm_default`). +/// +/// This step states the scenario's *premise*, not its verdict: the low-VRAM +/// warning itself is long-standing behaviour, so the warning text alone proves +/// nothing about the vLLM guidance this scenario exists for — that is +/// `assert_serve_plan_names_memory_knob`'s job. What it does pin is that the +/// warning names the GPU the serve was *pinned to* (`--gpu 0`), so a warning +/// raised for some other device cannot stand in for the premise. #[then("the serve plan warns the GPU is low on VRAM")] async fn assert_serve_plan_low_vram_warning(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no serve output"); @@ -885,8 +892,19 @@ async fn assert_serve_plan_low_vram_warning(world: &mut E2eWorld) { output.contains("serving may fail on VRAM"), "expected a low-VRAM serve-plan warning, got:\n{output}" ); + assert!( + output.contains("GPU 0 has only"), + "the low-VRAM warning must be about the pinned GPU 0, got:\n{output}" + ); } +/// The scenario's discriminating assertion — the behaviour added for EAI-8058. +/// +/// Matching the flag name alone would be too weak to be that: `rocm serve` echoes +/// engine recipe flags in the same plan, so a `--gpu-memory-utilization` the user +/// passed themselves would satisfy it. Require the note's own explanation of *why* +/// a busy GPU OOMs (vLLM's fixed total-VRAM reservation) alongside the flag, so +/// only the pre-launch hint can satisfy this step. #[then("the serve plan explains how to lower vLLM's memory reservation")] async fn assert_serve_plan_names_memory_knob(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no serve output"); @@ -894,6 +912,10 @@ async fn assert_serve_plan_names_memory_knob(world: &mut E2eWorld) { output.contains("--gpu-memory-utilization"), "expected the vLLM memory-utilization note in the serve plan, got:\n{output}" ); + assert!( + output.contains("vLLM reserves ~90%"), + "the note must explain vLLM's total-VRAM reservation, not merely name the flag, got:\n{output}" + ); } /// Serve under the GPU-required default (no `--device`), pinning the engine to the From 2e692efeed77da49ed92183555b96eaca97ad8e4 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 11 Sep 2026 13:42:12 +0300 Subject: [PATCH 14/39] test(diagnose): retire the WSL out-of-scope routing assertions The WSL2 catalog (#340) makes WSL a covered platform, so `diagnose` no longer emits an out-of-scope routing note there. Two tests written against the old behaviour were asserting that note, and one of them also assumed a default WSL examination produces no match -- it now legitimately does, since a host with no /dev/dxg really is a `fix-wsl-1-gpu-not-exposed` finding. Both now build on `wsl_base()`, the healthy-WSL fixture, so the vLLM OOM signal is the only thing under test. The load-bearing half is unchanged and still asserted: a score-25 `out of memory` mention stays visible in `matched` as a WEAK row and does not promote itself into a match. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 37 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 1e2c2a91a..de401c9e8 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -3471,20 +3471,27 @@ mod tests { } #[test] - fn wsl_sub_threshold_vllm_signal_is_preserved_but_still_routed_out_of_scope() { + fn wsl_sub_threshold_vllm_signal_is_preserved_in_matched() { // A weak (sub-threshold) vLLM OOM signal on WSL must still surface in - // `matched` per the `DiagnoseReport::matched` contract -- but it must - // NOT suppress the out-of-scope routing note. Suppression is gated on - // `has_match` (score >= MIN_SCORE_FOR_MATCH), not on `matched` being - // non-empty, so a score-25 mention no longer buries the WSL routing note - // for a user whose real problem is a missing WSL driver. - let mut e = linux_base(); - e.is_wsl = true; - let report = diagnose(&e, "vllm: out of memory"); + // `matched` per the `DiagnoseReport::matched` contract: `run_all_checks` + // drops only zero-score results, so a score-25 mention is reported as a + // WEAK row rather than silently discarded. + // + // This used to also assert that the weak hit did not bury a WSL + // out-of-scope routing note. There is no such note any more -- WSL2 is a + // covered platform with its own catalog entries -- so what is left to pin + // is the half that still has teeth: the entry survives in `matched`, and + // it does not promote itself into a match. + // + // `wsl_base()` (a healthy WSL GPU stack) rather than a default + // examination: a default one has no /dev/dxg, so fix-wsl-1 would fire at + // 55 and the report would match for reasons that have nothing to do with + // the signal under test. + let report = diagnose(&wsl_base(), "vllm: out of memory"); assert!(!report.has_match()); assert!( - report.out_of_scope.is_some(), - "a sub-threshold hit must not suppress the WSL routing note" + report.out_of_scope.is_none(), + "WSL2 is a covered platform, not an out-of-scope one" ); let oom = report .matched @@ -3497,10 +3504,10 @@ mod tests { #[test] fn a_genuine_vllm_oom_match_on_wsl_is_surfaced_not_routed_out_of_scope() { // The other side of the has_match gate: a real, at-or-above-threshold - // vLLM OOM on WSL surfaces the diagnosis instead of the routing note. - let mut e = linux_base(); - e.is_wsl = true; - let report = diagnose(&e, VLLM_OOM_CANONICAL_SYMPTOM); + // vLLM OOM on WSL is surfaced, and ranks top on a host whose own WSL GPU + // stack is healthy -- so the verdict is the OOM, not an incidental WSL + // finding. + let report = diagnose(&wsl_base(), VLLM_OOM_CANONICAL_SYMPTOM); assert!(report.has_match()); assert!(report.out_of_scope.is_none()); assert_eq!(report.matched[0].id, "fix-16-vllm-oom"); From 13bd32c0c369976b2739cb27b40734eee5618841 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 11 Sep 2026 16:13:49 +0300 Subject: [PATCH 15/39] fix(fix): let fix-16-vllm-oom run on WSL2, where its checker answers `check_16_vllm_oom` is registered for `["linux", "wsl"]` -- deliberately, per the comment at diagnose.rs -- but the matching recipe was `LINUX_ONLY`. `rocm fix` re-gates a fix-id on the *recipe's* platform list against the running OS, where WSL2 is its own family, so on WSL2 `rocm diagnose` printed `apply with: rocm fix fix-16-vllm-oom` and that command then printed the plan and exited 3 with "This fix only applies on: linux. Running OS is: wsl." Nothing about the remediation is bare-metal: it is vLLM CLI flags. WSL2 is the supported way to serve from a Windows host and no `fix-wsl-*` entry covers the failure mode, so the platform had a diagnosis and no working fix path. The class, not just the instance: the two platform lists are written in separate modules and each looks right alone. Add `every_checker_platform_is_covered_by_its_recipe`, which walks `CHECKERS` and requires every family a checker answers on to be one the recipe will act on. Containment rather than equality -- a recipe may legitimately apply more widely than its checker fires, since `rocm fix ` is reachable without a diagnosis. It found no other mismatch across the remaining 23 entries. The loop is user-observable, so diagnose-20 gains a step that runs the fix-id the report named and requires the success rc (the wrong-OS refusal prints the recipe first, so output alone cannot tell the outcomes apart). It is `@requires-os:linux`, which WSL2 matches, so the `e2e-wsl` self-hosted lane is the one that can observe the difference; every Linux lane runs the step. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 32 +++++++++++++++++++ crates/rocm-core/src/fix.rs | 23 ++++++++++++- tests/e2e-cucumber/features/diagnose.feature | 7 ++++ .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 31 ++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index de401c9e8..d6e2f7fbd 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -3771,6 +3771,38 @@ mod tests { assert!(!vllm_oom_symptom_is_diagnosable("vllm: out of memory")); } + #[test] + fn every_checker_platform_is_covered_by_its_recipe() { + // `render_report_text` ends a matched diagnosis with `apply with: rocm + // fix {id}`, and `fix::apply` then gates that id on the recipe's + // `applies_on` against the *running* OS. So any platform family a + // checker is registered for but its recipe omits is a platform where the + // tool names a command and then refuses to run it -- which is exactly + // what `fix-16-vllm-oom` did on WSL2, where the checker opts into `wsl` + // and the recipe was LINUX_ONLY. + // + // Only one direction is an error. A recipe may legitimately apply more + // widely than its checker answers (a user can reach `rocm fix ` + // directly, without a diagnosis), so this asserts containment rather + // than equality. + for (check, families) in CHECKERS { + let id = check(&Examination::default(), "").id; + let applies_on = crate::fix::recipe_applies_on(&id).unwrap_or_else(|| { + panic!( + "checker id `{id}` has no recipe in the fix catalog, so the \ + `apply with: rocm fix {id}` line diagnose prints is dead" + ) + }); + for family in *families { + assert!( + applies_on.contains(family), + "`{id}` is diagnosed on `{family}` but its recipe only applies on \ + {applies_on:?}; `rocm fix {id}` would print the plan and then exit 3 there" + ); + } + } + } + #[test] fn diagnosis_remediation_matches_the_fix_catalog_for_shared_fix_ids() { // fix-15 and fix-16 live in two places: the `rocm fix` catalog diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 46ea79586..a5766da34 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -74,6 +74,12 @@ struct FixRecipe { /// move are the ones about wheels, environment variables and PATH. const LINUX_WINDOWS_AND_WSL: &[&str] = &["linux", "windows", "wsl"]; const LINUX_AND_WINDOWS: &[&str] = &["linux", "windows"]; +/// For entries whose fault is not tied to bare-metal kernel plumbing. +/// +/// `fix-16-vllm-oom` is the case: its remediation is vLLM CLI flags, and its +/// checker already opts into `wsl` for that reason, so a Linux-only recipe made +/// `rocm diagnose` name a command `rocm fix` then refused to run. +const LINUX_AND_WSL: &[&str] = &["linux", "wsl"]; const LINUX_ONLY: &[&str] = &["linux"]; const WINDOWS_ONLY: &[&str] = &["windows"]; const WSL_ONLY: &[&str] = &["wsl"]; @@ -410,7 +416,12 @@ const RECIPES: &[FixRecipe] = &[ "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.", "This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).", ], - applies_on: LINUX_ONLY, + // Matches `check_16_vllm_oom`'s `["linux", "wsl"]` registration: WSL2 is + // the supported way to serve from a Windows host and the failure mode is + // just as real there, so the fix has to be runnable where the diagnosis + // is reachable. Pinned by + // `every_checker_platform_is_covered_by_its_recipe`. + applies_on: LINUX_AND_WSL, runner: None, }, // The number is a stable handle, not a position: `fix-16` is held by the @@ -715,6 +726,16 @@ pub(crate) fn recipe_verify_notes_and_run_commands( }) } +/// Test-only view of the platform families a recipe is runnable on. +/// +/// `None` when no recipe carries `fix_id`. Lets the diagnosis catalog assert +/// that every platform a checker answers on is a platform `rocm fix` will +/// actually act on, which `recipe_verify_notes_and_run_commands` cannot see. +#[cfg(test)] +pub(crate) fn recipe_applies_on(fix_id: &str) -> Option<&'static [&'static str]> { + find_recipe(fix_id).map(|recipe| recipe.applies_on) +} + /// The platform family a recipe's `applies_on` is matched against. /// /// WSL2 is its own family rather than `linux`, mirroring `diagnose`. That is what diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 2f296297c..fd9bb8636 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -276,3 +276,10 @@ Feature: Diagnosing failures and listing fixes When the user asks the CLI to diagnose that symptom in machine-readable form Then the diagnosis identifies the vLLM startup OOM And the OOM remedy distinguishes a busy GPU from a model that does not fit + # A report that names `rocm fix ` and a `rocm fix` that then refuses that + # id leaves the user worse off than no fix path at all. The two halves are + # selected by separate platform lists -- the checker's, and the recipe's + # against the RUNNING os, where WSL2 is its own family -- so they can + # disagree while each looks right alone. Every Linux lane runs this step; the + # WSL one is where the two lists can differ. + And the CLI can act on the fix the diagnosis named diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 206e41d97..99dce26ef 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -536,6 +536,37 @@ async fn assert_diagnosis_identifies_vllm_oom(world: &mut E2eWorld) { ); } +/// The other half of the loop: the `apply with: rocm fix ` the report prints +/// has to name a command this host will actually act on. +/// +/// `rocm diagnose` selects a checker by the platform families the checker +/// registers; `rocm fix` then re-gates the same id on the *recipe's* list +/// against the running OS. WSL2 is its own family on both sides, so a recipe +/// that omits it printed the plan and then exited 3 on a platform its checker +/// deliberately answers for. This asserts the rc, not merely the output: the +/// wrong-OS refusal prints the recipe first, so the text alone cannot tell the +/// two outcomes apart. +#[then("the CLI can act on the fix the diagnosis named")] +async fn assert_named_fix_is_runnable_here(world: &mut E2eWorld) { + let (report, output) = parsed_diagnosis(world); + let fix_id = find_vllm_oom(&report, &output) + .pointer("/fix/fix_id") + .and_then(serde_json::Value::as_str) + .unwrap_or_else(|| panic!("the matched cause must name a fix id:\n{output}")) + .to_owned(); + let (fix_output, fix_stderr, rc) = crate::run_rocm(world, &["fix", &fix_id]); + assert_eq!( + rc, + 0, + "{}", + e2e_cucumber::cli_failure_report(&["fix", &fix_id], rc, &fix_output, &fix_stderr) + ); + assert!( + fix_output.to_lowercase().contains("print-only"), + "{fix_id} is advisory, so it must say it only printed a plan:\n{fix_output}" + ); +} + #[then("the OOM remedy distinguishes a busy GPU from a model that does not fit")] async fn assert_oom_remedy_is_conditional(world: &mut E2eWorld) { let (report, output) = parsed_diagnosis(world); From 78cdcef77af8e39f9cbcb20fa9182ce9ebacfe57 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 11 Sep 2026 16:14:08 +0300 Subject: [PATCH 16/39] fix(core): the shared OOM hint recommended 0.1 while every command said 0.5 `VLLM_GPU_MEMORY_UTILIZATION_HINT` still carried "(e.g. 0.1 for a small model)". It is printed verbatim by `rocm serve`'s low-VRAM note and the vLLM engine's post-failure OOM hint, and interpolated into the `fix-16-vllm-oom` diagnosis summary -- so one `rocm diagnose` printed "e.g. 0.1 for a small model" directly above `rocm serve --gpu-memory-utilization 0.5`. 0.5 is the settled value; every command surface already used it (the recipe, the diagnosis Fix, docs/vllm.md). 2c3a84e moved the docs and left the const behind, and that commit's own message says why 0.1 is wrong: a 0.1 budget is below the weights of most models people serve, so it trades one startup failure for another. The harmful advice was what the product actually printed. Pin the two together with `the_utilization_hint_example_matches_the_recipe_command`, which compares the hint's worked value against the value the recipe commands. Neither existing guard covers this: the diagnose/fix cross-check exempts the summary as prose framing, and a line-based grep misses the const because the flag and the value sit on different continuation lines -- which is exactly how it was missed the first time. Also correct the const's own doc claim, which named two sharing surfaces when there are three, and record that the catalog rationale is deliberately separate prose whose worked value is nonetheless pinned to this one. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/fix.rs | 37 +++++++++++++++++++++++++++++++++++++ crates/rocm-core/src/lib.rs | 17 +++++++++++++---- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index a5766da34..76c5afe48 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -1520,6 +1520,43 @@ mod tests { find_recipe(fix_id).is_some_and(|r| r.applies_on.contains(¤t_os())) } + #[test] + fn the_utilization_hint_example_matches_the_recipe_command() { + // The shared hint carries a worked value, and `fix-16-vllm-oom` hands + // the user a command carrying another. They drifted once already: the + // docs and the recipe moved to 0.5 while the const kept recommending + // 0.1, so one `rocm diagnose` printed "e.g. 0.1 for a small model" + // directly above `rocm serve --gpu-memory-utilization 0.5`. + // Nothing caught it -- the cross-check in `diagnose` exempts the summary + // as prose framing, and a line-based grep misses the const because the + // flag and the value sit on different continuation lines. + const FLAG: &str = "--gpu-memory-utilization"; + let hint = crate::VLLM_GPU_MEMORY_UTILIZATION_HINT; + let example = hint + .split_once("e.g. ") + .and_then(|(_, rest)| rest.split_whitespace().next()) + .unwrap_or_else(|| { + panic!("the shared hint no longer carries a worked `e.g. `:\n{hint}") + }); + let recipe = find_recipe("fix-16-vllm-oom").expect("fix-16-vllm-oom is in the catalog"); + let commanded = recipe + .commands + .iter() + .find_map(|line| line.split_once(FLAG)) + .and_then(|(_, rest)| rest.split_whitespace().next()) + .unwrap_or_else(|| { + panic!( + "fix-16-vllm-oom no longer commands `{FLAG}`:\n{:#?}", + recipe.commands + ) + }); + assert_eq!( + example, commanded, + "the shared hint recommends `{FLAG} {example}` while `rocm fix fix-16-vllm-oom` \ + commands `{FLAG} {commanded}`; both can print in a single `rocm diagnose` report" + ); + } + #[test] fn auto_applicable_recipes_have_a_runner() { for r in RECIPES { diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index d672f7189..2ed53030f 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -7607,15 +7607,24 @@ pub fn resolve_amd_smi_binary() -> OsString { /// The `--gpu-memory-utilization` workaround for a shared/busy GPU. /// -/// Shared by both the `rocm` CLI (pre-launch low-VRAM note) and the vLLM engine -/// adapter (post-failure OOM hint) so the two surfaces never drift into -/// different wording for the same fix. vLLM reserves a fixed fraction of each +/// Shared by the `rocm` CLI (pre-launch low-VRAM note), the vLLM engine adapter +/// (post-failure OOM hint) and the `fix-16-vllm-oom` diagnosis summary, so those +/// surfaces never drift into different wording for the same fix. The +/// `rocm fix fix-16-vllm-oom` catalog rationale is deliberately NOT this text — +/// it frames the same fault for a different reader — but its worked value is +/// pinned to this one (see below). vLLM reserves a fixed fraction of each /// GPU's *total* VRAM by default (~0.9), independent of the model size or how /// much is currently free, so on a shared or busy card that reservation /// collides with memory already in use and the engine OOMs even a tiny model. +/// +/// The worked example must stay the value the `fix-16-vllm-oom` recipe and the +/// docs hand the user (`0.5`). A smaller budget such as `0.1` sits below the +/// weights of most models people actually serve, so it trades one startup +/// failure for another. Pinned by +/// `the_utilization_hint_example_matches_the_recipe_command`. pub const VLLM_GPU_MEMORY_UTILIZATION_HINT: &str = "vLLM reserves ~90% of the GPU's total VRAM by default; on a shared or busy GPU this can \ collide with memory already in use. Lower the reservation with `--gpu-memory-utilization \ - <0-1>` (e.g. 0.1 for a small model), or target a less-busy GPU with `--gpu `."; + <0-1>` (e.g. 0.5 for a small model), or target a less-busy GPU with `--gpu `."; /// Locate `amd-smi` inside the bin directories of the newest managed ROCm SDK /// runtime recorded in the registry. The binary ships with the TheRock wheel From 1e23717d0ad4a966f79f929a334d7bd32350d235 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 14 Sep 2026 16:58:29 +0300 Subject: [PATCH 17/39] fix(diagnose): drop the duplicate LINUX_AND_WSL const from the merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both branches added an identical const — this branch for the fix-16 WSL gate, main for #386 — at different offsets, so git merged both copies without a conflict marker and the lib test failed to compile (E0428). Signed-off-by: Roman Sirokov --- crates/rocm-core/src/fix.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 2f39476e7..5a86ba5f4 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -83,9 +83,6 @@ const LINUX_AND_WSL: &[&str] = &["linux", "wsl"]; const LINUX_ONLY: &[&str] = &["linux"]; const WINDOWS_ONLY: &[&str] = &["windows"]; const WSL_ONLY: &[&str] = &["wsl"]; -/// Both Linux families. For a problem that is neither about the `amdgpu` module -/// nor about the Windows host driver, and so is real on either. -const LINUX_AND_WSL: &[&str] = &["linux", "wsl"]; /// The recipe registry. Mirrors the diagnosis catalog; only the four small, /// safe fixes carry a `runner` and are auto-applicable. From 0b5d66a1b3ce6039c76c4bc8f67e8133c217bcd3 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 14 Sep 2026 19:06:07 +0300 Subject: [PATCH 18/39] fix(vllm): keep untrusted log text out of the suggested diagnose command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a failed vLLM serve the post-failure hint printed the user's own failing log line inside a single-quoted `rocm diagnose --symptom '...'` command it invites them to paste into a shell. The line is the vLLM subprocess's own startup-log tail and nothing escaped it, so a routine apostrophe in Python error text ("can't allocate", "model 'foo'") closed the quote: at best the command the feature exists to provide was syntactically broken, at worst text the user does not control landed outside the quotes. The neighbouring human-readable echo passed the same bytes through verbatim, ANSI and all. A line that cannot be rendered as one intact single-quoted word is now treated as unusable and takes the fallback to VLLM_OOM_CANONICAL_SYMPTOM that already existed for exactly that case, rather than being escaped: escaping preserves the bytes but yields a command a reader cannot check by eye, and a wrong escape is runnable and misleading instead of obviously broken. The echoed line is stripped of ANSI sequences and control bytes. Also restores the `out_of_scope` doc comment to what `diagnose` does. It had been changed to claim `matched` may hold sub-threshold entries alongside it and that the field is set when nothing clears MIN_SCORE_FOR_MATCH; neither holds — `matched` is cleared whenever the field is set, and the field is decided purely by platform coverage. The WSL2 example can no longer occur now that WSL2 is a covered platform. That is the documented JSON contract, so a consumer trusting it waits for rows that can never arrive. The relationship is now asserted. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 50 +++++++++-- docs/vllm.md | 4 + engines/vllm/src/lib.rs | 143 +++++++++++++++++++++++++++++-- 3 files changed, 182 insertions(+), 15 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 00381276c..b0217843c 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -82,12 +82,14 @@ pub struct DiagnoseReport { pub min_score_for_match: i32, pub high_confidence_threshold: i32, pub route_when_no_match: Route, - /// Set when the host is out of scope for this catalog (e.g. WSL2). Present - /// whenever no checker cleared [`MIN_SCORE_FOR_MATCH`] on such a host; the - /// bare-metal Linux catalog is deliberately not run, so this carries the - /// routing note instead. `matched` may still hold sub-threshold entries - /// (e.g. a weak `out of memory` mention) alongside it — read - /// [`DiagnoseReport::has_match`] to tell a real cause from a weak signal. + /// Set when the catalog has no entries at all for the running platform, and + /// carrying the routing note that says so. + /// + /// Decided purely by platform coverage ([`catalog_covers`]) — never by a + /// score. When this is `Some`, no checker ran, so `matched` is empty and + /// `has_match` is false; a consumer must not wait for sub-threshold rows + /// alongside it. Pinned by + /// `an_out_of_scope_report_never_carries_matched_entries`. #[serde(default)] pub out_of_scope: Option, } @@ -3544,6 +3546,42 @@ mod tests { assert!(!report.has_match()); } + #[test] + fn an_out_of_scope_report_never_carries_matched_entries() { + // The serialized JSON contract on `out_of_scope`: when it is set the + // catalog did not run, so `matched` cannot hold anything -- not even a + // weak sub-threshold signal. One direction only: a covered platform with + // nothing wrong also has an empty `matched`, with `out_of_scope` unset. + // Nothing else in the tree pins the pair, which is how the field's own + // doc comment came to claim the opposite. Two independent mechanisms + // uphold it today -- `diagnose` clears `matched` when `out_of_scope` is + // set, and `run_all_checks` filters on the same platform family + // `catalog_covers` consults -- so this fails once both are gone, which + // is exactly the "a checker somehow ran on an uncovered host" shape the + // JSON contract must not develop. + let mut uncovered = linux_base(); + uncovered.os_family = "other".to_owned(); + let mut windows = linux_base(); + windows.os_family = "windows".to_owned(); + for e in [linux_base(), wsl_base(), windows, uncovered] { + // A symptom that scores for at least one entry where the catalog + // applies, so "matched is empty" is a real consequence of the + // platform being uncovered rather than of a silent symptom. + let report = diagnose(&e, VLLM_OOM_CANONICAL_SYMPTOM); + assert!( + report.out_of_scope.is_none() || report.matched.is_empty(), + "out_of_scope set with matched entries on {}: {:?}", + e.os_family, + report.matched.iter().map(|d| &d.id).collect::>() + ); + assert!( + report.out_of_scope.is_none() || !report.has_match(), + "out_of_scope set alongside has_match on {}", + e.os_family + ); + } + } + #[test] fn non_wsl_still_diagnoses_normally() { let mut e = linux_base(); diff --git a/docs/vllm.md b/docs/vllm.md index a741fdcf3..934a6d391 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -173,6 +173,10 @@ a tiny model. rocm-cli helps in three ways: reservation, e.g. `--gpu-memory-utilization 0.5`, or targeting a less-busy GPU with `--gpu `, and points at `rocm diagnose --symptom ''` for the full conditional remediation (busy GPU vs. a model that does not fit). + The printed command quotes your actual failing line when it can be rendered as + one intact single-quoted argument; a line carrying an apostrophe or terminal + control bytes falls back to the canonical symptom below rather than handing you + a command whose quoting the log text broke. Explicitly, the workaround for an OOM on a shared card is: diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index ae67a8dac..fa9839010 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -2231,20 +2231,26 @@ fn oom_utilization_hint(log_tail: &str) -> String { return String::new(); } // Route the user's *actual* failing line into the `--symptom` example when - // the diagnose checker would actually score it; otherwise fall back to the - // canonical symptom so the printed command always reports a cause. The - // detector here is a coarse substring scan that accepts lines the scorer - // rates sub-threshold (e.g. a bare "... out of memory"), so without this - // fallback the diagnose command could report nothing -- which reads as "the - // tool checked and there's no known cause", worse than not printing it. - let symptom_line = log_tail + // the diagnose checker would actually score it *and* it can be rendered as + // one intact single-quoted argument; otherwise fall back to the canonical + // symptom so the printed command always reports a cause. The detector here + // is a coarse substring scan that accepts lines the scorer rates + // sub-threshold (e.g. a bare "... out of memory"), so without this fallback + // the diagnose command could report nothing -- which reads as "the tool + // checked and there's no known cause", worse than not printing it. + let raw_line = log_tail .lines() .rev() .map(str::trim) .find(|line| !line.is_empty() && log_tail_shows_oom(line)) .unwrap_or("out of memory"); - let candidate = format!("vllm: {symptom_line}"); - let symptom = if rocm_core::vllm_oom_symptom_is_diagnosable(&candidate) { + // The line is subprocess output, so it is echoed only after the terminal + // control bytes vLLM's colourised logger emits are removed. + let symptom_line = strip_terminal_control_sequences(raw_line); + let candidate = format!("vllm: {raw_line}"); + let symptom = if quotable_in_single_quotes(&candidate) + && rocm_core::vllm_oom_symptom_is_diagnosable(&candidate) + { candidate } else { rocm_core::VLLM_OOM_CANONICAL_SYMPTOM.to_owned() @@ -2256,6 +2262,57 @@ fn oom_utilization_hint(log_tail: &str) -> String { ) } +/// Whether `symptom` can be placed inside a `'...'` shell word verbatim. +/// +/// The value is untrusted subprocess output (the vLLM startup log tail) and the +/// message it lands in invites the user to paste the command into a shell, so a +/// bare `'` would close the quote and let text nobody vetted become shell +/// syntax. An apostrophe is routine in Python error text (`can't allocate`, +/// `model 'foo'`), and a line only has to mention running out of memory to be +/// selected, so this is an ordinary case rather than an exotic one. +/// +/// Rejecting instead of escaping (`'` -> `'\''`) is deliberate. Escaping keeps +/// the exact bytes but yields a command a reader cannot check by eye, and a +/// wrong escape is *runnable* and misleading rather than obviously broken; +/// control bytes would still reach the terminal. The canonical fallback is the +/// branch that already exists for "this line cannot be used", and it is +/// guaranteed to report a cause. The user's own line stays visible in the +/// human-readable sentence above the command (and in the log tail printed with +/// it), so nothing is lost but the copy-paste convenience. +fn quotable_in_single_quotes(symptom: &str) -> bool { + !symptom.contains('\'') && !symptom.chars().any(char::is_control) +} + +/// Removes ANSI escape sequences and any remaining control characters, so a +/// colourised or bell-bearing log line cannot repaint the user's terminal from +/// inside rocm-cli's own error message. +fn strip_terminal_control_sequences(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\u{1b}' { + if chars.peek() == Some(&'[') { + // CSI (what a colourised logger emits): skip the parameter and + // intermediate bytes up to and including the final byte. + chars.next(); + for next in chars.by_ref() { + if ('\u{40}'..='\u{7e}').contains(&next) { + break; + } + } + } else { + // Any other escape: drop the byte it introduces too. + chars.next(); + } + continue; + } + if !c.is_control() { + out.push(c); + } + } + out +} + /// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a /// HIP allocation failure (e.g. `torch.OutOfMemoryError: HIP out of memory`). fn log_tail_shows_oom(log_tail: &str) -> bool { @@ -2781,6 +2838,74 @@ mod tests { } } + /// The `--symptom` value the hint actually hands the user, read back out of + /// the rendered text exactly the way a shell would: everything between the + /// opening quote that follows `--symptom ` and the next `'`. + fn quoted_symptom_argument(hint: &str) -> &str { + hint.split("--symptom '") + .nth(1) + .and_then(|rest| rest.split('\'').next()) + .expect("hint must carry a --symptom value") + } + + #[test] + fn an_apostrophe_in_the_failing_line_cannot_break_out_of_the_printed_command() { + // A realistic vLLM/PyTorch failing line: apostrophes are routine in + // Python error text, and this one scores well above MIN_SCORE_FOR_MATCH + // (torch.OutOfMemoryError + HIP out of memory), so the "route the user's + // real line" branch selects it. + let log_tail = concat!( + "ERROR 09-14 12:00:01 engine.py:389] torch.OutOfMemoryError: HIP out of memory. ", + "Tried to allocate 7.21 GiB. GPU 0 can't allocate the model's weights." + ); + assert!(log_tail_shows_oom(log_tail)); + let hint = oom_utilization_hint(log_tail); + + // The rendered command must be one intact single-quoted argument: no + // user-controlled byte may close the quote, so nothing the subprocess + // printed can land outside it in a command the user is told to paste. + let command = hint + .split("run `") + .nth(1) + .and_then(|rest| rest.split('`').next()) + .expect("hint must print a runnable command"); + assert_eq!( + command.matches('\'').count(), + 2, + "the --symptom argument must stay a single balanced quoted word: {command}" + ); + let symptom = quoted_symptom_argument(&hint); + assert!( + !symptom.contains('\''), + "no apostrophe may reach the single-quoted argument: {symptom:?}" + ); + // ...and the command it does print must still report a cause. + assert!( + rocm_core::vllm_oom_symptom_is_diagnosable(symptom), + "the fallback symptom must still be diagnosable: {symptom:?}" + ); + } + + #[test] + fn control_bytes_from_the_log_are_stripped_from_the_echoed_line() { + // vLLM's logger colourises; an ANSI-coloured OOM line must not repaint + // the user's terminal from inside rocm-cli's own error message. + let log_tail = "\u{1b}[31mRuntimeError: HIP out of memory\u{1b}[0m\u{7}"; + let hint = oom_utilization_hint(log_tail); + assert!( + !hint.chars().any(|c| c.is_control() && c != '\n'), + "no control byte may survive into the printed hint: {hint:?}" + ); + assert!( + !hint.contains("[31m") && !hint.contains("[0m"), + "the ANSI sequence must be removed whole, not just its escape byte: {hint:?}" + ); + assert!( + hint.contains("RuntimeError: HIP out of memory"), + "the readable part of the failing line must survive: {hint:?}" + ); + } + #[test] fn oom_utilization_hint_stays_quiet_for_unrelated_failures() { let unrelated = "ValueError: model architecture 'FooForCausalLM' is not supported"; From 4e81c72edb92a678e47186d7ff69ce755adb59d0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 10:35:28 +0300 Subject: [PATCH 19/39] test(vllm): pin the exact rendering instead of fragment absence The control-byte test carried the message "the ANSI sequence must be removed whole, not just its escape byte" but only asserted that "[31m" and "[0m" were absent. A stripper that drops the escape byte and the "[" it introduces and nothing further leaves "31m"/"0m" behind, which satisfies both absence checks and carries no control byte, so the test passed under exactly the defect it named. Verified by running that mutant: green before, failing now. Assert the stripper's output and the full rendered hint by equality instead. The neighbouring quote-rejection test had the same shape -- "no apostrophe in the symptom" also holds if the apostrophes were silently deleted, the escaping-style rewrite the canonical fallback exists to avoid -- so pin the fallback symptom by equality too. That mutant was also green before and fails now. Tests only; no behaviour change. Signed-off-by: Roman Sirokov --- engines/vllm/src/lib.rs | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index fa9839010..5d0a6ee06 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -2875,9 +2875,15 @@ mod tests { "the --symptom argument must stay a single balanced quoted word: {command}" ); let symptom = quoted_symptom_argument(&hint); - assert!( - !symptom.contains('\''), - "no apostrophe may reach the single-quoted argument: {symptom:?}" + // Pin which branch ran, not just that no apostrophe survived: "contains + // no `'`" also holds if the apostrophes were silently deleted from the + // user's line, which is the escaping-style behaviour the fallback exists + // to avoid. Rejection means the canonical symptom, exactly. + assert_eq!( + symptom, + rocm_core::VLLM_OOM_CANONICAL_SYMPTOM, + "a quote-bearing line must be rejected in favour of the canonical symptom, \ + not silently rewritten: {symptom:?}" ); // ...and the command it does print must still report a cause. assert!( @@ -2896,13 +2902,28 @@ mod tests { !hint.chars().any(|c| c.is_control() && c != '\n'), "no control byte may survive into the printed hint: {hint:?}" ); - assert!( - !hint.contains("[31m") && !hint.contains("[0m"), - "the ANSI sequence must be removed whole, not just its escape byte: {hint:?}" + // Pin the exact rendering rather than the absence of a few fragments. + // Absence checks cannot fail for the defect they name: a stripper that + // drops the escape byte and the `[` it introduces and nothing further + // leaves `31m`/`0m` behind, which contains neither `[31m` nor `[0m` and + // carries no control byte, so every absence check still passes. + assert_eq!( + strip_terminal_control_sequences(log_tail), + "RuntimeError: HIP out of memory", + "the ANSI sequence must be removed whole, not just its escape byte" ); - assert!( - hint.contains("RuntimeError: HIP out of memory"), - "the readable part of the failing line must survive: {hint:?}" + // The colourised line is not quotable (control bytes), so the command + // falls back to the canonical symptom while the echoed sentence keeps + // the user's own line, stripped. + assert_eq!( + hint, + format!( + "\n\nDetected an out-of-memory failure (RuntimeError: HIP out of memory). {}\n\ + For conditional remediation, run `rocm diagnose --symptom '{}'`.", + rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT, + rocm_core::VLLM_OOM_CANONICAL_SYMPTOM + ), + "the echoed line must render exactly, with the whole escape sequence gone" ); } From ba2b74117264570b3625aefc3e60f9a40ee571b8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 12:41:31 +0300 Subject: [PATCH 20/39] test(serve): pin the all-busy auto-GPU fallback to a reported ordinal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `select_auto_gpu_index` now sources its candidates from the actual VRAM rows, because a visibility mask can make reported ordinals non-contiguous. That removed the invariant the terminal all-candidates-busy fallback quietly relied on: with candidates drawn from a dense `0..count`, ordinal 0 was always a real device, so returning a literal `0` was safe by construction. With rows reported at [2, 3] it is not — the busy filter empties the candidate list, every pass falls through, and a fabricated 0 goes straight into `HIP_VISIBLE_DEVICES`, pinning a device nothing ever reported. Under the strict GPU-required / no-silent-fallback policy that is a silently wrong pin, not a fail-fast. The merge resolution already routes the fallback through the reported, mask-filtered set, so the behaviour is correct. What was missing is coverage: `auto_selection_ranks_the_actual_reported_row_indices` claims pass 3 "never returns a fabricated index 0", but it only exercises the *partially* busy case (busy = [3]), which never reaches the fallback at all. This adds the all-busy case that does. Falsified: with the fallback reverted to `vec![0]` this test fails with left: [0], right: [2] (and takes main's `auto_selection_all_visible_busy_falls_back_to_lowest_visible_not_zero` down with it); both pass with the fallback restored. Equality assertion rather than fragment absence, so a future regression cannot pass it vacuously. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 8d1a4ae2a..c5750f2a9 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -26831,6 +26831,30 @@ install therock"; ); } + #[test] + fn auto_selection_all_reported_busy_falls_back_to_a_reported_ordinal() { + // Same sparse-ordinal host as above ([2, 3] reported, nothing at 0 or 1) + // but now BOTH reported GPUs are pinned by a running service, so every + // pass falls through to the terminal "all busy, pick one anyway" + // fallback. That fallback used to hand back a hardcoded `0` — safe only + // while candidates were a dense `0..count`, and simply wrong once they + // come from the reported rows: index 0 is a device nothing reported, yet + // it would be exported verbatim as `HIP_VISIBLE_DEVICES`. It must return + // the lowest ordinal that was actually reported instead. + let usage = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(None, None, &[2, 3], Some(&usage)), + vec![2], + "the all-busy fallback must name a reported GPU, never a fabricated index 0" + ); + // The same holds when the busy set is given in descending order — the + // fallback tracks the reported ordinals, not the order they arrive in. + assert_eq!( + select_auto_gpu_index(None, None, &[3, 2], Some(&usage)), + vec![2] + ); + } + #[test] fn validate_pinned_gpu_index_rejects_out_of_range() { // Index equal to or beyond the detected count is rejected. From dffee3ea75df2f277bdadd86fa3cd4668f331e8c Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 12:44:15 +0300 Subject: [PATCH 21/39] docs(e2e): drop a serve-16 note the merge left dangling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment on `serve-16` described `--gpu` validation as falling back to membership in the KFD/DRM usable set and pointed at `validate_pinned_gpu_index_falls_back_to_usable_set_membership` as the unit coverage for the masked sub-case. Merging #267 replaced that validator with one that consults the visibility-resolved set directly, so the named test no longer exists, and the masked sub-case it called uncovered now has two scenarios of its own (serve-19, serve-20). Nothing here changes behaviour or which lane runs what; the paragraph is rewritten to say what serve-16 still covers and to point at the scenarios that own the rest. The conflict was textual-merge-clean — this block came wholly from one side — so only reading it caught the dangling reference. Signed-off-by: Roman Sirokov --- .../e2e-cucumber/features/model_serving.feature | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 42c22f6ce..e02995a0a 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -161,16 +161,12 @@ Feature: Model serving # refuses ("no usable AMD GPU") before the index is ever validated, so the # index-specific rejection can only be observed where a real device is present. # - # This is the live coverage for `--gpu` validation under either authority: the - # amd-smi device count, or — when amd-smi cannot count — membership in the - # KFD/DRM usable set (EAI-8058). Which one answers depends on the lane, so the - # step accepts either rejection wording. The *masked* sub-case that motivated - # the membership rule (a visibility mask leaving a non-contiguous usable set, - # e.g. `HIP_VISIBLE_DEVICES=2` on a 4-GPU host, where `--gpu 2` must pass and - # `--gpu 0` must be refused) has no scenario of its own: it is reachable only - # where amd-smi is *absent*, and faking that absence would assert against the - # seam rather than the product. It is unit-covered by - # `validate_pinned_gpu_index_falls_back_to_usable_set_membership`. + # This is the live coverage for an index the host does not have. The *masked* + # sub-case — an index that exists but is hidden by an active visibility mask — + # is a different refusal and owns its own scenarios, serve-19 and serve-20. + # The step here accepts several refusal wordings only because which authority + # answers depends on the lane: the visible set when it could be enumerated, + # the raw amd-smi count when it could not. @id:serve-absent-gpu-index-rejected @requires-gpu @requires-os:linux Scenario: serve-16 - Serving pinned to a GPU that does not exist is refused When the user serves a model pinned to a GPU index that does not exist From 18b13a305a7d7227a119b72e0429a9c50cae3602 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 13:47:28 +0300 Subject: [PATCH 22/39] fix(diagnose): treat a bare CR as a line boundary in the vLLM anchor filter `vllm_anchored_lines` split with `str::lines()`, which splits only on `\n` and `\r\n` -- a lone `\r` is ordinary text to it. Every progress bar in this ecosystem repaints with a bare CR (tqdm, pip, huggingface), and pasting a captured terminal scrollback is the expected way to use `rocm diagnose --symptom '...'`, so a real capture collapsed into one giant "line". The anchor then matched anywhere in it and every keyword in the paste scored at full weight: a llama.cpp OOM in a capture that also mentions vLLM came back at 95, well above HIGH_CONFIDENCE, as "vLLM ran the GPU out of memory at startup". That is the whole-paste scoring `only_the_anchored_lines_are_scored_not_the_ whole_paste` exists to prevent, reinstated by one byte. Neither existing guard saw it: both use `\n`-only fixtures. Split on `\r` as well, so the line boundary is the one the terminal actually renders, and add a regression case built from a bare-CR capture. It also pins the `\n` form of the same content and an anchored-across-CRs control, so the case stays a boundary question rather than a scoring one. EAI-8058 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 63 +++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index b0217843c..96b796dce 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -1502,12 +1502,23 @@ pub const VLLM_OOM_CANONICAL_SYMPTOM: &str = "vllm: torch.OutOfMemoryError: HIP /// line with no OOM token contributes nothing, so the score is 0 and the checker /// returns no diagnosis. The engine's emitted `vllm: ` keeps the /// anchor and the error on one line by construction. +/// +/// The split is on `\r` as well as `\n`, not `str::lines()`. `lines()` treats a +/// lone `\r` as ordinary text, and a bare CR is what every progress bar in this +/// ecosystem emits to repaint its line (tqdm, pip, huggingface). A raw terminal +/// capture pasted into `rocm diagnose --symptom '...'` therefore collapses into +/// one giant "line", the anchor matches anywhere in it, and every keyword in the +/// paste scores -- which is exactly the whole-paste scoring this function exists +/// to prevent, reinstated by one byte +/// (`a_bare_carriage_return_is_a_line_boundary_not_scoreable_text`). Splitting on +/// both makes the line boundary the one the terminal actually renders. `\r\n` +/// yields an empty segment, which carries no anchor and is dropped. fn vllm_anchored_lines(symptom: &str) -> String { let Ok(anchor) = Regex::new(VLLM_ANCHOR_PATTERN) else { return String::new(); }; symptom - .lines() + .split(['\n', '\r']) .filter(|line| anchor.is_match(&line.to_lowercase())) .collect::>() .join("\n") @@ -4116,6 +4127,56 @@ mod tests { assert!(oom.score >= HIGH_CONFIDENCE, "score was {}", oom.score); } + #[test] + fn a_bare_carriage_return_is_a_line_boundary_not_scoreable_text() { + // Regression: the anchored-lines filter used `str::lines()`, which + // splits only on `\n` and `\r\n` -- a lone `\r` is ordinary text to it. + // Every progress bar in this ecosystem (tqdm, pip, huggingface) + // repaints with a bare CR, and a pasted terminal capture is the + // expected way to use `--symptom`, so the whole capture collapsed into + // one "line": the vLLM anchor matched somewhere in it and another + // framework's OOM elsewhere in it scored at full weight. That is + // `only_the_anchored_lines_are_scored_not_the_whole_paste` defeated by + // one byte, and neither existing guard saw it because both use + // `\n`-only fixtures. + let capture = "Downloading shards: 10%\rvllm serve starting up\r\ + Downloading shards: 90%\r\ + llama.cpp: torch.OutOfMemoryError: CUDA out of memory. \ + Tried to allocate 7.21 GiB.\n"; + let report = diagnose(&linux_base(), capture); + assert!( + report.matched.iter().all(|d| d.id != "fix-16-vllm-oom"), + "a CR-separated llama.cpp OOM must not be attributed to vLLM: {:?}", + report.matched + ); + + // The same content with `\n` in place of `\r` is the already-covered + // shape; pinning both together is what makes the CR case a boundary + // question rather than a scoring question. + let newline_form = capture.replace('\r', "\n"); + assert!( + diagnose(&linux_base(), &newline_form) + .matched + .iter() + .all(|d| d.id != "fix-16-vllm-oom"), + "the `\\n` form of the same capture must not match either" + ); + + // The control: a CR-separated capture whose *anchored* segment carries + // the OOM must still match, so splitting on CR did not simply blind the + // checker to carriage-returned input. + let anchored = diagnose( + &linux_base(), + "Downloading shards: 90%\rvllm: torch.OutOfMemoryError: HIP out of memory\r", + ); + let oom = anchored + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("an anchored OOM segment must still match across CRs"); + assert!(oom.score >= MIN_SCORE_FOR_MATCH, "score was {}", oom.score); + } + #[test] fn rocm_cli_s_own_low_vram_hint_is_never_a_verdict_on_its_own() { // `gpu[-_]memory[-_]utilization` is both an anchor and a scoring entry, From 043e02cc77aa8f065a8128e3f68861998b89e7db Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 13:47:46 +0300 Subject: [PATCH 23/39] fix(vllm): parse the escape grammar and reject format characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OOM hint quotes a line of untrusted subprocess output into a `rocm diagnose --symptom '...'` command and echoes it in a `bail!`-ed message, so the sanitizer in front of it is load-bearing. Three holes: The escape scan ran to the next byte in `0x40..=0x7E` with no notion of the CSI grammar's parameter, intermediate and final classes. On well-formed colour codes that is indistinguishable from correct; on the malformed and truncated sequences a *killed* process emits -- which is the only case this code path exists for -- every one of them corrupted the message. A truncated CSI swallowed the next sequence's introducer (`ESC [ 1 ; 2 ESC [ 0 m Killed` rendered as `0mKilled`); a multi-byte scalar can never be in that ASCII range, so the scan ran past it and ate to the next byte that happened to land in it (`ESC [ 1 2 é out of memory` rendered as `ut of memory`); an OSC title leaked its body as text; a stray `ESC ESC` left `[0m` printed literally. Follow the grammar, and on an invalid byte stop *without consuming it*, so the damage is bounded to the malformed sequence and a following well-formed one is still recognised. `char::is_control` is Unicode category `Cc` only, so `Cf` format characters survived both the stripper and `quotable_in_single_quotes`. U+202E in a log line cannot escape the single quotes, but it does reverse how the printed command renders -- the same "untrusted output must not control what the terminal shows" concern, so it gets the same answer. The stripper's doc comment promised control bytes could not repaint the terminal "from inside rocm-cli's own error message", while the log tail a few lines above it in the same message went out raw. Sanitize the tail too, per line so it keeps its shape. Also states why `.rev()` is deliberate (the last OOM line is the proximate failure) and what it costs, since nothing said so; behaviour is unchanged. Tests: the existing control-byte test only exercised an `'m'`-terminated CSI, so narrowing the final-byte range to `'m'` left it green. The new case table pins the range, the byte classes and every introducer. The diagnosability test could not fail for the feature it is named for either -- a mutant that always returns VLLM_OOM_CANONICAL_SYMPTOM satisfies it, because that symptom is diagnosable by construction -- so it now pins the exact symptom per line, over a table that deliberately straddles the routing and fallback branches. EAI-8058 Signed-off-by: Roman Sirokov --- engines/vllm/src/lib.rs | 392 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 360 insertions(+), 32 deletions(-) diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 5d0a6ee06..ed921dd61 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -2210,6 +2210,15 @@ fn print_json(value: &T) -> Result<()> { /// Builds a human-readable summary of the tail of the startup log, if available. /// Returns an empty string when no log is present or it cannot be read. +/// +/// The tail is sanitized before it is embedded. It is untrusted subprocess +/// output and this whole string is `bail!`-ed to the user's terminal, so +/// stripping control bytes out of the one echoed line further down while +/// printing the raw tail a few lines above would have left the escapes a clear +/// path to the terminal inside the same message. Stripping is per line so the +/// tail keeps its shape; `oom_utilization_hint` still reads the *raw* tail, +/// because what it selects from it is quoted into a command and has its own, +/// stricter admissibility test. fn startup_log_context(log_path: Option<&Path>) -> String { let summary = log_path .and_then(|p| summarize_startup_log_tail(p, STARTUP_FAILURE_LOG_TAIL_LINES).ok()) @@ -2218,6 +2227,11 @@ fn startup_log_context(log_path: Option<&Path>) -> String { return String::new(); } let hint = oom_utilization_hint(&summary); + let summary = summary + .lines() + .map(strip_terminal_control_sequences) + .collect::>() + .join("\n"); format!("\n\nLast {STARTUP_FAILURE_LOG_TAIL_LINES} lines of startup log:\n{summary}{hint}") } @@ -2238,6 +2252,16 @@ fn oom_utilization_hint(log_tail: &str) -> String { // sub-threshold (e.g. a bare "... out of memory"), so without this fallback // the diagnose command could report nothing -- which reads as "the tool // checked and there's no known cause", worse than not printing it. + // `.rev()` is deliberate, not incidental: when several lines of the tail + // mention running out of memory, the *last* one is the proximate failure — + // the point where the process actually gave up — and the earlier ones are + // usually the allocator's own retry chatter leading up to it. The cost is + // real and worth stating: on a tail like "torch.OutOfMemoryError: ... Tried + // to allocate 7.21 GiB. GPU 0 has a total capacity of 24.00 GiB." followed + // by a bare "RuntimeError: ... killed: out of memory", the vaguer line wins + // even though the first carries the allocation size and the capacity. Both + // are diagnosable, so this picks which line is quoted, never whether one is. + // The user still sees every line: the log tail is printed above this hint. let raw_line = log_tail .lines() .rev() @@ -2279,40 +2303,164 @@ fn oom_utilization_hint(log_tail: &str) -> String { /// guaranteed to report a cause. The user's own line stays visible in the /// human-readable sentence above the command (and in the log tail printed with /// it), so nothing is lost but the copy-paste convenience. +/// +/// The character test is [`is_control_or_format`], not `char::is_control`: the +/// latter is Unicode `Cc` only, so a bidi override in the failing line survived +/// into the printed command and reordered how it renders. fn quotable_in_single_quotes(symptom: &str) -> bool { - !symptom.contains('\'') && !symptom.chars().any(char::is_control) + !symptom.contains('\'') && !symptom.chars().any(is_control_or_format) } -/// Removes ANSI escape sequences and any remaining control characters, so a -/// colourised or bell-bearing log line cannot repaint the user's terminal from -/// inside rocm-cli's own error message. +/// Removes ANSI escape sequences and any remaining control or format +/// characters, so a colourised or bell-bearing log line cannot repaint the +/// user's terminal, or reorder how the message renders, from inside rocm-cli's +/// own error message. +/// +/// The sequence grammar is ECMA-48's, followed exactly, and the reason is that +/// the input is a *killed* process's output: truncated and interleaved escapes +/// are the normal case here, not an exotic one. A scan that just ran to the +/// next byte in `0x40..=0x7E` mis-handled every one of them — it swallowed the +/// following sequence's introducer (`ESC [ 1 ; 2 ESC [ 0 m Killed` lost +/// `Killed`'s first six characters), ran straight past a multi-byte scalar +/// (which can never be in that ASCII range) and ate everything up to the next +/// byte that happened to land in it, emitted the body of an OSC title +/// sequence as text, and left a stray `ESC ESC` unrecoverable. +/// +/// Consuming only what the grammar allows and then *stopping without consuming* +/// the offending byte bounds the damage to the malformed sequence itself: the +/// text after it survives, and a following well-formed sequence is still +/// recognised because its `ESC` is left for the main loop to re-read. +/// +/// What this deliberately does not do is second-guess a well-formed sequence. +/// `ESC [ SP K` is a valid CSI (`SP` is an intermediate, `K` the final byte), so +/// it is consumed whole even though the `K` may have been the first letter of a +/// truncated process's "Killed" — a real terminal consumes it too, and the +/// contract here is "render what the terminal would have rendered", which is +/// the only rule that stays decidable on a byte stream with no framing. fn strip_terminal_control_sequences(line: &str) -> String { let mut out = String::with_capacity(line.len()); let mut chars = line.chars().peekable(); while let Some(c) = chars.next() { - if c == '\u{1b}' { - if chars.peek() == Some(&'[') { - // CSI (what a colourised logger emits): skip the parameter and - // intermediate bytes up to and including the final byte. - chars.next(); - for next in chars.by_ref() { - if ('\u{40}'..='\u{7e}').contains(&next) { - break; - } - } - } else { - // Any other escape: drop the byte it introduces too. - chars.next(); + if c != '\u{1b}' { + if !is_control_or_format(c) { + out.push(c); } continue; } - if !c.is_control() { - out.push(c); + match chars.peek() { + // CSI — what a colourised logger emits. + Some('[') => { + chars.next(); + skip_csi_body(&mut chars); + } + // The string-argument sequences: OSC, DCS, SOS, PM, APC. Their + // bodies are arbitrary text (a window title, say) and must not be + // emitted as if the process had printed it. + Some(']' | 'P' | 'X' | '^' | '_') => { + chars.next(); + skip_string_sequence_body(&mut chars); + } + // `ESC` with nothing after it, or `ESC ESC`: drop just this one and + // let the loop re-read the next as a fresh introducer. + None | Some('\u{1b}') => {} + // Any other escape: optional intermediates, then one final byte. + Some(_) => skip_simple_escape_body(&mut chars), } } out } +/// Consumes a CSI body — parameter bytes, then intermediate bytes, then one +/// final byte — from just after the `ESC [`. Stops without consuming anything +/// that does not belong, leaving it to be treated as text. +fn skip_csi_body(chars: &mut std::iter::Peekable>) { + while chars + .next_if(|c| ('\u{30}'..='\u{3f}').contains(c)) + .is_some() + {} + while chars + .next_if(|c| ('\u{20}'..='\u{2f}').contains(c)) + .is_some() + {} + chars.next_if(|c| ('\u{40}'..='\u{7e}').contains(c)); +} + +/// Consumes a string-argument sequence's body and terminator (`BEL`, or `ST` = +/// `ESC \`) from just after the introducer. A body truncated by anything else — +/// including a bare `ESC` starting the next sequence — ends the scan with that +/// byte left in place. +fn skip_string_sequence_body(chars: &mut std::iter::Peekable>) { + while let Some(&c) = chars.peek() { + if c == '\u{7}' { + chars.next(); + return; + } + if c == '\u{1b}' { + let mut lookahead = chars.clone(); + lookahead.next(); + if lookahead.peek() == Some(&'\\') { + chars.next(); + chars.next(); + } + return; + } + chars.next(); + } +} + +/// Consumes a non-CSI escape's body — optional intermediates, then one final +/// byte — from just after the `ESC`. +fn skip_simple_escape_body(chars: &mut std::iter::Peekable>) { + while chars + .next_if(|c| ('\u{20}'..='\u{2f}').contains(c)) + .is_some() + {} + chars.next_if(|c| ('\u{30}'..='\u{7e}').contains(c)); +} + +/// Whether `c` is a character that carries no glyph of its own but changes how +/// the text around it renders. +/// +/// `char::is_control` is Unicode category `Cc` only, so it misses the `Cf` +/// format characters — and `U+202E RIGHT-TO-LEFT OVERRIDE` in a log line +/// reverses how the rest of the printed `rocm diagnose --symptom '...'` command +/// renders in the user's terminal. That cannot escape the single quotes (no +/// ASCII `'` is involved), so it is display spoofing rather than shell +/// injection, but it is the same "untrusted subprocess output must not control +/// what the terminal shows" concern the escape stripping above exists for, and +/// the answer has to be the same. +/// +/// The ranges are the `Cf` category, enumerated rather than pulled from a +/// Unicode-tables dependency: the set is small, stable, and a new dependency in +/// an engine binary for one predicate is a worse trade. Over-inclusion is safe +/// here — the only cost of rejecting a character is falling back to the +/// canonical symptom, which is guaranteed to report a cause. +fn is_control_or_format(c: char) -> bool { + c.is_control() + || matches!(c, + '\u{00ad}' + | '\u{0600}'..='\u{0605}' + | '\u{061c}' + | '\u{06dd}' + | '\u{070f}' + | '\u{0890}'..='\u{0891}' + | '\u{08e2}' + | '\u{180e}' + | '\u{200b}'..='\u{200f}' + | '\u{202a}'..='\u{202e}' + | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{206f}' + | '\u{feff}' + | '\u{fff9}'..='\u{fffb}' + | '\u{110bd}' + | '\u{110cd}' + | '\u{13430}'..='\u{1343f}' + | '\u{1bca0}'..='\u{1bca3}' + | '\u{1d173}'..='\u{1d17a}' + | '\u{e0001}' + | '\u{e0020}'..='\u{e007f}') +} + /// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a /// HIP allocation failure (e.g. `torch.OutOfMemoryError: HIP out of memory`). fn log_tail_shows_oom(log_tail: &str) -> bool { @@ -2809,33 +2957,73 @@ mod tests { } #[test] - fn every_emitted_oom_symptom_is_diagnosable() { + fn every_emitted_oom_symptom_is_diagnosable_and_names_the_branch_that_produced_it() { // Closes the loop between the two layers: the engine prints // `rocm diagnose --symptom ''`, so whatever it emits must // actually score for the diagnose checker -- including for lines the // coarse substring detector accepts but the scorer rates sub-threshold // on their own (those fall back to the canonical symptom). + // + // Each case pins the *exact* symptom, not just that one is diagnosable. + // Diagnosability alone cannot fail for the defect this test is named + // for: the canonical symptom is diagnosable by construction, so a + // mutant that deletes the "route the user's real failing line" feature + // entirely and always returns VLLM_OOM_CANONICAL_SYMPTOM satisfies + // every such assertion. Naming the expected symptom per line is what + // makes the routing branch and the fallback branch separately + // falsifiable -- and it is why the table below deliberately mixes the + // two: three lines score on their own and must be quoted verbatim, + // three do not and must fall back. let accepted_lines = [ - "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", - "RuntimeError: hipErrorOutOfMemory", - "torch.cuda.OutOfMemoryError", - "HIP error: out of memory", - "the process was killed: out of memory", - "CUDA out of memory", + ( + "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + ), + ( + "RuntimeError: hipErrorOutOfMemory", + "vllm: RuntimeError: hipErrorOutOfMemory", + ), + ("CUDA out of memory", "vllm: CUDA out of memory"), + // Sub-threshold on their own: the class name with no allocator + // message, the allocator message with no class name, and a bare + // kill notice. These must reach the canonical fallback. + ( + "torch.cuda.OutOfMemoryError", + rocm_core::VLLM_OOM_CANONICAL_SYMPTOM, + ), + ( + "HIP error: out of memory", + rocm_core::VLLM_OOM_CANONICAL_SYMPTOM, + ), + ( + "the process was killed: out of memory", + rocm_core::VLLM_OOM_CANONICAL_SYMPTOM, + ), ]; - for line in accepted_lines { + let mut routed_verbatim = 0; + for (line, expected) in accepted_lines { assert!(log_tail_shows_oom(line), "detector must accept: {line}"); let hint = oom_utilization_hint(line); - let symptom = hint - .split("--symptom '") - .nth(1) - .and_then(|rest| rest.split('\'').next()) - .expect("hint must carry a --symptom value"); + let symptom = quoted_symptom_argument(&hint); + assert_eq!( + symptom, expected, + "wrong branch for {line:?}: the engine must quote the user's own line when \ + the checker scores it and fall back to the canonical symptom only when it \ + does not" + ); assert!( rocm_core::vllm_oom_symptom_is_diagnosable(symptom), "emitted symptom must be diagnosable, got {symptom:?} for line {line:?}" ); + if symptom != rocm_core::VLLM_OOM_CANONICAL_SYMPTOM { + routed_verbatim += 1; + } } + assert_eq!( + routed_verbatim, 3, + "the table must keep exercising both branches; if a scoring change moved a line \ + across the threshold, re-pick the fixture rather than relaxing the expectation" + ); } /// The `--symptom` value the hint actually hands the user, read back out of @@ -2927,6 +3115,146 @@ mod tests { ); } + #[test] + fn the_stripper_follows_the_escape_grammar_not_just_the_colour_case() { + // The `'m'`-terminated SGR case above is the *easy* one, and on its own + // it pins almost nothing: narrowing the CSI final-byte range from + // `0x40..=0x7E` to just `'m'` leaves it green. These cases pin the + // range, the parameter/intermediate classes, and the introducers. + // + // They are not academic. This input is a *killed* process's output, so + // truncated and interleaved sequences are the normal case on this code + // path, and every one of them used to corrupt the message the user + // reads -- the failures are quoted per case below. + for (raw, expected, defect) in [ + // Non-`m` CSI finals: `K` (erase-in-line) and `A` (cursor-up) are + // ordinary logger output, and narrowing the final-byte range to + // `'m'` leaves them in the message verbatim. + ( + "\u{1b}[2KRuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "a non-`m` CSI final must terminate the sequence", + ), + ( + "\u{1b}[1ARuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "a non-`m` CSI final must terminate the sequence", + ), + // Truncated CSI immediately followed by a well-formed one: the old + // scan consumed the second sequence's `ESC [` as the first one's + // parameters and stopped at `0`, yielding "0mKilled: out of memory". + ( + "\u{1b}[1;2\u{1b}[0mKilled: out of memory", + "Killed: out of memory", + "a truncated CSI must not swallow the next sequence's introducer", + ), + // A multi-byte scalar can never be in `0x40..=0x7E`, so the old scan + // ran past it and ate to the next byte that happened to land in + // range -- this yielded "ut of memory", losing the `o`. + ( + "\u{1b}[12\u{e9} out of memory", + "\u{e9} out of memory", + "an invalid CSI byte must end the sequence, not be scanned past", + ), + // `ESC ESC`: the old code dropped the second `ESC` as the first + // one's argument, then emitted `[0m` as literal text. + ( + "\u{1b}\u{1b}[0mRuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "a stray ESC must not consume the next sequence's introducer", + ), + // OSC: the old code took the `else` branch on `]`, so the window + // title leaked into the message as "0;titleRuntimeError: ...". + ( + "\u{1b}]0;title\u{7}RuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "an OSC body must not be emitted as text", + ), + // ...and with the ST terminator (`ESC \`) rather than BEL. + ( + "\u{1b}]0;title\u{1b}\\RuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "an OSC terminated by ST must be consumed whole", + ), + // A two-character escape with no CSI at all. + ( + "\u{1b}7RuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "a simple escape must consume exactly its final byte", + ), + // Cf format characters: `char::is_control` is category Cc only, so + // U+202E survived and reversed how the rest of the line renders. + ( + "RuntimeError: \u{202e}HIP out of memory", + "RuntimeError: HIP out of memory", + "a bidi override must not survive into the message", + ), + // The text-only control: nothing is removed from a clean line. + ( + "RuntimeError: HIP out of memory", + "RuntimeError: HIP out of memory", + "a clean line must pass through untouched", + ), + ] { + assert_eq!( + strip_terminal_control_sequences(raw), + expected, + "{defect}: {raw:?}" + ); + } + + // Cf characters must also be inadmissible in the quoted command, not + // merely stripped from the echoed sentence: the two tests are separate + // because they are separate call sites and only one of them used + // `char::is_control`. + assert!( + !quotable_in_single_quotes("vllm: \u{202e}HIP out of memory"), + "a bidi override must make a line unquotable, not ride into the command" + ); + assert!( + quotable_in_single_quotes("vllm: HIP out of memory"), + "the rejection must not be so broad that ordinary lines stop qualifying" + ); + } + + #[test] + fn the_printed_log_tail_is_sanitized_not_just_the_echoed_line() -> Result<()> { + // The stripper's contract is that nothing in this message can repaint + // the user's terminal. The log tail is printed a few lines above the + // echoed line in the same `bail!`-ed string, and it used to go out raw, + // so the escapes removed from one had a clear path to the terminal via + // the other. + let path = std::env::temp_dir().join(format!( + "rocm-vllm-sanitize-{}-{}.log", + std::process::id(), + current_unix_millis() + )); + fs::write( + &path, + "\u{1b}[31mINFO starting\u{1b}[0m\n\u{1b}]0;pwned\u{7}RuntimeError: HIP out of memory\n", + )?; + let context = startup_log_context(Some(path.as_path())); + fs::remove_file(&path).ok(); + + assert!( + !context.chars().any(|c| c.is_control() && c != '\n'), + "no control byte may survive into the printed failure message: {context:?}" + ); + assert!( + !context.contains("pwned"), + "an OSC body in the tail must not be emitted as text: {context:?}" + ); + assert_eq!( + context + .split("startup log:\n") + .nth(1) + .and_then(|rest| rest.split("\n\nDetected").next()), + Some("INFO starting\nRuntimeError: HIP out of memory"), + "the tail must keep its text and its line structure, minus the escapes: {context:?}" + ); + Ok(()) + } + #[test] fn oom_utilization_hint_stays_quiet_for_unrelated_failures() { let unrelated = "ValueError: model architecture 'FooForCausalLM' is not supported"; From 025e2da20330e42f9a995f950deebc141d26cb3a Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 13:48:00 +0300 Subject: [PATCH 24/39] test(diagnose): pin the fix-16 plan to its catalog copy The command block is maintained twice -- once on the `Fix` that `diagnose` attaches, once on the `rocm fix` catalog recipe -- and the two had already drifted. The cross-check that runs over every matched finding could not see it: `recipe_verify_notes_and_run_commands` filters `#` lines by construction, and only the comments differed. That prose is the point of this entry. Two different faults produce the same OOM and need opposite responses, so the comments are what tell the user which of the two each step addresses -- a user who meets the `diagnose` copy and a user who meets the `rocm fix` copy must be told the same thing. Align the `diagnose` copy to the catalog's Case 1/Case 2 wording and wire fix-16 into `assert_plan_matches_the_catalog_copy`, the byte-identical guard that already existed for fix-17-torch-dlpack. Also restores the general rule to `LINUX_AND_WSL`'s doc. Deduplicating the const in 1e23717d kept the fix-16-specific comment and dropped main's general one, leaving a shared const documented as if fix-16 were its only user; fix-19-shm-too-small uses it too. EAI-8058 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 33 ++++++++++++++++++++++++++++---- crates/rocm-core/src/fix.rs | 13 ++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 96b796dce..c3eca5bea 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -1573,13 +1573,16 @@ fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { ); let fix = Fix { summary, + // Byte-identical to the `fix-16-vllm-oom` catalog recipe's block, and + // pinned there by `the_oom_plan_matches_the_catalog_copy`. The two are + // one plan in two modules and a user may meet either copy. commands: vec![ - "# If the GPU is shared/busy (tenancy collision), lower vLLM's reservation:".to_owned(), + "# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,".to_owned(), + "# or steer vLLM onto a less-busy device:".to_owned(), "rocm serve --gpu-memory-utilization 0.5".to_owned(), - "# ...or steer the server onto a less-busy device:".to_owned(), "rocm serve --gpu ".to_owned(), - "# If the model genuinely does not fit, the reservation is not the problem:".to_owned(), - "# pick a smaller or quantized model (single-GPU serving only).".to_owned(), + "# Case 2 -- the model genuinely does not fit: the reservation is not the".to_owned(), + "# problem; pick a smaller or quantized model (single-GPU serving only).".to_owned(), ], fix_id: "fix-16-vllm-oom".to_owned(), auto_applicable: false, @@ -2916,6 +2919,28 @@ mod tests { crate::fix::assert_plan_matches_the_catalog_copy(&fix.fix_id, &commands); } + #[test] + fn the_oom_plan_matches_the_catalog_copy() { + // Same two-copies-of-one-plan risk as `fix-17-torch-dlpack`, and it had + // already opened: the executable lines agreed, so the cross-check that + // runs over every matched finding + // (`the_reported_fix_agrees_with_the_catalog_recipe`) saw nothing -- + // it filters `#` lines by construction -- while the comments explaining + // *which of the two faults each step addresses* had drifted apart. That + // prose is the whole point of this entry: the two faults need opposite + // responses, so a user meeting the `diagnose` copy and a user meeting + // the `rocm fix` copy must be told the same thing. + let report = diagnose(&linux_base(), VLLM_OOM_CANONICAL_SYMPTOM); + let fix = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .and_then(|d| d.fix.as_ref()) + .expect("the finding must carry a plan"); + let commands: Vec<&str> = fix.commands.iter().map(String::as_str).collect(); + crate::fix::assert_plan_matches_the_catalog_copy(&fix.fix_id, &commands); + } + #[test] fn the_extension_name_alone_does_not_establish_the_variant_failure() { // The extension appearing in a traceback says it is involved, not that diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 5a86ba5f4..abdf2a522 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -76,9 +76,16 @@ const LINUX_WINDOWS_AND_WSL: &[&str] = &["linux", "windows", "wsl"]; const LINUX_AND_WINDOWS: &[&str] = &["linux", "windows"]; /// For entries whose fault is not tied to bare-metal kernel plumbing. /// -/// `fix-16-vllm-oom` is the case: its remediation is vLLM CLI flags, and its -/// checker already opts into `wsl` for that reason, so a Linux-only recipe made -/// `rocm diagnose` name a command `rocm fix` then refused to run. +/// The amdgpu module, `/dev/kfd` and the render group are what WSL2 does not +/// have, so a recipe that touches any of them is `LINUX_ONLY`. A recipe whose +/// fault lives above that layer reproduces on WSL2 exactly as it does on Linux, +/// and its platform set has to say so — otherwise `rocm fix` refuses to run a +/// fix `rocm diagnose` has just named. The rule is the checker's own platform +/// set: a recipe must be runnable everywhere its checker answers +/// (`every_checker_platform_is_covered_by_its_recipe`). +/// +/// `fix-16-vllm-oom` (remediated with vLLM CLI flags) and `fix-19-shm-too-small` +/// (a container's `/dev/shm` size) are both that shape. const LINUX_AND_WSL: &[&str] = &["linux", "wsl"]; const LINUX_ONLY: &[&str] = &["linux"]; const WINDOWS_ONLY: &[&str] = &["windows"]; From 7e4f664a9dc472b41bbb788b76907ef30b6a1628 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 15 Sep 2026 13:48:14 +0300 Subject: [PATCH 25/39] refactor(serve): drop a vacuous guard and the duplicated AMD-device test Three leftovers from the auto-selection rewrite and the merge around it: `select_auto_gpu_index`'s pass-2 `max_by` still carried `&& usage_for(index).is_some()`. Inside `if vram.is_some()` every candidate came from a reported VRAM row, so that is vacuously true -- it implies a failure mode (a selected ordinal telemetry never reported) that only existed when candidates were a synthetic `0..count` range. Replace it with a comment saying why no guard is needed. `drm_device_is_amd` re-implemented the private `rocm_core::is_amdgpu_device`, with a doc comment asserting the two were "the same two-signal test" and that divergence would let the multi-card ordinal guard slip -- a promise nothing enforced. They must agree because both count AMD cards and this one feeds that guard, so make the rocm-core function public and delegate. One copy is better than two copies plus a cross-check. The second assertion in `auto_selection_all_reported_busy_falls_back_to_a_ reported_ordinal` varied the order of the `busy` slice, which is only ever read through `.contains()` -- so it was a duplicate of the first while its comment claimed an order-independence property nobody was testing. Vary the order of the VRAM *rows* instead: `amd-smi` orders by enumeration, not by index, and that is what `reported.sort_unstable()` is there for. Dropping the sort now turns it red with `left: [3], right: [2]`; it could not before. EAI-8058 Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 44 ++++++++++++++++++++----------------- crates/rocm-core/src/lib.rs | 17 ++++++++++++-- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index c5750f2a9..3c5b9284e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -19177,7 +19177,11 @@ fn select_auto_gpu_index( } // Pass 2: the non-busy GPU with the most free VRAM in absolute terms // (not free percentage, which can favor a smaller GPU on heterogeneous - // VRAM systems). + // VRAM systems). No "does this ordinal have a row?" guard is needed: + // inside this branch every candidate came *from* a reported row, so + // `usage_for` is total over `candidate_indices`. It was needed when + // candidates were a synthetic `0..count` range, where an ordinal could + // be selected that telemetry had never reported. if let Some(&index) = candidate_indices.iter().max_by(|left, right| { let left_free = usage_for(**left).map(|usage| usage.free_mb()); let right_free = usage_for(**right).map(|usage| usage.free_mb()); @@ -19185,8 +19189,7 @@ fn select_auto_gpu_index( .cmp(&right_free) // Break ties toward the lowest index. .then(right.cmp(left)) - }) && usage_for(index).is_some() - { + }) { return vec![index]; } } @@ -19367,21 +19370,17 @@ fn read_sysfs_u64(path: &Path) -> Option { fs::read_to_string(path).ok()?.trim().parse::().ok() } -/// Whether a DRM `device` directory belongs to an AMD GPU. Matches on the PCI -/// `vendor` id (`0x1002`) or, when that is unreadable, an `amdgpu` `uevent` -/// `DRIVER=` line — the same two-signal test as `rocm_core::is_amdgpu_device`, -/// so this fallback probe and the KFD/DRM count authority agree on what counts -/// as an AMD card (a vendor-only test would under-count and let the multi-card -/// ordinal guard slip). +/// Whether a DRM `device` directory belongs to an AMD GPU. +/// +/// Delegates rather than re-implementing. This probe and `rocm-core`'s KFD/DRM +/// count authority have to agree on what counts as an AMD card — if this one +/// recognises fewer, it under-counts and the multi-card ordinal guard it feeds +/// silently narrows — and a second copy of the test with a comment claiming the +/// two match is not agreement, it is a promise nothing enforces. Calling the +/// same function is. #[cfg(any(target_os = "linux", test))] fn drm_device_is_amd(device_dir: &Path) -> bool { - if fs::read_to_string(device_dir.join("vendor")) - .is_ok_and(|vendor| vendor.trim().eq_ignore_ascii_case("0x1002")) - { - return true; - } - fs::read_to_string(device_dir.join("uevent")) - .is_ok_and(|uevent| uevent.lines().any(|line| line.trim() == "DRIVER=amdgpu")) + rocm_core::is_amdgpu_device(device_dir) } /// Parse `amd-smi metric --json` output into per-GPU VRAM usage. Accepts both @@ -26847,11 +26846,16 @@ install therock"; vec![2], "the all-busy fallback must name a reported GPU, never a fabricated index 0" ); - // The same holds when the busy set is given in descending order — the - // fallback tracks the reported ordinals, not the order they arrive in. + // "Lowest reported" has to mean lowest by ordinal, not first in the + // rows: `amd-smi` orders its output by enumeration, not by index, so + // feed the same two rows in descending order. Varying the *busy* slice's + // order instead would prove nothing — it is only ever read through + // `.contains()`, so that assertion was a duplicate of the one above. + let descending = [vram(3, 190_000, 192_000), vram(2, 182_000, 192_000)]; assert_eq!( - select_auto_gpu_index(None, None, &[3, 2], Some(&usage)), - vec![2] + select_auto_gpu_index(None, None, &[2, 3], Some(&descending)), + vec![2], + "the fallback must take the lowest reported ordinal, not the first row" ); } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 19b315b63..e5b09bfb7 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -5357,8 +5357,21 @@ fn detect_linux_drm_ip_discovery_gfx_target() -> Option { None } -#[cfg(any(target_os = "linux", test))] -fn is_amdgpu_device(device_dir: &Path) -> bool { +/// Whether a DRM `device` directory belongs to an AMD GPU: the PCI `vendor` id +/// (`0x1002`), or, when that is unreadable, an `amdgpu` `uevent` `DRIVER=` line. +/// +/// Both signals are needed. A vendor-only test under-counts on hosts where +/// `vendor` is absent or unreadable, and the callers are counting *cards* — the +/// KFD/DRM count authority here and the `rocm` CLI's sysfs fallback probe — so +/// an under-count silently narrows the multi-card ordinal guard. +/// +/// `pub` and not `#[cfg(target_os = "linux")]` precisely so there is one copy: +/// the CLI's fallback probe used to carry its own, with a doc comment asserting +/// the two were "the same two-signal test" and nothing holding them to it. It +/// reads files, so there is nothing platform-specific to gate, and gating it +/// would put it out of reach of a dependent crate's `cfg(test)` build. +#[must_use] +pub fn is_amdgpu_device(device_dir: &Path) -> bool { if let Ok(vendor) = fs::read_to_string(device_dir.join("vendor")) && vendor.trim().eq_ignore_ascii_case("0x1002") { From fcf9915665d91c41a86d0bf54a4c236d75b0219e Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 11:18:29 +0300 Subject: [PATCH 26/39] fix(diagnose): split the vLLM anchor filter on rendered lines, not two bytes The previous round fixed the whole-paste collapse for a bare `\r` and left it open for every other line advance. `.split(['\n', '\r'])` is an allowlist of two characters under a doc comment claiming the result was "the line boundary the one the terminal actually renders", and it was not: with the `vllm` anchor and a llama.cpp OOM on genuinely separate rendered lines, `ESC E` (NEL), `ESC D` (IND), `CSI 1 B` (CUD), `\x0b`, `\x0c`, `\x85`, U+2028/U+2029 and any stray control byte each scored 95 against a `high_confidence_threshold` of 75 -- citing the llama.cpp line's tokens as the evidence for a vLLM verdict. `ESC E`, `ESC D` and `CSI n B` are terminfo's `nel`/`ind`/`cud1`, i.e. ordinary output from curses- and `rich`-style progress UIs, and `rocm serve`'s own failure path invites exactly this paste. The one-line `char::is_control()` split that suggests itself does close every one of those, but it trades the false positive for a false negative on the most ordinary input there is: a colourised logger emits SGR *inside* a line, so splitting at `ESC` cuts `ESC[31m vllm: ESC[1m torch.OutOfMemoryError` between its anchor and its error token. Measured on that candidate, that line went from 95 to 0. Getting both halves right needs the escape grammar, and this crate's dependents already have one -- `strip_terminal_control_sequences`, which took three rounds to get right in the vLLM engine. Rather than write a second parser, move the ECMA-48 walk down into `rocm-core::terminal` as a classifier over three tokens (`Text`, `LineBreak`, `Ignorable`) and give it two consumers: the engine's stripper, whose behaviour is unchanged, and a new `rendered_lines` the anchor filter splits on. The classification is deliberately lopsided rather than terminal-accurate, and the doc says so instead of claiming fidelity it does not deliver. Only SGR and the string-argument sequences (OSC/DCS/SOS/PM/APC) are `Ignorable`, because neither can move the cursor; `\t` stays intra-line text; everything else is a boundary. That is a one-sided guarantee -- two things drawn on different rows are never scored as one line, while one row may come back split -- and it is the right asymmetry here: merging rows produces a confident wrong cause, splitting one loses a diagnosis the canonical-symptom fallback already covers. A real terminal emulator is not decidable on a byte stream with no screen model (`CSI H` addresses a row that depends on everything before it), and over-splitting needs none of that. After: every separator above scores 0, `\n` and `\r` stay 0, and both the plain and the colourised genuine vLLM OOM stay at 95. Tests: the CR regression becomes a table over the separators rather than one test per byte, plus the SGR case in the other direction, plus a property test over the whole C0/C1 range pinned to `char::is_control` rather than to this module's own classifier. Reverting the boundary set turns three of them red (`nel esc-e`, `U+0000`, `a ESC E (NEL)-separated llama.cpp OOM ...`); making SGR a boundary instead turns the colour test red on `SGR inside a line must not change the verdict`. A confidently wrong cause is user-observable, so it is also pinned as `@id:diagnose-vllm-oom-not-attributed-across-rendered-lines` (diagnose-21), which fails on the pre-fix boundary set and passes after. EAI-8058 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 195 ++++++--- crates/rocm-core/src/lib.rs | 1 + crates/rocm-core/src/terminal.rs | 392 ++++++++++++++++++ engines/vllm/src/lib.rs | 151 +------ tests/e2e-cucumber/features/diagnose.feature | 15 + .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 38 ++ 6 files changed, 588 insertions(+), 204 deletions(-) create mode 100644 crates/rocm-core/src/terminal.rs diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index c3eca5bea..a7cc04d99 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -1503,22 +1503,48 @@ pub const VLLM_OOM_CANONICAL_SYMPTOM: &str = "vllm: torch.OutOfMemoryError: HIP /// returns no diagnosis. The engine's emitted `vllm: ` keeps the /// anchor and the error on one line by construction. /// -/// The split is on `\r` as well as `\n`, not `str::lines()`. `lines()` treats a -/// lone `\r` as ordinary text, and a bare CR is what every progress bar in this -/// ecosystem emits to repaint its line (tqdm, pip, huggingface). A raw terminal -/// capture pasted into `rocm diagnose --symptom '...'` therefore collapses into -/// one giant "line", the anchor matches anywhere in it, and every keyword in the -/// paste scores -- which is exactly the whole-paste scoring this function exists -/// to prevent, reinstated by one byte -/// (`a_bare_carriage_return_is_a_line_boundary_not_scoreable_text`). Splitting on -/// both makes the line boundary the one the terminal actually renders. `\r\n` -/// yields an empty segment, which carries no anchor and is dropped. +/// The split is [`crate::terminal::rendered_lines`], not `str::lines()` and not +/// a set of separator characters. `lines()` splits on `\n` and `\r\n` only, so a +/// lone `\r` was ordinary text to it — and a bare CR is what every progress bar +/// in this ecosystem emits to repaint its line (tqdm, pip, huggingface). A raw +/// terminal capture pasted into `rocm diagnose --symptom '...'` therefore +/// collapsed into one giant "line", the anchor matched somewhere in it, and +/// every keyword in the paste scored — exactly the whole-paste scoring this +/// function exists to prevent, reinstated by one byte +/// (`any_line_advance_is_a_boundary_not_scoreable_text`). +/// +/// Adding `\r` to the split fixed that one byte and nothing else, because a +/// two-character allowlist is not the boundary a terminal renders. Every other +/// way of starting a new line — `ESC E` (`NEL`), `ESC D` (`IND`), `CSI n B` +/// (`CUD`), which are terminfo's `nel`/`ind`/`cud1` and ordinary output from +/// curses- and `rich`-style progress UIs — collapsed the paste just as +/// completely, as did `\x0b`, `\x0c`, `\u{85}`, `U+2028`/`U+2029` and any stray +/// control byte. Each one scored a `llama.cpp` OOM at 95 against a `vllm` anchor +/// from a different rendered line, past a `HIGH_CONFIDENCE` threshold of 75, and +/// cited the other engine's tokens as its evidence. +/// +/// What this is *not* is a terminal emulator; see +/// [`crate::terminal::rendered_lines`] for the contract. It is a one-sided +/// guarantee — two things drawn on different rows are never scored as one line, +/// while one row may come back split — and the residual is exactly that second +/// half: a capture whose cursor addressing this does not model loses a diagnosis +/// rather than inventing one, and the canonical-symptom fallback already covers +/// that. The misattribution is the failure mode with a user-visible cost, and it +/// is the one closed here. +/// +/// Sharing that walk with the vLLM engine's sanitizer is also what keeps the +/// `SGR` exception right. A colourised logger emits `ESC [ 31 m` *inside* a +/// line, so a splitter that treats every escape as a boundary cuts a genuine +/// `vllm: ... torch.OutOfMemoryError` between its anchor and its error token and +/// silently stops diagnosing it (measured: 95 → 0). +/// +/// `\r\n` yields an empty segment, which carries no anchor and is dropped. fn vllm_anchored_lines(symptom: &str) -> String { let Ok(anchor) = Regex::new(VLLM_ANCHOR_PATTERN) else { return String::new(); }; - symptom - .split(['\n', '\r']) + crate::terminal::rendered_lines(symptom) + .into_iter() .filter(|line| anchor.is_match(&line.to_lowercase())) .collect::>() .join("\n") @@ -4152,54 +4178,115 @@ mod tests { assert!(oom.score >= HIGH_CONFIDENCE, "score was {}", oom.score); } + /// The `fix-16-vllm-oom` score for `symptom`, or 0 when it did not match. + fn vllm_oom_score(symptom: &str) -> i32 { + diagnose(&linux_base(), symptom) + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .map_or(0, |d| d.score) + } + #[test] - fn a_bare_carriage_return_is_a_line_boundary_not_scoreable_text() { - // Regression: the anchored-lines filter used `str::lines()`, which - // splits only on `\n` and `\r\n` -- a lone `\r` is ordinary text to it. - // Every progress bar in this ecosystem (tqdm, pip, huggingface) - // repaints with a bare CR, and a pasted terminal capture is the - // expected way to use `--symptom`, so the whole capture collapsed into - // one "line": the vLLM anchor matched somewhere in it and another - // framework's OOM elsewhere in it scored at full weight. That is + fn any_line_advance_is_a_boundary_not_scoreable_text() { + // Regression, in two rounds. + // + // The filter first used `str::lines()`, which splits on `\n` and `\r\n` + // only -- a lone `\r` is ordinary text to it. Every progress bar in this + // ecosystem (tqdm, pip, huggingface) repaints with a bare CR, and a + // pasted terminal capture is the expected way to use `--symptom`, so the + // whole capture collapsed into one "line": the vLLM anchor matched + // somewhere in it and another framework's OOM elsewhere in it scored at + // full weight. That is // `only_the_anchored_lines_are_scored_not_the_whole_paste` defeated by - // one byte, and neither existing guard saw it because both use - // `\n`-only fixtures. - let capture = "Downloading shards: 10%\rvllm serve starting up\r\ - Downloading shards: 90%\r\ - llama.cpp: torch.OutOfMemoryError: CUDA out of memory. \ - Tried to allocate 7.21 GiB.\n"; - let report = diagnose(&linux_base(), capture); - assert!( - report.matched.iter().all(|d| d.id != "fix-16-vllm-oom"), - "a CR-separated llama.cpp OOM must not be attributed to vLLM: {:?}", - report.matched - ); + // one byte, and neither existing guard saw it because both use `\n`-only + // fixtures. + // + // Adding `\r` to the split fixed that byte and left the hole open for + // every other one. Measured on the round that did so: each separator + // below put the anchor and a llama.cpp OOM on genuinely separate + // rendered lines, and each scored 95 -- past a HIGH_CONFIDENCE of 75, + // citing the llama.cpp line's tokens as the evidence for a vLLM verdict. + // Hence a table rather than one test per byte: the property is "a line + // advance is a line advance", and enumerating it is what stops the next + // separator from reopening this. + for (label, sep) in [ + ("LF", "\n"), + ("CR", "\r"), + ("CRLF", "\r\n"), + ("ESC E (NEL)", "\u{1b}E"), + ("ESC D (IND)", "\u{1b}D"), + ("CSI 1 B (CUD)", "\u{1b}[1B"), + ("CSI H (CUP)", "\u{1b}[2;1H"), + ("VT", "\u{b}"), + ("FF", "\u{c}"), + ("C1 NEL", "\u{85}"), + ("U+2028 LINE SEPARATOR", "\u{2028}"), + ("U+2029 PARAGRAPH SEPARATOR", "\u{2029}"), + ("a stray control byte", "\u{1}"), + ] { + let capture = format!( + "Downloading shards: 10%{sep}vllm serve starting up{sep}\ + Downloading shards: 90%{sep}\ + llama.cpp: torch.OutOfMemoryError: CUDA out of memory. \ + Tried to allocate 7.21 GiB.{sep}" + ); + assert_eq!( + vllm_oom_score(&capture), + 0, + "a {label}-separated llama.cpp OOM must not be attributed to vLLM" + ); + } - // The same content with `\n` in place of `\r` is the already-covered - // shape; pinning both together is what makes the CR case a boundary - // question rather than a scoring question. - let newline_form = capture.replace('\r', "\n"); - assert!( - diagnose(&linux_base(), &newline_form) - .matched - .iter() - .all(|d| d.id != "fix-16-vllm-oom"), - "the `\\n` form of the same capture must not match either" + // The control, per separator: a capture whose *anchored* segment carries + // the OOM must still match, so the boundary set did not simply blind the + // checker to every capture that contains one. + for (label, sep) in [ + ("CR", "\r"), + ("ESC E (NEL)", "\u{1b}E"), + ("VT", "\u{b}"), + ("a stray control byte", "\u{1}"), + ] { + let capture = format!( + "Downloading shards: 90%{sep}\ + vllm: torch.OutOfMemoryError: HIP out of memory{sep}" + ); + let score = vllm_oom_score(&capture); + assert!( + score >= MIN_SCORE_FOR_MATCH, + "an anchored OOM segment must still match across {label}: score was {score}" + ); + } + } + + #[test] + fn colour_codes_inside_a_line_do_not_split_a_genuine_vllm_oom() { + // The other half of the boundary question, and the reason this is a + // grammar walk and not `char::is_control()`. A colourised logger emits + // SGR *inside* a line -- level name in colour, message plain -- so a + // splitter that treats every escape, or every control character, as a + // boundary cuts a real `vllm: ... torch.OutOfMemoryError` between its + // anchor and its error token. Measured on the one-line + // `is_control()`-split candidate: this case went from 95 to 0, trading + // the false positive above for a silent miss on the most ordinary real + // input there is. + let plain = "vllm: torch.OutOfMemoryError: HIP out of memory"; + let coloured = + "\u{1b}[31mvllm: \u{1b}[1mtorch.OutOfMemoryError: HIP out of memory\u{1b}[0m"; + assert_eq!( + vllm_oom_score(coloured), + vllm_oom_score(plain), + "SGR inside a line must not change the verdict" ); + let score = vllm_oom_score(coloured); + assert!(score >= HIGH_CONFIDENCE, "score was {score}"); - // The control: a CR-separated capture whose *anchored* segment carries - // the OOM must still match, so splitting on CR did not simply blind the - // checker to carriage-returned input. - let anchored = diagnose( - &linux_base(), - "Downloading shards: 90%\rvllm: torch.OutOfMemoryError: HIP out of memory\r", + // And the same line after a progress-bar repaint, which is how it + // actually arrives: the CR is a boundary, the SGR around it is not. + assert!( + vllm_oom_score(&format!("Loading: 90%\r{coloured}\n")) >= HIGH_CONFIDENCE, + "a colourised OOM after a CR repaint must still match" ); - let oom = anchored - .matched - .iter() - .find(|d| d.id == "fix-16-vllm-oom") - .expect("an anchored OOM segment must still match across CRs"); - assert!(oom.score >= MIN_SCORE_FOR_MATCH, "score was {}", oom.score); } #[test] diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index e5b09bfb7..2d844e92b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -34,6 +34,7 @@ pub mod fix; pub mod openmpi; pub mod proc_lifecycle; pub mod runtime; +pub mod terminal; pub mod uv; pub use diagnose::{ DiagnoseReport, Diagnosis, Fix, VLLM_OOM_CANONICAL_SYMPTOM, diagnose as run_diagnose, diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs new file mode 100644 index 000000000..f214c2799 --- /dev/null +++ b/crates/rocm-core/src/terminal.rs @@ -0,0 +1,392 @@ +//! One ECMA-48 walk over untrusted terminal output, shared by the two callers +//! that need it. +//! +//! Both callers are handling the same input — a raw terminal capture from a +//! subprocess, pasted or quoted back at the user — and both need the same +//! question answered: *what would a terminal have rendered here?* They use the +//! answer differently. +//! +//! * The vLLM engine strips the sequences out, so a colourised or bell-bearing +//! log line cannot repaint the user's terminal from inside rocm-cli's own +//! error message ([`strip_terminal_control_sequences`]). +//! * The vLLM-OOM diagnostic needs the *line boundaries*, so that two things the +//! terminal drew on separate rows are never scored as one line +//! ([`rendered_lines`]). +//! +//! A second, independent scan for the second caller would have been a second +//! grammar to get wrong, and the first one took three rounds to get right. So +//! the grammar lives here once, as [`next_token`], and each caller interprets +//! the classified tokens it yields. + +/// What one step of the ECMA-48 walk found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Token { + /// A character the terminal draws at the cursor. + Text(char), + /// Something that can move the cursor off the current row, so text either + /// side of it was rendered on different lines. + LineBreak, + /// Something that draws nothing and cannot move the cursor, so it neither + /// contributes text nor breaks a line. + Ignorable, +} + +/// Whether `c` is a character that carries no glyph of its own but changes how +/// the text around it renders. +/// +/// `char::is_control` is Unicode category `Cc` only, so it misses the `Cf` +/// format characters — and `U+202E RIGHT-TO-LEFT OVERRIDE` in a log line +/// reverses how the rest of the printed `rocm diagnose --symptom '...'` command +/// renders in the user's terminal. That cannot escape the single quotes (no +/// ASCII `'` is involved), so it is display spoofing rather than shell +/// injection, but it is the same "untrusted subprocess output must not control +/// what the terminal shows" concern the escape stripping exists for, and the +/// answer has to be the same. +/// +/// The ranges are the `Cf` category, enumerated rather than pulled from a +/// Unicode-tables dependency: the set is small, stable, and a new dependency for +/// one predicate is a worse trade. Over-inclusion is safe here — the only cost +/// of rejecting a character is falling back to the canonical symptom, which is +/// guaranteed to report a cause. +#[must_use] +pub fn is_control_or_format(c: char) -> bool { + c.is_control() + || matches!(c, + '\u{00ad}' + | '\u{0600}'..='\u{0605}' + | '\u{061c}' + | '\u{06dd}' + | '\u{070f}' + | '\u{0890}'..='\u{0891}' + | '\u{08e2}' + | '\u{180e}' + | '\u{200b}'..='\u{200f}' + | '\u{202a}'..='\u{202e}' + | '\u{2060}'..='\u{2064}' + | '\u{2066}'..='\u{206f}' + | '\u{feff}' + | '\u{fff9}'..='\u{fffb}' + | '\u{110bd}' + | '\u{110cd}' + | '\u{13430}'..='\u{1343f}' + | '\u{1bca0}'..='\u{1bca3}' + | '\u{1d173}'..='\u{1d17a}' + | '\u{e0001}' + | '\u{e0020}'..='\u{e007f}') +} + +/// Consumes one glyph, control character or escape sequence and says which of +/// the three it was. +/// +/// The sequence grammar is ECMA-48's, followed exactly, and the reason is that +/// the input is a *killed* process's output: truncated and interleaved escapes +/// are the normal case here, not an exotic one. A scan that just ran to the +/// next byte in `0x40..=0x7E` mis-handled every one of them — it swallowed the +/// following sequence's introducer (`ESC [ 1 ; 2 ESC [ 0 m Killed` lost +/// `Killed`'s first six characters), ran straight past a multi-byte scalar +/// (which can never be in that ASCII range) and ate everything up to the next +/// byte that happened to land in it, emitted the body of an OSC title +/// sequence as text, and left a stray `ESC ESC` unrecoverable. +/// +/// Consuming only what the grammar allows and then *stopping without consuming* +/// the offending byte bounds the damage to the malformed sequence itself: the +/// text after it survives, and a following well-formed sequence is still +/// recognised because its `ESC` is left for the main loop to re-read. +/// +/// What this deliberately does not do is second-guess a well-formed sequence. +/// `ESC [ SP K` is a valid CSI (`SP` is an intermediate, `K` the final byte), so +/// it is consumed whole even though the `K` may have been the first letter of a +/// truncated process's "Killed" — a real terminal consumes it too, and the +/// contract here is "render what the terminal would have rendered", which is +/// the only rule that stays decidable on a byte stream with no framing. +/// +/// # Why [`Token::LineBreak`] is the default for anything unrecognised +/// +/// The classification is deliberately lopsided, because the two ways of being +/// wrong do not cost the same. Calling a real boundary [`Token::Ignorable`] +/// merges two rendered lines into one, which is what lets a `llama.cpp` OOM be +/// scored against a `vllm` anchor from a different line — a confident, wrong +/// answer. Calling a non-boundary a [`Token::LineBreak`] splits one rendered +/// line in two, which at worst loses a diagnosis the tool would otherwise have +/// made — a silent miss, and one the canonical-symptom fallback already covers. +/// +/// So only two classes are [`Token::Ignorable`], and only because neither can +/// move the cursor at all: +/// +/// * `SGR` (`CSI ... m`) — colour and attributes, and the overwhelmingly common +/// thing to find *inside* a real log line. This is the one exception that has +/// to be right: treating it as a boundary would split +/// `ESC[31m vllm: ESC[1m torch.OutOfMemoryError` between its anchor and its +/// error token and lose a genuine OOM. +/// * The string-argument sequences (`OSC`, `DCS`, `SOS`, `PM`, `APC`) — their +/// bodies are arbitrary text, like a window title, that is never drawn at the +/// cursor. +/// +/// Everything else with an `ESC` in it — `ESC E` (`NEL`), `ESC D` (`IND`), +/// `CSI n B` (`CUD`), and equally the sequences that do *not* move the cursor, +/// such as a charset designator — is a boundary. Enumerating the cursor-moving +/// finals exactly would be a terminal emulator, and a terminal emulator is not +/// decidable on a byte stream with no screen model: `CSI H` addresses a row that +/// depends on what came before, and overwriting text with `\r` makes "which line +/// was this on" a question about the whole capture. Over-splitting needs none of +/// that and cannot produce the misattribution. +fn next_token(chars: &mut std::iter::Peekable>) -> Option { + let c = chars.next()?; + if c != '\u{1b}' { + return Some(classify_char(c)); + } + match chars.peek() { + // CSI — what a colourised logger emits. + Some('[') => { + chars.next(); + Some(if skip_csi_body(chars) { + Token::Ignorable + } else { + Token::LineBreak + }) + } + // The string-argument sequences: OSC, DCS, SOS, PM, APC. Their bodies + // are arbitrary text (a window title, say) and must not be emitted as if + // the process had printed it, but they draw nothing at the cursor. + Some(']' | 'P' | 'X' | '^' | '_') => { + chars.next(); + skip_string_sequence_body(chars); + Some(Token::Ignorable) + } + // `ESC` with nothing after it, or `ESC ESC`: drop just this one and let + // the loop re-read the next as a fresh introducer. A bare `ESC` draws + // nothing and moves nothing on its own. + None | Some('\u{1b}') => Some(Token::Ignorable), + // Any other escape: optional intermediates, then one final byte. This is + // where `ESC E` (NEL) and `ESC D` (IND) live. + Some(_) => { + skip_simple_escape_body(chars); + Some(Token::LineBreak) + } + } +} + +/// Classifies a character that is not an escape introducer. +/// +/// `\t` is [`Token::Text`] rather than a boundary: it moves the cursor along the +/// row it is already on, so it is legitimate intra-line whitespace. Every other +/// `Cc` control is a boundary — `\n` and `\r` obviously, but equally `\x0b` +/// (`VT`) and `\x0c` (`FF`), which advance a line, `\u{85}` (`NEL`), and the +/// bytes that do nothing at all. The last group is the lopsidedness above: a +/// stray `\x01` between two rendered lines is far likelier to be a mangled line +/// advance than intra-line text, and guessing "boundary" costs a missed +/// diagnosis where guessing "text" costs a wrong one. +/// +/// `U+2028`/`U+2029` are `Zl`/`Zp` rather than `Cc`, so `char::is_control` does +/// not cover them, but Unicode defines both as mandatory line breaks. +fn classify_char(c: char) -> Token { + if c == '\t' { + return Token::Text(c); + } + if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { + return Token::LineBreak; + } + if is_control_or_format(c) { + return Token::Ignorable; + } + Token::Text(c) +} + +/// Consumes a CSI body — parameter bytes, then intermediate bytes, then one +/// final byte — from just after the `ESC [`. Stops without consuming anything +/// that does not belong, leaving it to be treated as text. +/// +/// Returns whether the sequence was a well-formed `SGR` (final byte `m`), the +/// only CSI that cannot move the cursor. A CSI truncated before its final byte +/// returns `false`: an unterminated sequence is exactly the mangled output the +/// boundary question matters for, so it takes the conservative branch. +fn skip_csi_body(chars: &mut std::iter::Peekable>) -> bool { + while chars + .next_if(|c| ('\u{30}'..='\u{3f}').contains(c)) + .is_some() + {} + while chars + .next_if(|c| ('\u{20}'..='\u{2f}').contains(c)) + .is_some() + {} + chars.next_if(|c| ('\u{40}'..='\u{7e}').contains(c)) == Some('m') +} + +/// Consumes a string-argument sequence's body and terminator (`BEL`, or `ST` = +/// `ESC \`) from just after the introducer. A body truncated by anything else — +/// including a bare `ESC` starting the next sequence — ends the scan with that +/// byte left in place. +fn skip_string_sequence_body(chars: &mut std::iter::Peekable>) { + while let Some(&c) = chars.peek() { + if c == '\u{7}' { + chars.next(); + return; + } + if c == '\u{1b}' { + let mut lookahead = chars.clone(); + lookahead.next(); + if lookahead.peek() == Some(&'\\') { + chars.next(); + chars.next(); + } + return; + } + chars.next(); + } +} + +/// Consumes a non-CSI escape's body — optional intermediates, then one final +/// byte — from just after the `ESC`. +fn skip_simple_escape_body(chars: &mut std::iter::Peekable>) { + while chars + .next_if(|c| ('\u{20}'..='\u{2f}').contains(c)) + .is_some() + {} + chars.next_if(|c| ('\u{30}'..='\u{7e}').contains(c)); +} + +/// Removes ANSI escape sequences and any remaining control or format character. +/// +/// A colourised or bell-bearing log line therefore cannot repaint the user's +/// terminal, or reorder how the message renders, from inside rocm-cli's own +/// error message. +/// +/// Line boundaries are dropped rather than preserved: every caller here has +/// already split its input into lines and wants a single-line result. +#[must_use] +pub fn strip_terminal_control_sequences(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut chars = line.chars().peekable(); + while let Some(token) = next_token(&mut chars) { + if let Token::Text(c) = token { + // `\t` is intra-line text to the boundary question but still a + // control byte in a message this crate is about to print, and the + // callers' contract is that nothing control-ish survives. + if !is_control_or_format(c) { + out.push(c); + } + } + } + out +} + +/// Splits `text` into the lines a terminal would have rendered it as, with the +/// escape sequences and non-drawing characters removed from each. +/// +/// This is a deliberate *over*-approximation, not a terminal emulator: it +/// guarantees that two pieces of text the terminal drew on different rows never +/// end up in the same returned line, and makes no promise in the other +/// direction — one rendered line may come back split. See [`next_token`] for why +/// that asymmetry is the right one, and for exactly which sequences are treated +/// as boundaries. +/// +/// Empty segments are returned as-is; callers that filter their lines drop them +/// for free. +#[must_use] +pub fn rendered_lines(text: &str) -> Vec { + let mut lines = vec![String::new()]; + let mut chars = text.chars().peekable(); + while let Some(token) = next_token(&mut chars) { + match token { + Token::Text(c) => { + if let Some(last) = lines.last_mut() { + last.push(c); + } + } + Token::LineBreak => lines.push(String::new()), + Token::Ignorable => {} + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_line_advance_of_any_form_is_a_boundary_and_sgr_is_not() { + // The table is the reviewer's reproduction on PR #251, generalised: the + // fix that landed before it split on `['\n', '\r']`, an allowlist of two + // characters behind a doc comment claiming the boundary was "the one the + // terminal actually renders". Every other way a terminal is told to + // start a new line collapsed the paste back into one line. + // + // `sgr` is the control in the other direction, and the reason this is a + // grammar walk rather than the one-line `char::is_control` split that + // was suggested: splitting at `ESC` fixes the rows below but cuts a + // genuine colourised log line between its anchor and its error token. + let cases: &[(&str, &str, &[&str])] = &[ + ("lf", "a\nb", &["a", "b"]), + ("cr", "a\rb", &["a", "b"]), + ("crlf", "a\r\nb", &["a", "", "b"]), + ("nel esc-e", "a\u{1b}Eb", &["a", "b"]), + ("ind esc-d", "a\u{1b}Db", &["a", "b"]), + ("ri esc-m", "a\u{1b}Mb", &["a", "b"]), + ("cud csi-1-b", "a\u{1b}[1Bb", &["a", "b"]), + ("cuu csi-a", "a\u{1b}[Ab", &["a", "b"]), + ("cup csi-h", "a\u{1b}[2;3Hb", &["a", "b"]), + ("el csi-k", "a\u{1b}[Kb", &["a", "b"]), + ("vt", "a\u{b}b", &["a", "b"]), + ("ff", "a\u{c}b", &["a", "b"]), + ("c1 nel", "a\u{85}b", &["a", "b"]), + ("u2028 line sep", "a\u{2028}b", &["a", "b"]), + ("u2029 para sep", "a\u{2029}b", &["a", "b"]), + ("stray c0", "a\u{1}b", &["a", "b"]), + ("bel", "a\u{7}b", &["a", "b"]), + // `ESC [ 1 ; 2 b` is *complete*, not truncated -- `b` is a valid CSI + // final byte (`REP`), so the grammar consumes it and it is not text. + // A genuinely truncated CSI is one the next `ESC` interrupts; it + // must not swallow that introducer, so the SGR after it still reads + // as an SGR and only the truncated sequence is a boundary. + ("csi with a letter final", "a\u{1b}[1;2b", &["a", ""]), + ("truncated csi", "a\u{1b}[1;2\u{1b}[0mb", &["a", "b"]), + // Not boundaries. + ("sgr", "a\u{1b}[31mb", &["ab"]), + ("sgr reset", "a\u{1b}[0mb", &["ab"]), + ("sgr multi param", "a\u{1b}[1;38;5;9mb", &["ab"]), + ("osc title", "a\u{1b}]0;title\u{7}b", &["ab"]), + ("dcs", "a\u{1b}Pq\u{1b}\\b", &["ab"]), + ("tab is intra-line", "a\tb", &["a\tb"]), + ("zero-width format char", "a\u{200b}b", &["ab"]), + ("bidi override", "a\u{202e}b", &["ab"]), + ("bare esc", "a\u{1b}", &["a"]), + ]; + for (label, input, want) in cases { + let got = rendered_lines(input); + assert_eq!( + &got.iter().map(String::as_str).collect::>(), + want, + "{label}" + ); + } + } + + #[test] + fn the_split_never_merges_two_rendered_lines() { + // The property the over-approximation actually promises, over every + // non-drawing scalar that could sit between two lines of a capture: the + // whole C0 range, DEL, the whole C1 range, and the Unicode separators + // and format characters. The printable range is excluded because those + // are text by definition and merging is what they are for. + // + // The expectation is derived from `char::is_control` (std) and the + // enumerated `Cf` set rather than from this module's own classifier, so + // a change to `classify_char` cannot move the goalposts with it. + for c in (0u32..=0x1f) + .chain(0x7f..=0x9f) + .filter_map(char::from_u32) + .chain(['\u{2028}', '\u{2029}', '\u{200b}', '\u{feff}', '\u{202e}']) + { + let joined = format!("left{c}right"); + let lines = rendered_lines(&joined); + let merged = lines + .iter() + .any(|l| l.contains("left") && l.contains("right")); + // Exactly the characters a terminal draws on the same row may merge: + // tab, and the non-drawing format characters that move no cursor. + let may_merge = c == '\t' || (!c.is_control() && is_control_or_format(c)); + assert_eq!(merged, may_merge, "U+{:04X}", c as u32); + } + } +} diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index ed921dd61..00d0651e8 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -4,6 +4,7 @@ use anyhow::{Context, Result, anyhow, bail}; use clap::{Parser, Subcommand}; +use rocm_core::terminal::{is_control_or_format, strip_terminal_control_sequences}; use rocm_core::{ AppPaths, DEFAULT_LOCAL_PORT, DependencyViolation, check_dependencies, ensure_uv_binary, format_http_base_url, openai_models_endpoint_has_model, require_nonempty, split_local_version, @@ -2311,156 +2312,6 @@ fn quotable_in_single_quotes(symptom: &str) -> bool { !symptom.contains('\'') && !symptom.chars().any(is_control_or_format) } -/// Removes ANSI escape sequences and any remaining control or format -/// characters, so a colourised or bell-bearing log line cannot repaint the -/// user's terminal, or reorder how the message renders, from inside rocm-cli's -/// own error message. -/// -/// The sequence grammar is ECMA-48's, followed exactly, and the reason is that -/// the input is a *killed* process's output: truncated and interleaved escapes -/// are the normal case here, not an exotic one. A scan that just ran to the -/// next byte in `0x40..=0x7E` mis-handled every one of them — it swallowed the -/// following sequence's introducer (`ESC [ 1 ; 2 ESC [ 0 m Killed` lost -/// `Killed`'s first six characters), ran straight past a multi-byte scalar -/// (which can never be in that ASCII range) and ate everything up to the next -/// byte that happened to land in it, emitted the body of an OSC title -/// sequence as text, and left a stray `ESC ESC` unrecoverable. -/// -/// Consuming only what the grammar allows and then *stopping without consuming* -/// the offending byte bounds the damage to the malformed sequence itself: the -/// text after it survives, and a following well-formed sequence is still -/// recognised because its `ESC` is left for the main loop to re-read. -/// -/// What this deliberately does not do is second-guess a well-formed sequence. -/// `ESC [ SP K` is a valid CSI (`SP` is an intermediate, `K` the final byte), so -/// it is consumed whole even though the `K` may have been the first letter of a -/// truncated process's "Killed" — a real terminal consumes it too, and the -/// contract here is "render what the terminal would have rendered", which is -/// the only rule that stays decidable on a byte stream with no framing. -fn strip_terminal_control_sequences(line: &str) -> String { - let mut out = String::with_capacity(line.len()); - let mut chars = line.chars().peekable(); - while let Some(c) = chars.next() { - if c != '\u{1b}' { - if !is_control_or_format(c) { - out.push(c); - } - continue; - } - match chars.peek() { - // CSI — what a colourised logger emits. - Some('[') => { - chars.next(); - skip_csi_body(&mut chars); - } - // The string-argument sequences: OSC, DCS, SOS, PM, APC. Their - // bodies are arbitrary text (a window title, say) and must not be - // emitted as if the process had printed it. - Some(']' | 'P' | 'X' | '^' | '_') => { - chars.next(); - skip_string_sequence_body(&mut chars); - } - // `ESC` with nothing after it, or `ESC ESC`: drop just this one and - // let the loop re-read the next as a fresh introducer. - None | Some('\u{1b}') => {} - // Any other escape: optional intermediates, then one final byte. - Some(_) => skip_simple_escape_body(&mut chars), - } - } - out -} - -/// Consumes a CSI body — parameter bytes, then intermediate bytes, then one -/// final byte — from just after the `ESC [`. Stops without consuming anything -/// that does not belong, leaving it to be treated as text. -fn skip_csi_body(chars: &mut std::iter::Peekable>) { - while chars - .next_if(|c| ('\u{30}'..='\u{3f}').contains(c)) - .is_some() - {} - while chars - .next_if(|c| ('\u{20}'..='\u{2f}').contains(c)) - .is_some() - {} - chars.next_if(|c| ('\u{40}'..='\u{7e}').contains(c)); -} - -/// Consumes a string-argument sequence's body and terminator (`BEL`, or `ST` = -/// `ESC \`) from just after the introducer. A body truncated by anything else — -/// including a bare `ESC` starting the next sequence — ends the scan with that -/// byte left in place. -fn skip_string_sequence_body(chars: &mut std::iter::Peekable>) { - while let Some(&c) = chars.peek() { - if c == '\u{7}' { - chars.next(); - return; - } - if c == '\u{1b}' { - let mut lookahead = chars.clone(); - lookahead.next(); - if lookahead.peek() == Some(&'\\') { - chars.next(); - chars.next(); - } - return; - } - chars.next(); - } -} - -/// Consumes a non-CSI escape's body — optional intermediates, then one final -/// byte — from just after the `ESC`. -fn skip_simple_escape_body(chars: &mut std::iter::Peekable>) { - while chars - .next_if(|c| ('\u{20}'..='\u{2f}').contains(c)) - .is_some() - {} - chars.next_if(|c| ('\u{30}'..='\u{7e}').contains(c)); -} - -/// Whether `c` is a character that carries no glyph of its own but changes how -/// the text around it renders. -/// -/// `char::is_control` is Unicode category `Cc` only, so it misses the `Cf` -/// format characters — and `U+202E RIGHT-TO-LEFT OVERRIDE` in a log line -/// reverses how the rest of the printed `rocm diagnose --symptom '...'` command -/// renders in the user's terminal. That cannot escape the single quotes (no -/// ASCII `'` is involved), so it is display spoofing rather than shell -/// injection, but it is the same "untrusted subprocess output must not control -/// what the terminal shows" concern the escape stripping above exists for, and -/// the answer has to be the same. -/// -/// The ranges are the `Cf` category, enumerated rather than pulled from a -/// Unicode-tables dependency: the set is small, stable, and a new dependency in -/// an engine binary for one predicate is a worse trade. Over-inclusion is safe -/// here — the only cost of rejecting a character is falling back to the -/// canonical symptom, which is guaranteed to report a cause. -fn is_control_or_format(c: char) -> bool { - c.is_control() - || matches!(c, - '\u{00ad}' - | '\u{0600}'..='\u{0605}' - | '\u{061c}' - | '\u{06dd}' - | '\u{070f}' - | '\u{0890}'..='\u{0891}' - | '\u{08e2}' - | '\u{180e}' - | '\u{200b}'..='\u{200f}' - | '\u{202a}'..='\u{202e}' - | '\u{2060}'..='\u{2064}' - | '\u{2066}'..='\u{206f}' - | '\u{feff}' - | '\u{fff9}'..='\u{fffb}' - | '\u{110bd}' - | '\u{110cd}' - | '\u{13430}'..='\u{1343f}' - | '\u{1bca0}'..='\u{1bca3}' - | '\u{1d173}'..='\u{1d17a}' - | '\u{e0001}' - | '\u{e0020}'..='\u{e007f}') -} - /// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a /// HIP allocation failure (e.g. `torch.OutOfMemoryError: HIP out of memory`). fn log_tail_shows_oom(log_tail: &str) -> bool { diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index fd9bb8636..0ce64bb9c 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -283,3 +283,18 @@ Feature: Diagnosing failures and listing fixes # disagree while each looks right alone. Every Linux lane runs this step; the # WSL one is where the two lists can differ. And the CLI can act on the fix the diagnosis named + + # `--symptom` is where a user pastes a raw terminal capture, and a terminal + # capture is full of line advances that are not `\n`: a progress bar repaints + # with `\r`, and curses- and `rich`-style progress UIs move down with `ESC E`, + # `ESC D` or `CSI n B` (terminfo's `nel`, `ind` and `cud1`). If those are not + # line boundaries the whole paste collapses into a single line, a `vllm` + # mention anywhere in it anchors another engine's OOM, and the user is told + # with high confidence that vLLM ran out of memory -- quoting the other + # engine's error text back as the evidence for it. A confidently wrong cause is + # worse than no cause, so this is pinned at the level the user sees it. + @id:diagnose-vllm-oom-not-attributed-across-rendered-lines @requires-os:linux + Scenario: diagnose-21 - Another engine's OOM is not blamed on a vLLM mention elsewhere in the paste + Given a user who pasted a capture naming vLLM and another engine's OOM on separate rendered lines + When the user asks the CLI to diagnose that symptom in machine-readable form + Then no vLLM startup OOM is reported diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 9c9612c14..bed1ee668 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -191,6 +191,24 @@ async fn user_hit_vllm_oom(world: &mut E2eWorld) { ); } +/// The separator is `ESC E` (`NEL`), deliberately, not `\n` or `\r`. Both of +/// those were already split on by the time this scenario was written; `NEL` is +/// what a `rich`/curses progress UI emits to advance a line, and it is what the +/// two-character allowlist did not cover. The anchor (`vllm`) and the OOM are on +/// different rendered lines, so no part of this paste is a vLLM fault. +#[given( + "a user who pasted a capture naming vLLM and another engine's OOM on separate rendered lines" +)] +async fn user_pasted_a_capture_with_a_foreign_oom(world: &mut E2eWorld) { + world.model_name = Some( + "Downloading shards: 10%\u{1b}Evllm serve starting up\u{1b}E\ + Downloading shards: 90%\u{1b}E\ + llama.cpp: torch.OutOfMemoryError: CUDA out of memory. \ + Tried to allocate 7.21 GiB.\u{1b}E" + .to_string(), + ); +} + #[given("a user who hit the vLLM engine-startup import failure")] async fn user_hit_engine_import_failure(world: &mut E2eWorld) { world.model_name = Some(ENGINE_IMPORT_SYMPTOM.to_string()); @@ -516,6 +534,26 @@ fn find_vllm_oom<'a>(report: &'a serde_json::Value, output: &str) -> &'a serde_j .unwrap_or_else(|| panic!("expected fix-16-vllm-oom in diagnosis:\n{output}")) } +#[then("no vLLM startup OOM is reported")] +async fn assert_no_vllm_oom_reported(world: &mut E2eWorld) { + assert_eq!( + world.cli_rc, + Some(0), + "diagnose should exit 0 (it is a query)" + ); + let (report, output) = parsed_diagnosis(world); + let matched = report + .get("matched") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("report must carry a `matched` array:\n{output}")); + assert!( + !matched.iter().any(|diagnosis| { + diagnosis.get("id").and_then(serde_json::Value::as_str) == Some("fix-16-vllm-oom") + }), + "another engine's OOM must not be attributed to vLLM:\n{output}" + ); +} + #[then("the diagnosis identifies the vLLM startup OOM")] async fn assert_diagnosis_identifies_vllm_oom(world: &mut E2eWorld) { assert_eq!(world.cli_rc, Some(0), "diagnose should exit 0"); From e3e3ae2e54926cea009d0f577e86c3183db7dedf Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 11:55:22 +0300 Subject: [PATCH 27/39] chore(core): add the licence header the new terminal module was missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fcf99156` created `crates/rocm-core/src/terminal.rs` without the SPDX header every other source file carries, so `License header check (hawkeye)` failed — its `licenserc.toml` includes `**/*.rs`. Header only; no code change. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/terminal.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index f214c2799..e42dd235f 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -1,3 +1,7 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + //! One ECMA-48 walk over untrusted terminal output, shared by the two callers //! that need it. //! From f34e4580d3e4d2a33d31f709a0091369f42212f5 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 14:06:51 +0300 Subject: [PATCH 28/39] docs(core): scope the consumption rule and name the one stripper change Two prose defects in the module the previous commit added, both of which a reviewer would rely on. `next_token`'s doc asserted that "the contract here is 'render what the terminal would have rendered', which is the only rule that stays decidable on a byte stream with no framing". That was true of the stripper the paragraph came from, where the only question was how much of a well-formed sequence to consume. It is false of `next_token`, which also decides line boundaries and deliberately over-splits -- twenty-eight lines further down the same comment says a terminal emulator "is not decidable on a byte stream with no screen model", and the section between explains the classification is lopsided rather than terminal-accurate. Left standing, that sentence is a documented licence to "fix" the over-splitting toward terminal accuracy, which reopens the row-merging misattribution the previous commit closed. It is now scoped to the consumption decision it is actually about: a well-formed sequence is consumed whole because a real terminal consumes it too, and which token it is then labelled with is the separate, deliberately non-terminal-like question below. The same commit said the engine stripper's "behaviour is unchanged" by the move. It is unchanged except for U+2028/U+2029. The engine-local stripper classified non-escape characters with `is_control_or_format` alone, which is false for both (neither is `Cc`, neither is in the enumerated `Cf` set), so both survived into the stripped message; `classify_char` puts them in the boundary set, so they are now dropped. Measured on this head: the old code returns "a\u{2028}b" for input "a\u{2028}b", the new one returns "ab". The widening is the better behaviour -- a mandatory Unicode line break is exactly what this stripper exists to keep untrusted output from injecting -- but nothing stated it and no test pinned either scalar, so a silent drift back would have gone unnoticed. `classify_char` now says so, and the vLLM stripper table gains both cases: reverting the pair out of the boundary set turns `the_stripper_follows_the_escape_grammar_not_just_ the_colour_case` red on "a line separator must not survive into the message". Also from the same review, three prose corrections with no behaviour change: `rendered_lines`'s lead sentence now states the over-approximation up front instead of stating the strong property and retracting it a paragraph later; the string-argument arm documents the one crafted-input hole in the never-merge guarantee (a properly terminated OSC body containing a line feed is swallowed whole, so `"vllm ESC ] 0 ; ti \n tle BEL llama.cpp OOM"` comes back as one line, where common terminals abort the control string and draw two rows); and the `fix-16` catalog test's comment cites the cross-check that actually exists, `diagnosis_remediation_matches_the_fix_catalog_for_shared_fix_ids`, rather than a name that is in neither the tree nor its history. Last, `effective_gpu_count`'s doc claimed the VRAM-row fallback applies when the amd-smi count is absent, while the code also falls through when it is present and zero -- a host amd-smi cannot enumerate but sysfs can. The doc now names both triggers and the second one gains the test case it never had; dropping the `count > 0` filter on `detected` turns it red. EAI-8058 Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 18 +++++++++++--- crates/rocm-core/src/diagnose.rs | 4 +-- crates/rocm-core/src/terminal.rs | 42 ++++++++++++++++++++++++++------ engines/vllm/src/lib.rs | 20 +++++++++++++++ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 3c5b9284e..d405d26d1 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -19025,12 +19025,13 @@ fn validate_pinned_gpu_index( } /// The GPU count to rank `--gpu auto` candidates over. Prefers the `amd-smi` -/// device count; when `amd-smi` is unavailable (`detected` is `None`) but the -/// DRM sysfs VRAM fallback still produced rows, its length stands in so -/// auto-selection can still consider those devices. Used only to decide whether +/// device count, but only when that count is usable — `None` (amd-smi +/// unavailable) and `Some(0)` (amd-smi ran and saw no devices) both fall through +/// to the DRM sysfs VRAM fallback's row count, so auto-selection can still +/// consider devices amd-smi did not enumerate. Used only to decide whether /// `--gpu auto` has anything at all to rank, never as a `--gpu ` /// validation bound — see [`validate_pinned_gpu_index`]. `None` when neither -/// source is available. +/// source yields a non-zero count. fn effective_gpu_count(detected: Option, vram: Option<&[GpuVramUsage]>) -> Option { detected .filter(|&count| count > 0) @@ -26893,9 +26894,18 @@ install therock"; assert_eq!(effective_gpu_count(Some(4), Some(&usage)), Some(4)); // amd-smi unavailable (`None`): the sysfs VRAM fallback row count stands in. assert_eq!(effective_gpu_count(None, Some(&usage)), Some(2)); + // The second, previously untested fallback trigger: amd-smi ran and + // reported zero devices. `Some(0)` must fall through to the VRAM rows + // just as `None` does, or a host amd-smi cannot enumerate but sysfs can + // would have nothing for `--gpu auto` to rank. Dropping the + // `filter(|&count| count > 0)` on `detected` turns this line red. + assert_eq!(effective_gpu_count(Some(0), Some(&usage)), Some(2)); // Neither source available. assert_eq!(effective_gpu_count(None, None), None); assert_eq!(effective_gpu_count(None, Some(&[])), None); + // Both present but both empty: still nothing to rank. + assert_eq!(effective_gpu_count(Some(0), Some(&[])), None); + assert_eq!(effective_gpu_count(Some(0), None), None); } #[test] diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index a7cc04d99..882fdf98e 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -2950,8 +2950,8 @@ mod tests { // Same two-copies-of-one-plan risk as `fix-17-torch-dlpack`, and it had // already opened: the executable lines agreed, so the cross-check that // runs over every matched finding - // (`the_reported_fix_agrees_with_the_catalog_recipe`) saw nothing -- - // it filters `#` lines by construction -- while the comments explaining + // (`diagnosis_remediation_matches_the_fix_catalog_for_shared_fix_ids`) + // saw nothing -- it filters `#` lines by construction -- while the comments explaining // *which of the two faults each step addresses* had drifted apart. That // prose is the whole point of this entry: the two faults need opposite // responses, so a user meeting the `diagnose` copy and a user meeting diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index e42dd235f..b39858b9c 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -100,9 +100,11 @@ pub fn is_control_or_format(c: char) -> bool { /// What this deliberately does not do is second-guess a well-formed sequence. /// `ESC [ SP K` is a valid CSI (`SP` is an intermediate, `K` the final byte), so /// it is consumed whole even though the `K` may have been the first letter of a -/// truncated process's "Killed" — a real terminal consumes it too, and the -/// contract here is "render what the terminal would have rendered", which is -/// the only rule that stays decidable on a byte stream with no framing. +/// truncated process's "Killed": a well-formed sequence is consumed whole +/// because a real terminal consumes it too. That rule settles *how much* of the +/// byte stream one step eats, and only that — which [`Token`] the consumed +/// sequence is then labelled with is a separate question, answered below, and +/// answered deliberately unlike a terminal. /// /// # Why [`Token::LineBreak`] is the default for anything unrecognised /// @@ -152,6 +154,18 @@ fn next_token(chars: &mut std::iter::Peekable>) -> Option { chars.next(); skip_string_sequence_body(chars); @@ -183,6 +197,17 @@ fn next_token(chars: &mut std::iter::Peekable>) -> Option Token { if c == '\t' { return Token::Text(c); @@ -274,12 +299,13 @@ pub fn strip_terminal_control_sequences(line: &str) -> String { out } -/// Splits `text` into the lines a terminal would have rendered it as, with the -/// escape sequences and non-drawing characters removed from each. +/// Splits `text` into segments no coarser than the lines a terminal would have +/// rendered it as, with the escape sequences and non-drawing characters removed +/// from each. /// -/// This is a deliberate *over*-approximation, not a terminal emulator: it -/// guarantees that two pieces of text the terminal drew on different rows never -/// end up in the same returned line, and makes no promise in the other +/// That is deliberately an *over*-approximation rather than a terminal emulator: +/// it guarantees that two pieces of text the terminal drew on different rows +/// never end up in the same returned segment, and makes no promise in the other /// direction — one rendered line may come back split. See [`next_token`] for why /// that asymmetry is the right one, and for exactly which sequences are treated /// as boundaries. diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 00d0651e8..a7d8374d7 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -3040,6 +3040,26 @@ mod tests { "RuntimeError: HIP out of memory", "a bidi override must not survive into the message", ), + // U+2028/U+2029 are `Zl`/`Zp`, not `Cc`, and are in neither the + // enumerated `Cf` set, so the engine-local stripper this one + // replaced let both through verbatim -- it classified every + // non-escape character with `is_control_or_format` alone. Moving the + // walk into `rocm-core::terminal` widened that one case on purpose: + // both are mandatory Unicode line breaks, so a terminal draws the + // rest of the message on the next row, which is exactly what this + // stripper exists to stop untrusted output doing. These two cases + // are the only thing pinning it -- drop the pair from + // `classify_char`'s boundary set and both go red. + ( + "RuntimeError: HIP\u{2028}out of memory", + "RuntimeError: HIPout of memory", + "a line separator must not survive into the message", + ), + ( + "RuntimeError: HIP\u{2029}out of memory", + "RuntimeError: HIPout of memory", + "a paragraph separator must not survive into the message", + ), // The text-only control: nothing is removed from a clean line. ( "RuntimeError: HIP out of memory", From 4782444fd488d5f0ecf25848ec0b289e2e0727b6 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 15:04:47 +0300 Subject: [PATCH 29/39] docs(core): qualify the never-merge guarantee where a consumer reads it `rendered_lines` promised, unqualified, that "two pieces of text the terminal drew on different rows never end up in the same returned segment". That is the invariant the module exists for -- it is what stops a `llama.cpp` OOM being scored against a `vllm` anchor -- and it is not what the code holds. The counterexample was already written down in the same commit, 140 lines above, but in a body comment inside the private `next_token`, so it reached no reader of the generated documentation, and the public doc's pointer to it (`See [next_token]`) was a link to a private item that rustdoc will not resolve. `cargo doc -p rocm-core` warned on exactly that line. This is the third finding of the same class on this module, so the fix is structural rather than a reworded sentence. The exception now has one authoritative statement, in the module documentation, which is public and therefore rendered: a line break inside a well-formed string-argument body (`OSC`, `DCS`, `SOS`, `PM`, `APC`) is consumed with that body, with the worked example spelled out. `rendered_lines`'s own doc carries the qualifier inline -- so the only absolute a consumer can read is true -- and links to the module docs rather than to a private item. The `next_token` arm that creates the exception no longer restates it; it points at the module paragraph and names what has to change with it. The behaviour is unchanged. Following the grammar rather than guessing where an unterminated body ends is the right call: the alternative reintroduces the leak of OSC bodies as text. `a_line_break_inside_a_well_formed_string_body_is_consumed_with_it` pins the worked example so prose and behaviour move together; flipping its expectation to the two-segment reading fails with `left: ["vllmllama.cpp OOM"]`. It also pins that the same break outside a string body still splits, and that an unterminated body still stops at the following `ESC`, so the hole cannot widen unnoticed. Also, both non-blocking wording items and one more instance of the same dangling-link defect the PR introduced: - `the_split_never_merges_two_rendered_lines` overstated what the module guarantees; its body already scoped itself to single non-drawing scalars, so it is renamed to `a_lone_non_drawing_scalar_never_merges_two_rendered_lines`. - the citation in `the_oom_plan_matches_the_catalog_copy` called `diagnosis_remediation_matches_the_fix_catalog_for_shared_fix_ids` "the cross-check that runs over every matched finding"; it iterates a fixed two-element array of hand-built diagnoses, so it is now "the cross-check over the fixes shared with the catalog". - `DiagnoseReport::out_of_scope`'s doc linked `[catalog_covers]`, also a private item, also warned on by `cargo doc`, and also added by this branch. It now names the rule in prose instead. `cargo doc -p rocm-core --no-deps` goes from 6 warnings to 3; the remaining three are pre-existing on `main` and are not touched here. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 8 +-- crates/rocm-core/src/terminal.rs | 109 +++++++++++++++++++++++-------- 2 files changed, 87 insertions(+), 30 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 882fdf98e..5845facf4 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -85,8 +85,8 @@ pub struct DiagnoseReport { /// Set when the catalog has no entries at all for the running platform, and /// carrying the routing note that says so. /// - /// Decided purely by platform coverage ([`catalog_covers`]) — never by a - /// score. When this is `Some`, no checker ran, so `matched` is empty and + /// Decided purely by whether the catalog covers the running platform — never + /// by a score. When this is `Some`, no checker ran, so `matched` is empty and /// `has_match` is false; a consumer must not wait for sub-threshold rows /// alongside it. Pinned by /// `an_out_of_scope_report_never_carries_matched_entries`. @@ -2948,8 +2948,8 @@ mod tests { #[test] fn the_oom_plan_matches_the_catalog_copy() { // Same two-copies-of-one-plan risk as `fix-17-torch-dlpack`, and it had - // already opened: the executable lines agreed, so the cross-check that - // runs over every matched finding + // already opened: the executable lines agreed, so the cross-check over + // the fixes shared with the catalog // (`diagnosis_remediation_matches_the_fix_catalog_for_shared_fix_ids`) // saw nothing -- it filters `#` lines by construction -- while the comments explaining // *which of the two faults each step addresses* had drifted apart. That diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index b39858b9c..b754c228d 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -14,13 +14,40 @@ //! log line cannot repaint the user's terminal from inside rocm-cli's own //! error message ([`strip_terminal_control_sequences`]). //! * The vLLM-OOM diagnostic needs the *line boundaries*, so that two things the -//! terminal drew on separate rows are never scored as one line -//! ([`rendered_lines`]). +//! terminal drew on separate rows are not scored as one line +//! ([`rendered_lines`]) — with the single exception documented below. //! //! A second, independent scan for the second caller would have been a second //! grammar to get wrong, and the first one took three rounds to get right. So -//! the grammar lives here once, as [`next_token`], and each caller interprets -//! the classified tokens it yields. +//! the grammar lives here once, in one private stepping function, and each +//! caller interprets the classified tokens it yields. +//! +//! # The one exception to the never-merge property +//! +//! [`rendered_lines`] is an over-approximation of what a terminal would have +//! drawn: it may split one rendered line in two, and in the other direction it +//! merges two rendered rows in exactly one shape. A line break *inside* a +//! well-formed string-argument sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`) is +//! consumed with that body, because ECMA-48 defines the body as running to its +//! terminator, so the text either side of the sequence joins up: +//! +//! ```text +//! rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}llama.cpp OOM") +//! == ["vllmllama.cpp OOM"] +//! ``` +//! +//! Common terminals abort a control string on an embedded C0 byte and would +//! draw two rows there. Following the grammar is still the right call: the +//! alternative is guessing where an unterminated body ends, which reintroduces +//! the leak of `OSC` bodies as text that this walk exists to stop. Ordinary log +//! output cannot produce the shape either — it needs a well-formed introducer +//! and terminator around the line break — so this is a crafted-input case, not +//! one a killed subprocess falls into. +//! +//! This paragraph is the single authoritative statement of the exception. The +//! code that creates it and the test that pins it both point here rather than +//! restating it, so that a change to the behaviour cannot leave a stale copy of +//! the guarantee behind in a doc a consumer reads. /// What one step of the ECMA-48 walk found. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -126,7 +153,9 @@ pub fn is_control_or_format(c: char) -> bool { /// error token and lose a genuine OOM. /// * The string-argument sequences (`OSC`, `DCS`, `SOS`, `PM`, `APC`) — their /// bodies are arbitrary text, like a window title, that is never drawn at the -/// cursor. +/// cursor. Consuming such a body whole is the one exception to the never-merge +/// property, stated in full in the module documentation because +/// [`rendered_lines`]'s public callers are the ones it binds. /// /// Everything else with an `ESC` in it — `ESC E` (`NEL`), `ESC D` (`IND`), /// `CSI n B` (`CUD`), and equally the sequences that do *not* move the cursor, @@ -155,17 +184,13 @@ fn next_token(chars: &mut std::iter::Peekable>) -> Option { chars.next(); skip_string_sequence_body(chars); @@ -299,16 +324,21 @@ pub fn strip_terminal_control_sequences(line: &str) -> String { out } -/// Splits `text` into segments no coarser than the lines a terminal would have -/// rendered it as, with the escape sequences and non-drawing characters removed -/// from each. +/// Approximates the lines a terminal would have rendered `text` as, with the +/// escape sequences and non-drawing characters removed from each segment. /// -/// That is deliberately an *over*-approximation rather than a terminal emulator: -/// it guarantees that two pieces of text the terminal drew on different rows -/// never end up in the same returned segment, and makes no promise in the other -/// direction — one rendered line may come back split. See [`next_token`] for why -/// that asymmetry is the right one, and for exactly which sequences are treated -/// as boundaries. +/// It is deliberately an *over*-approximation rather than a terminal emulator, +/// and the approximation is one-sided. Nothing at all is promised in the +/// splitting direction: one rendered line may come back split in two. +/// +/// In the merging direction the promise holds with exactly one exception. Two +/// pieces of text the terminal drew on different rows share a returned segment +/// only when the break between them sits inside a well-formed string-argument +/// sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`), whose body ECMA-48 defines as +/// running to its terminator and which is therefore consumed whole. Ordinary log +/// output cannot produce that shape; crafted input can. See [the module +/// documentation](self) for the worked example, for why that exception is not +/// worth closing, and for exactly which sequences are treated as boundaries. /// /// Empty segments are returned as-is; callers that filter their lines drop them /// for free. @@ -393,7 +423,7 @@ mod tests { } #[test] - fn the_split_never_merges_two_rendered_lines() { + fn a_lone_non_drawing_scalar_never_merges_two_rendered_lines() { // The property the over-approximation actually promises, over every // non-drawing scalar that could sit between two lines of a capture: the // whole C0 range, DEL, the whole C1 range, and the Unicode separators @@ -419,4 +449,31 @@ mod tests { assert_eq!(merged, may_merge, "U+{:04X}", c as u32); } } + + #[test] + fn a_line_break_inside_a_well_formed_string_body_is_consumed_with_it() { + // The one exception to the never-merge property, and the worked example + // the module documentation prints. It is pinned here so the prose and + // the behaviour move together: if this expectation ever changes, the + // module paragraph and `rendered_lines`'s own qualifier are wrong and + // have to change in the same commit. The sweep above cannot catch it -- + // it feeds one scalar at a time, and this shape needs a well-formed + // introducer and terminator around the break. + assert_eq!( + rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}llama.cpp OOM"), + ["vllmllama.cpp OOM"], + ); + // The same break outside a string body still splits, so the exception is + // the sequence's framing and not the newline losing its meaning. + assert_eq!( + rendered_lines("vllm\u{1b}]0;title\u{7}\nllama.cpp OOM"), + ["vllm", "llama.cpp OOM"], + ); + // And an *unterminated* body stops at the `ESC` that follows, so the + // hole does not widen to swallow the rest of a capture. + assert_eq!( + rendered_lines("vllm\u{1b}]0;ti\ntle\u{1b}[0mllama.cpp OOM"), + ["vllmllama.cpp OOM"], + ); + } } From a874302cece9f28be930224e154c7d0cfb85a6cd Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 16:40:19 +0300 Subject: [PATCH 30/39] docs(core): drop the terminator from the merge exception; it is not required `rendered_lines`'s doc said two rendered rows share a segment "only when the break between them sits inside a well-formed string-argument sequence ... whose body ECMA-48 defines as running to its terminator", and the module section said the shape "needs a well-formed introducer and terminator around the line break". Both make a terminator necessary for the merge. It is not. `skip_string_sequence_body` ends the body scan on `BEL`, on `ST`, on any other bare `ESC` (left in place), or on end of input, and `next_token` returns `Token::Ignorable` for all of them. The break is swallowed either way. The test added in this PR is itself the counterexample and passes: its input carries no `BEL` and no `ST` anywhere, and the two rows still merge. The private helper's own doc comment already said this, so the public doc was contradicting the private one. State what the code does instead, and add the unterminated input as a second worked example so the prose has to move with it. That also invalidates the reassurance built on top of the terminator being required. An unterminated body followed by more output is reachable from ordinary output by two routes: a process killed part-way through writing an `OSC` title, which is the case this walk exists for, and the interleaved capture itself, since the serve log hands the subprocess's stdout and stderr the same file handle and either stream can cut the other's title. Say so, and say why following the grammar is still right -- guessing where an unterminated body ends reintroduces the leak of `OSC` bodies as text. `vllm_anchored_lines` still carried the original unqualified "two things drawn on different rows are never scored as one line", which is the sentence the module section claims to be the single authoritative statement of. Replace the restatement with a pointer rather than a second qualified copy; a second copy is a stale copy waiting to happen, and this is the call site where a stale one does the damage. Docs, comments and one test rename only; no behaviour change. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 19 ++++--- crates/rocm-core/src/terminal.rs | 89 +++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 38 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 5845facf4..714f0665e 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -1523,14 +1523,17 @@ pub const VLLM_OOM_CANONICAL_SYMPTOM: &str = "vllm: torch.OutOfMemoryError: HIP /// from a different rendered line, past a `HIGH_CONFIDENCE` threshold of 75, and /// cited the other engine's tokens as its evidence. /// -/// What this is *not* is a terminal emulator; see -/// [`crate::terminal::rendered_lines`] for the contract. It is a one-sided -/// guarantee — two things drawn on different rows are never scored as one line, -/// while one row may come back split — and the residual is exactly that second -/// half: a capture whose cursor addressing this does not model loses a diagnosis -/// rather than inventing one, and the canonical-symptom fallback already covers -/// that. The misattribution is the failure mode with a user-visible cost, and it -/// is the one closed here. +/// What this is *not* is a terminal emulator. The contract is one-sided, and it +/// is stated once — in [`crate::terminal::rendered_lines`] and the module +/// section it points at, which also names the single shape in which two +/// rendered rows still come back as one segment. It is deliberately not +/// restated here: a second copy is a stale copy waiting to happen, and this +/// function is where a stale one would do the damage. What matters at this call +/// site is the direction of the error that remains: a capture whose cursor +/// addressing that walk does not model loses a diagnosis rather than inventing +/// one, and the canonical-symptom fallback already covers that. The +/// misattribution is the failure mode with a user-visible cost, and this is +/// what narrows it to the one shape that contract calls out. /// /// Sharing that walk with the vLLM engine's sanitizer is also what keeps the /// `SGR` exception right. A colourised logger emits `ESC [ 31 m` *inside* a diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index b754c228d..32b120fd1 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -26,25 +26,46 @@ //! //! [`rendered_lines`] is an over-approximation of what a terminal would have //! drawn: it may split one rendered line in two, and in the other direction it -//! merges two rendered rows in exactly one shape. A line break *inside* a -//! well-formed string-argument sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`) is -//! consumed with that body, because ECMA-48 defines the body as running to its -//! terminator, so the text either side of the sequence joins up: +//! merges two rendered rows in exactly one shape. A line break *inside* the +//! body of a string-argument sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`) is +//! consumed with that body, so the text either side of the sequence joins up: //! //! ```text //! rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}llama.cpp OOM") //! == ["vllmllama.cpp OOM"] +//! rendered_lines("vllm\u{1b}]0;ti\ntle\u{1b}[0mllama.cpp OOM") +//! == ["vllmllama.cpp OOM"] //! ``` //! +//! A terminator is not what creates the shape, as the second example shows: it +//! contains no `BEL` and no `ST` anywhere, and the two rows still merge. The +//! body scan runs to the next `BEL`, to `ST` (`ESC \`), to an unrelated `ESC` +//! — which it leaves in place for the next step to re-read, so the hole stays +//! one sequence wide — or to the end of the input, and the break is swallowed +//! in every one of those cases, terminated or not. +//! //! Common terminals abort a control string on an embedded C0 byte and would -//! draw two rows there. Following the grammar is still the right call: the -//! alternative is guessing where an unterminated body ends, which reintroduces -//! the leak of `OSC` bodies as text that this walk exists to stop. Ordinary log -//! output cannot produce the shape either — it needs a well-formed introducer -//! and terminator around the line break — so this is a crafted-input case, not -//! one a killed subprocess falls into. +//! draw two rows there, so this is a real divergence rather than a technicality. +//! Nor does it take crafted input to reach, which an earlier wording of this +//! section claimed on the strength of a terminator being required. Two ordinary +//! routes produce it. A process killed part-way through writing an `OSC` title +//! leaves an unterminated body, and killed output is the case this walk is built +//! for. And the capture being read is an interleaved one: the serve log hands +//! the subprocess's stdout and stderr the *same* file handle, so a title written +//! on one stream can be cut by the other stream's next line landing between the +//! introducer and the terminator, with neither writer doing anything unusual. +//! +//! Following the grammar is still the right call. The alternative is guessing +//! where an unterminated body ends, and the guess would have to be made on +//! exactly the truncated, interleaved output that is the normal case here. Guess +//! short and an `OSC` body is emitted as text — the leak this walk exists to +//! stop, and a second, quieter source of the very misattribution the line +//! boundaries are for, since a window title would then be scored as if the +//! process had printed it. The residual is therefore disclosed here rather than +//! papered over: a break inside a string body is the one way two rendered rows +//! come back as one segment. //! -//! This paragraph is the single authoritative statement of the exception. The +//! This section is the single authoritative statement of the exception. The //! code that creates it and the test that pins it both point here rather than //! restating it, so that a change to the behaviour cannot leave a stale copy of //! the guarantee behind in a doc a consumer reads. @@ -185,12 +206,14 @@ fn next_token(chars: &mut std::iter::Peekable>) -> Option { chars.next(); skip_string_sequence_body(chars); @@ -333,12 +356,14 @@ pub fn strip_terminal_control_sequences(line: &str) -> String { /// /// In the merging direction the promise holds with exactly one exception. Two /// pieces of text the terminal drew on different rows share a returned segment -/// only when the break between them sits inside a well-formed string-argument -/// sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`), whose body ECMA-48 defines as -/// running to its terminator and which is therefore consumed whole. Ordinary log -/// output cannot produce that shape; crafted input can. See [the module -/// documentation](self) for the worked example, for why that exception is not -/// worth closing, and for exactly which sequences are treated as boundaries. +/// only when the break between them sits inside the body of a string-argument +/// sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`), which is consumed whole. A +/// terminator is not required for that: the body scan runs to the next `BEL`, +/// to `ST`, to an unrelated `ESC`, or to the end of the input, and swallows the +/// break in all four cases. Ordinary log output can reach that shape — a +/// process killed mid-title is enough. See [the module documentation](self) for +/// the worked examples, for why the exception is not worth closing anyway, and +/// for exactly which sequences are treated as boundaries. /// /// Empty segments are returned as-is; callers that filter their lines drop them /// for free. @@ -451,14 +476,14 @@ mod tests { } #[test] - fn a_line_break_inside_a_well_formed_string_body_is_consumed_with_it() { - // The one exception to the never-merge property, and the worked example - // the module documentation prints. It is pinned here so the prose and - // the behaviour move together: if this expectation ever changes, the - // module paragraph and `rendered_lines`'s own qualifier are wrong and - // have to change in the same commit. The sweep above cannot catch it -- - // it feeds one scalar at a time, and this shape needs a well-formed - // introducer and terminator around the break. + fn a_line_break_inside_a_string_sequence_body_is_consumed_with_it() { + // The one exception to the never-merge property, and the worked examples + // the module documentation prints. They are pinned here so the prose and + // the behaviour move together: if either expectation ever changes, the + // module section and `rendered_lines`'s own qualifier are wrong and + // have to change in the same commit. The sweep above cannot catch this + // -- it feeds one scalar at a time, and this shape needs a + // string-sequence introducer before the break. assert_eq!( rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}llama.cpp OOM"), ["vllmllama.cpp OOM"], @@ -469,8 +494,12 @@ mod tests { rendered_lines("vllm\u{1b}]0;title\u{7}\nllama.cpp OOM"), ["vllm", "llama.cpp OOM"], ); - // And an *unterminated* body stops at the `ESC` that follows, so the - // hole does not widen to swallow the rest of a capture. + // And an *unterminated* body is the same merge over again, not a + // separate containment case: there is no `BEL` and no `ST` anywhere + // below, and the break is still swallowed, so a terminator is not what + // creates the exception. What the following `ESC` bounds is only how far + // the body runs -- it is left in place -- so the hole does not widen to + // swallow the rest of a capture. assert_eq!( rendered_lines("vllm\u{1b}]0;ti\ntle\u{1b}[0mllama.cpp OOM"), ["vllmllama.cpp OOM"], From 909d50308cdd08b185bf53f08e8732af4fe1bc7f Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 18:05:30 +0300 Subject: [PATCH 31/39] docs(core): delete the restated reachability claim, and name the third behaviour `rendered_lines`'s own doc claimed a process killed mid-title was enough to reach the merge shape. It is not. The merge needs drawable text on both sides of the swallowed break landing in one segment, and a kill leaves the body running to the end of the input, where there is no far side to join to: rendered_lines("vllm\u{1b}]0;ti\ntle") == ["vllm"] The `tle` a terminal drew on its second row comes back in no segment at all. That is a third behaviour -- neither the split nor the merge the one-sided framing describes -- and it was undocumented, so the prose reached for it as an example of the merge instead. This is the fifth round on one sentence, and the sentence is always in the same place: `rendered_lines` restating the exception rather than pointing at the module section that owns it. So the restatement goes rather than gets reworded. The qualifier that binds callers stays; the terminator detail and the reachability claim are deleted and the `[the module documentation](self)` pointer now carries the routes, which makes the "single authoritative statement" claim true rather than aspirational. The drop is documented rather than filed as a defect: it is the direct, already-argued consequence of not guessing where an unterminated body ends, and it errs in the losing direction the contract leaves unpromised -- it costs a diagnosis, it cannot invent one. Both one-sided framings are widened to admit it, so neither now claims two directions while a third exists. Also corrected: the interleaved capture reaches `rendered_lines` through a user paste into `rocm diagnose --symptom`, not through the engine's own path, which pre-splits its log tail with `str::lines`; and the contract is not "stated once" in `vllm_anchored_lines`'s doc, which names two places in the same breath. The end-of-input case is now pinned by a test. Nothing covered it, which is why the prose could cite it wrongly twice. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 23 +++++----- crates/rocm-core/src/terminal.rs | 73 ++++++++++++++++++++++---------- 2 files changed, 63 insertions(+), 33 deletions(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 714f0665e..e18bf188f 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -1523,17 +1523,18 @@ pub const VLLM_OOM_CANONICAL_SYMPTOM: &str = "vllm: torch.OutOfMemoryError: HIP /// from a different rendered line, past a `HIGH_CONFIDENCE` threshold of 75, and /// cited the other engine's tokens as its evidence. /// -/// What this is *not* is a terminal emulator. The contract is one-sided, and it -/// is stated once — in [`crate::terminal::rendered_lines`] and the module -/// section it points at, which also names the single shape in which two -/// rendered rows still come back as one segment. It is deliberately not -/// restated here: a second copy is a stale copy waiting to happen, and this -/// function is where a stale one would do the damage. What matters at this call -/// site is the direction of the error that remains: a capture whose cursor -/// addressing that walk does not model loses a diagnosis rather than inventing -/// one, and the canonical-symptom fallback already covers that. The -/// misattribution is the failure mode with a user-visible cost, and this is -/// what narrows it to the one shape that contract calls out. +/// What this is *not* is a terminal emulator. The contract is one-sided, and +/// [`crate::terminal::rendered_lines`] is where to read it — that doc carries +/// the contract itself and points on to the module section holding the detail +/// behind it, including the single shape in which two rendered rows still come +/// back as one segment. It is deliberately not restated here: a second copy is +/// a stale copy waiting to happen, and this function is where a stale one would +/// do the damage. What matters at this call site is the direction of the error +/// that remains: a capture whose cursor addressing that walk does not model +/// loses a diagnosis rather than inventing one, and the canonical-symptom +/// fallback already covers that. The misattribution is the failure mode with a +/// user-visible cost, and this is what narrows it to the one shape that +/// contract calls out. /// /// Sharing that walk with the vLLM engine's sanitizer is also what keeps the /// `SGR` exception right. A colourised logger emits `ESC [ 31 m` *inside* a diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index 32b120fd1..adf7d7e99 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -25,10 +25,11 @@ //! # The one exception to the never-merge property //! //! [`rendered_lines`] is an over-approximation of what a terminal would have -//! drawn: it may split one rendered line in two, and in the other direction it -//! merges two rendered rows in exactly one shape. A line break *inside* the -//! body of a string-argument sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`) is -//! consumed with that body, so the text either side of the sequence joins up: +//! drawn: it may split one rendered line in two, or lose one altogether, and in +//! the other direction it merges two rendered rows in exactly one shape. A line +//! break *inside* the body of a string-argument sequence (`OSC`, `DCS`, `SOS`, +//! `PM`, `APC`) is consumed with that body, so the text either side of the +//! sequence joins up: //! //! ```text //! rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}llama.cpp OOM") @@ -44,16 +45,38 @@ //! one sequence wide — or to the end of the input, and the break is swallowed //! in every one of those cases, terminated or not. //! +//! Swallowing the break is not always a *merge*, though, and the end-of-input +//! case is the one where it is not. A merge needs drawable text on both sides of +//! the break landing in one segment; when the body runs out over the end of the +//! input there is no far side left to join to, and what the terminal drew past +//! the break comes back in no segment at all: +//! +//! ```text +//! rendered_lines("vllm\u{1b}]0;ti\ntle") == ["vllm"] +//! ``` +//! +//! That is a loss, not a misattribution, so it belongs with the splitting +//! direction where nothing is promised: it costs a diagnosis the tool would +//! otherwise have made, and the canonical-symptom fallback covers it. +//! //! Common terminals abort a control string on an embedded C0 byte and would //! draw two rows there, so this is a real divergence rather than a technicality. //! Nor does it take crafted input to reach, which an earlier wording of this -//! section claimed on the strength of a terminator being required. Two ordinary -//! routes produce it. A process killed part-way through writing an `OSC` title -//! leaves an unterminated body, and killed output is the case this walk is built -//! for. And the capture being read is an interleaved one: the serve log hands -//! the subprocess's stdout and stderr the *same* file handle, so a title written -//! on one stream can be cut by the other stream's next line landing between the -//! introducer and the terminator, with neither writer doing anything unusual. +//! section claimed on the strength of a terminator being required. What it takes +//! is an *interleaved* capture: `rocm serve` hands the subprocess's stdout and +//! stderr the *same* file handle, so a title written on one stream can be cut by +//! the other stream's next line landing between the introducer and the +//! terminator, with neither writer doing anything unusual. A process killed +//! part-way through the same title is the other ordinary half of it, leaving a +//! body with no terminator at all — and killed output is the case this walk is +//! built for. Which of the two shapes reads back is then only a question of +//! whether the capture carries on past the body: text after it joins up, and +//! text the body runs out over is lost. +//! +//! Such a capture reaches [`rendered_lines`] whole when a user pastes it into +//! `rocm diagnose --symptom`. The vLLM engine's own path cannot carry an +//! embedded `\n` there: it pre-splits its log tail with `str::lines` and builds +//! its candidate symptom from a single line of it. //! //! Following the grammar is still the right call. The alternative is guessing //! where an unterminated body ends, and the guess would have to be made on @@ -63,7 +86,8 @@ //! boundaries are for, since a window title would then be scored as if the //! process had printed it. The residual is therefore disclosed here rather than //! papered over: a break inside a string body is the one way two rendered rows -//! come back as one segment. +//! come back as one segment, and — where that body runs out over the end of the +//! input — the one way a rendered row comes back in none. //! //! This section is the single authoritative statement of the exception. The //! code that creates it and the test that pins it both point here rather than @@ -208,7 +232,7 @@ fn next_token(chars: &mut std::iter::Peekable>) -> Option String { /// escape sequences and non-drawing characters removed from each segment. /// /// It is deliberately an *over*-approximation rather than a terminal emulator, -/// and the approximation is one-sided. Nothing at all is promised in the -/// splitting direction: one rendered line may come back split in two. +/// and the approximation is one-sided. Nothing at all is promised in the losing +/// direction: one rendered line may come back split in two, and text the +/// terminal drew may come back in no segment at all. /// /// In the merging direction the promise holds with exactly one exception. Two /// pieces of text the terminal drew on different rows share a returned segment /// only when the break between them sits inside the body of a string-argument -/// sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`), which is consumed whole. A -/// terminator is not required for that: the body scan runs to the next `BEL`, -/// to `ST`, to an unrelated `ESC`, or to the end of the input, and swallows the -/// break in all four cases. Ordinary log output can reach that shape — a -/// process killed mid-title is enough. See [the module documentation](self) for -/// the worked examples, for why the exception is not worth closing anyway, and -/// for exactly which sequences are treated as boundaries. +/// sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`), which is consumed whole. See +/// [the module documentation](self) for the worked examples, for which ordinary +/// captures reach that shape, for why the exception is not worth closing +/// anyway, and for exactly which sequences are treated as boundaries. /// /// Empty segments are returned as-is; callers that filter their lines drop them /// for free. @@ -504,5 +526,12 @@ mod tests { rendered_lines("vllm\u{1b}]0;ti\ntle\u{1b}[0mllama.cpp OOM"), ["vllmllama.cpp OOM"], ); + // End of input is the one way of ending the body that does *not* merge: + // there is no far side to join to, so the `tle` a terminal would have + // drawn on its second row comes back in no segment at all. That is the + // losing direction, not the misattributing one, and it is why the + // module section documents a third behaviour rather than two. Pinned + // because prose has twice claimed this shape as a route to the merge. + assert_eq!(rendered_lines("vllm\u{1b}]0;ti\ntle"), ["vllm"]); } } From 3f2c32f3479797de03d2981d8be65a7d3efb41d0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 19:29:50 +0300 Subject: [PATCH 32/39] docs(core): the merge trigger is text following the sequence, not end of input The module section named end of input as the reason a swallowed break loses a row rather than merging. It is not the reason: a properly BEL-terminated body with nothing after it loses the row just the same, and so does one with text after it that sits past a break. What decides is whether drawable text follows the sequence in the same segment, however the body ended. Rather than restate that correctly in each of the three places the section said it, keep one statement where the third behaviour is introduced and delete the other two -- the divergence between copies is what produced this round and the five before it. Pin the two shapes that were not covered, so no future wording can name a narrower trigger than the code has without a test failing. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/terminal.rs | 46 ++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index adf7d7e99..33445a744 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -45,16 +45,21 @@ //! one sequence wide — or to the end of the input, and the break is swallowed //! in every one of those cases, terminated or not. //! -//! Swallowing the break is not always a *merge*, though, and the end-of-input -//! case is the one where it is not. A merge needs drawable text on both sides of -//! the break landing in one segment; when the body runs out over the end of the -//! input there is no far side left to join to, and what the terminal drew past -//! the break comes back in no segment at all: +//! Swallowing the break is not always a *merge*, though. A merge needs drawable +//! text on both sides of the break landing in one segment, so what decides is +//! whether drawable text follows the sequence in that same segment — however the +//! body ended. Where none does there is no far side to join to, and what the +//! terminal drew past the break comes back in no segment at all: //! //! ```text -//! rendered_lines("vllm\u{1b}]0;ti\ntle") == ["vllm"] +//! rendered_lines("vllm\u{1b}]0;ti\ntle") == ["vllm"] +//! rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}") == ["vllm"] +//! rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}\nx") == ["vllm", "x"] //! ``` //! +//! The last two bodies are properly `BEL`-terminated, and the last is not near +//! the end of the input either; the `tle` is lost all the same. +//! //! That is a loss, not a misattribution, so it belongs with the splitting //! direction where nothing is promised: it costs a diagnosis the tool would //! otherwise have made, and the canonical-symptom fallback covers it. @@ -69,9 +74,7 @@ //! terminator, with neither writer doing anything unusual. A process killed //! part-way through the same title is the other ordinary half of it, leaving a //! body with no terminator at all — and killed output is the case this walk is -//! built for. Which of the two shapes reads back is then only a question of -//! whether the capture carries on past the body: text after it joins up, and -//! text the body runs out over is lost. +//! built for. //! //! Such a capture reaches [`rendered_lines`] whole when a user pastes it into //! `rocm diagnose --symptom`. The vLLM engine's own path cannot carry an @@ -86,8 +89,8 @@ //! boundaries are for, since a window title would then be scored as if the //! process had printed it. The residual is therefore disclosed here rather than //! papered over: a break inside a string body is the one way two rendered rows -//! come back as one segment, and — where that body runs out over the end of the -//! input — the one way a rendered row comes back in none. +//! come back as one segment, and — where no drawable text follows the sequence — +//! the one way a rendered row comes back in none. //! //! This section is the single authoritative statement of the exception. The //! code that creates it and the test that pins it both point here rather than @@ -526,12 +529,21 @@ mod tests { rendered_lines("vllm\u{1b}]0;ti\ntle\u{1b}[0mllama.cpp OOM"), ["vllmllama.cpp OOM"], ); - // End of input is the one way of ending the body that does *not* merge: - // there is no far side to join to, so the `tle` a terminal would have - // drawn on its second row comes back in no segment at all. That is the - // losing direction, not the misattributing one, and it is why the - // module section documents a third behaviour rather than two. Pinned - // because prose has twice claimed this shape as a route to the merge. + // Swallowing the break merges only where drawable text follows the + // sequence in the same segment. Where none does, the `tle` a terminal + // would have drawn on its second row comes back in no segment at all -- + // and how the body ended is not what decides that. The second case below + // is properly `BEL`-terminated and the third is terminated with more of + // the capture still to come, only behind a break; both lose the row just + // as the unterminated first one does. That is the losing direction, not + // the misattributing one, and it is why the module section documents a + // third behaviour rather than two. Pinned because prose has three times + // now named a narrower trigger than the code has. assert_eq!(rendered_lines("vllm\u{1b}]0;ti\ntle"), ["vllm"]); + assert_eq!(rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}"), ["vllm"]); + assert_eq!( + rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}\nx"), + ["vllm", "x"] + ); } } From 5a470ab1205098b96caeda40356a69e904cb4cd5 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 19:33:13 +0300 Subject: [PATCH 33/39] docs(core): qualify the closing summary's loss condition to match the rule The section states the trigger precisely as drawable text following the sequence *in that same segment*, then the closing summary restated it without the qualifier. rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}\nx") is the case that splits them: text does follow the sequence, the row is lost anyway, because the text is past a break and so in another segment. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/terminal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index 33445a744..fe281654e 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -89,8 +89,8 @@ //! boundaries are for, since a window title would then be scored as if the //! process had printed it. The residual is therefore disclosed here rather than //! papered over: a break inside a string body is the one way two rendered rows -//! come back as one segment, and — where no drawable text follows the sequence — -//! the one way a rendered row comes back in none. +//! come back as one segment, and — where no drawable text follows the sequence +//! in that same segment — the one way a rendered row comes back in none. //! //! This section is the single authoritative statement of the exception. The //! code that creates it and the test that pins it both point here rather than From d35629ffc4fbdc5a4020b7fa8a1af63278f01a52 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 17 Sep 2026 21:30:34 +0300 Subject: [PATCH 34/39] docs(core): retitle the residual-behaviour section and stop restating its rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section documents two residual behaviours, not one exception, and its own claim about the file's structure was false: the code arm and the test do restate the rule, accurately and usefully, as local commentary. Say instead that this section is the normative statement. Three further corrections, all the same recurring defect of the same rule living in more than one place and drifting: - a break inside a string body can lose more than one rendered row (`rendered_lines("vllm\u{1b}]0;ti\ntle\nmore") == ["vllm"]`), so "one or more"; - the "what decides" clause named only the far side, but a row is lost with no near side too (`rendered_lines("\u{1b}]0;ti\ntle\u{7}x") == ["x"]`) — dropped in favour of the both-sides sentence before it; - the closing summary repeated the same far-side-only qualifier, so it is dropped there as well. Net two lines shorter. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/terminal.rs | 50 +++++++++++++++----------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index fe281654e..d59a5bec9 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -22,14 +22,14 @@ //! the grammar lives here once, in one private stepping function, and each //! caller interprets the classified tokens it yields. //! -//! # The one exception to the never-merge property +//! # Line breaks inside a string body: merging and losing rows //! //! [`rendered_lines`] is an over-approximation of what a terminal would have -//! drawn: it may split one rendered line in two, or lose one altogether, and in -//! the other direction it merges two rendered rows in exactly one shape. A line -//! break *inside* the body of a string-argument sequence (`OSC`, `DCS`, `SOS`, -//! `PM`, `APC`) is consumed with that body, so the text either side of the -//! sequence joins up: +//! drawn: it may split one rendered line in two, or lose one or more +//! altogether, and in the other direction it merges two rendered rows in +//! exactly one shape. A line break *inside* the body of a string-argument +//! sequence (`OSC`, `DCS`, `SOS`, `PM`, `APC`) is consumed with that body, so +//! the text either side of the sequence joins up: //! //! ```text //! rendered_lines("vllm\u{1b}]0;ti\ntle\u{7}llama.cpp OOM") @@ -46,10 +46,9 @@ //! in every one of those cases, terminated or not. //! //! Swallowing the break is not always a *merge*, though. A merge needs drawable -//! text on both sides of the break landing in one segment, so what decides is -//! whether drawable text follows the sequence in that same segment — however the -//! body ended. Where none does there is no far side to join to, and what the -//! terminal drew past the break comes back in no segment at all: +//! text on both sides of the break landing in one segment — however the body +//! ended. Where either side is missing, what the terminal drew past the break +//! comes back in no segment at all: //! //! ```text //! rendered_lines("vllm\u{1b}]0;ti\ntle") == ["vllm"] @@ -65,16 +64,16 @@ //! otherwise have made, and the canonical-symptom fallback covers it. //! //! Common terminals abort a control string on an embedded C0 byte and would -//! draw two rows there, so this is a real divergence rather than a technicality. -//! Nor does it take crafted input to reach, which an earlier wording of this -//! section claimed on the strength of a terminator being required. What it takes -//! is an *interleaved* capture: `rocm serve` hands the subprocess's stdout and -//! stderr the *same* file handle, so a title written on one stream can be cut by -//! the other stream's next line landing between the introducer and the -//! terminator, with neither writer doing anything unusual. A process killed -//! part-way through the same title is the other ordinary half of it, leaving a -//! body with no terminator at all — and killed output is the case this walk is -//! built for. +//! draw both sides of the break, so this is a real divergence rather than a +//! technicality. Nor does it take crafted input to reach, which an earlier +//! wording of this section claimed on the strength of a terminator being +//! required. What it takes is an *interleaved* capture: `rocm serve` hands the +//! subprocess's stdout and stderr the *same* file handle, so a title written on +//! one stream can be cut by the other stream's next line landing between the +//! introducer and the terminator, with neither writer doing anything unusual. +//! A process killed part-way through the same title is the other ordinary half +//! of it, leaving a body with no terminator at all — and killed output is the +//! case this walk is built for. //! //! Such a capture reaches [`rendered_lines`] whole when a user pastes it into //! `rocm diagnose --symptom`. The vLLM engine's own path cannot carry an @@ -89,13 +88,12 @@ //! boundaries are for, since a window title would then be scored as if the //! process had printed it. The residual is therefore disclosed here rather than //! papered over: a break inside a string body is the one way two rendered rows -//! come back as one segment, and — where no drawable text follows the sequence -//! in that same segment — the one way a rendered row comes back in none. +//! come back as one segment, and the one way a rendered row comes back in none. //! -//! This section is the single authoritative statement of the exception. The -//! code that creates it and the test that pins it both point here rather than -//! restating it, so that a change to the behaviour cannot leave a stale copy of -//! the guarantee behind in a doc a consumer reads. +//! This section states the rule normatively; the comments on the code that +//! creates it and on the test that pins it are local commentary, not the +//! statement of record. A change to the behaviour has to land here, and any +//! comment that has drifted from this section has to be brought back into line. /// What one step of the ECMA-48 walk found. #[derive(Debug, Clone, Copy, PartialEq, Eq)] From 73b8bfb89ab0d79dfe1717d6ecec242f0e58558e Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 24 Sep 2026 10:20:18 +0300 Subject: [PATCH 35/39] fix(serve): keep a detected GPU with no VRAM row as an auto-select candidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `select_auto_gpu_index` builds its candidate ordinals from the VRAM rows alone whenever telemetry is present. That was introduced so a visibility mask reporting non-contiguous ordinals is scanned by its real indices instead of a synthetic `0..count` range, which is right — but a short row set is not always a mask. `parse_gpu_vram_usage` drops any device entry missing `/mem_usage/used_vram/value` or `/mem_usage/total_vram/value`, so a device the lighter `list` enumeration counts can simply have no row. With device 1 absent from the rows and device 0 pinned by a managed service, `reported` was `[0]`, the busy filter emptied the candidate list, all three passes iterated nothing and the terminal fallback returned device 0 — `serve` pinned to an already-occupied GPU, then failing on VRAM, which is the fault this selection exists to reduce. Union the detected `0..count` range back in only when `visible` is known. The existing retain then validates every ordinal against the mask, so no unconfirmed index is ever synthesised: under a `[2, 3]` mask with `count == 2` the synthetic `[0, 1]` is dropped wholesale and the rows still stand alone. Without `visible` there is no second source to confirm an ordinal against, so the rows remain authoritative. Pins the case with a test: a two-GPU count, one row, an all-busy reported set, which the suite never exercised (every all-busy case passed `detected: None`). Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 70 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 78110320e..700489cc0 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -20912,11 +20912,34 @@ fn select_auto_gpu_index( // is the space the selection is exported through, so the retained ordinals // are directly selectable. When the host is unprobeable (`visible` is // `None`) the set is left unrestricted (mask-unaware, as before). + // + // A short row set is not always a mask, though: `parse_gpu_vram_usage` + // drops any device entry missing `/mem_usage/used_vram/value` or + // `/mem_usage/total_vram/value`, so a device the lighter `list` enumeration + // counts can simply have no row. Taking rows-only there would shrink the + // candidate set for a reason that has nothing to do with visibility, and if + // every *reported* device is busy the terminal fallback would hand back a + // busy GPU while an idle, merely untelemetried one went unconsidered — + // exactly the "serve pinned to an occupied GPU" fault this selection exists + // to avoid. So when `visible` is known, union the detected `0..count` range + // back in: the retain below validates every ordinal against that mask, so + // nothing unconfirmed survives (under a `[2, 3]` mask with `count == 2` the + // synthetic `[0, 1]` is dropped wholesale and the rows still stand alone). + // Without `visible` there is no second source to confirm an ordinal + // against, so the rows remain authoritative and no index is invented. let mut reported: Vec = match vram { - Some(rows) => rows.iter().map(|row| row.index).collect(), + Some(rows) => { + let mut indices: Vec = rows.iter().map(|row| row.index).collect(); + if visible.is_some() { + indices.extend(0..count as u32); + } + indices + } None => (0..count as u32).collect(), }; reported.sort_unstable(); + // The union above can repeat an ordinal that both sources name. + reported.dedup(); if let Some(visible) = visible { reported.retain(|index| visible.contains(index)); } @@ -30162,6 +30185,51 @@ install therock"; ); } + #[test] + fn auto_selection_considers_a_detected_gpu_that_reported_no_vram_row() { + // A short row set is not always a visibility mask. `parse_gpu_vram_usage` + // drops any device entry missing its `used_vram`/`total_vram` pointers, so + // here `list` counts two GPUs, both are visible, but only GPU 0 produced a + // row — and GPU 0 is pinned by a managed service. Driving candidates from + // the rows alone leaves `[0]`, the busy filter empties it, every pass + // iterates nothing and the terminal fallback hands back GPU 0: `serve` + // pinned to an already-occupied GPU, the exact failure this selection + // exists to prevent. The untelemetried GPU 1 is confirmed by both the + // count and the visible set, so it must be a candidate and must win. + assert_eq!( + select_auto_gpu_index( + Some(2), + Some(&[0, 1]), + &[0], + Some(&[vram(0, 182_000, 192_000)]) + ), + vec![1], + "a detected, visible GPU with no VRAM row must be preferred over a busy reported one" + ); + // The same shape without telemetry for the busy device being conclusive: + // GPU 1 is still the only non-busy ordinal either source confirms. + assert_eq!( + select_auto_gpu_index( + Some(2), + Some(&[0, 1]), + &[0], + Some(&[vram(0, 1_000, 192_000)]) + ), + vec![1], + "an idle-looking but service-pinned GPU 0 must still lose to the free GPU 1" + ); + // The masking behaviour this rows-only construction was introduced for is + // untouched: under a `[2, 3]` mask the synthetic `0..count` range is + // dropped wholesale by the visible retain, so no unconfirmed ordinal is + // ever synthesised and the all-busy fallback still names a reported GPU. + let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(Some(2), Some(&[2, 3]), &[2, 3], Some(&masked)), + vec![2], + "a masked host must not gain candidates 0/1 from the detected count" + ); + } + #[test] fn validate_pinned_gpu_index_rejects_out_of_range() { // Index equal to or beyond the detected count is rejected. From 2aa6434f7b3a404a0cecf554515e5c9cbc377dbe Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 24 Sep 2026 14:46:34 +0300 Subject: [PATCH 36/39] fix(serve): discriminate a masked row set by its indices, not by `visible` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit unioned the detected `0..count` range back into the auto-select candidates only `if visible.is_some()`, so the fix reached only hosts whose visibility set could be probed. `probe_usable_amd_gpu_indices` returns `None` unconditionally for `#[cfg(not(target_os = "linux"))]`, and `serve` threads that straight into `visible` through `select_gpu_indices_under_launch_lock`. So `visible` is always `None` on Windows — a supported platform — and on any Linux host whose KFD topology and DRM cards are both unreadable. With `count == 2`, one VRAM row for GPU 0 and GPU 0 pinned by a managed service, those hosts still returned the busy GPU 0 where the base branch returned the idle GPU 1: the exact fault this selection exists to reduce. `visible` was never the right discriminator. The rows are a re-indexed or sparse set precisely when some row index falls at or above `count` (`[2, 3]` against `count == 2`), which is the shape the rows-only construction was introduced for; below the count they are a subset of a dense range the count already asserts exists, and unioning `0..count` there is no stronger an assumption than the telemetry-less arm already makes. Require `detected` to be `Some(n > 0)` as well, so the count comes from the `list` enumeration rather than from `rows.len()` restating the rows' own size — `parse_gpu_vram_usage` only falls back to the array position when an entry has no `gpu` field, so a payload naming `"gpu": 0` twice yields duplicate ordinals that would otherwise conjure a GPU 1. The visible retain still runs on top whenever a mask is known, so nothing synthesised survives where the mask contradicts it. Pins both halves of the condition: forcing either to `true` turns an assertion of the new test red, and the masked assertion now holds with `visible: None` so it pins the row-index rule itself rather than the retain. Also replaces the Pass 2 rationale, which the union falsified. `usage_for` is no longer total over `candidate_indices`, but no row guard is needed for a different reason: the comparator keys are `Option` and `None` sorts below every `Some`, so a rowless ordinal can win only when no candidate has a row, and then the index tie-break returns the lowest — exactly what Pass 3 would return. Re-adding the guard leaves all 771 unit tests green, so the guard is behaviourally inert and stays out. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 105 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 15 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 700489cc0..6a01e71d1 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -20921,16 +20921,34 @@ fn select_auto_gpu_index( // every *reported* device is busy the terminal fallback would hand back a // busy GPU while an idle, merely untelemetried one went unconsidered — // exactly the "serve pinned to an occupied GPU" fault this selection exists - // to avoid. So when `visible` is known, union the detected `0..count` range - // back in: the retain below validates every ordinal against that mask, so - // nothing unconfirmed survives (under a `[2, 3]` mask with `count == 2` the - // synthetic `[0, 1]` is dropped wholesale and the rows still stand alone). - // Without `visible` there is no second source to confirm an ordinal - // against, so the rows remain authoritative and no index is invented. + // to avoid. So the `0..count` range is unioned back in — but only where it + // is evidence rather than invention, which is decided by the rows, not by + // `visible`: + // + // * `detected` must be `Some(n > 0)`, so `count` is the `list` device + // count — an independent source asserting those ordinals exist. When it + // is absent `effective_gpu_count` falls back to `rows.len()`, which is + // only the rows restating their own size and confirms nothing they omit. + // * every row index must be below `count`. A row at or above it is the one + // available proof that the ordinal space is re-indexed or sparse rather + // than dense from 0 (`[2, 3]` with `count == 2`), and that is exactly the + // masked shape the rows-only construction was introduced for, so there + // the rows stay authoritative. Below the count the rows are a subset of a + // dense range the count already asserts, and `0..count` is no stronger an + // assumption than the telemetry-less arm below already makes. + // + // Gating on `visible.is_some()` instead would have left the bug standing + // wherever the mask is unknown, which is not a corner: `usable_amd_gpu_indices` + // is `None` unconditionally off Linux, so `visible` is always `None` on + // Windows. The retain below still runs on top whenever a mask *is* known, so + // no synthesised ordinal survives that the mask contradicts. let mut reported: Vec = match vram { Some(rows) => { let mut indices: Vec = rows.iter().map(|row| row.index).collect(); - if visible.is_some() { + let count_is_independently_sourced = detected.is_some_and(|detected| detected > 0); + let rows_are_a_dense_range_subset = + indices.iter().all(|&index| (index as usize) < count); + if count_is_independently_sourced && rows_are_a_dense_range_subset { indices.extend(0..count as u32); } indices @@ -20971,11 +20989,17 @@ fn select_auto_gpu_index( } // Pass 2: the non-busy GPU with the most free VRAM in absolute terms // (not free percentage, which can favor a smaller GPU on heterogeneous - // VRAM systems). No "does this ordinal have a row?" guard is needed: - // inside this branch every candidate came *from* a reported row, so - // `usage_for` is total over `candidate_indices`. It was needed when - // candidates were a synthetic `0..count` range, where an ordinal could - // be selected that telemetry had never reported. + // VRAM systems). `usage_for` is only *partial* over `candidate_indices` — + // the union above puts back ordinals the `list` count confirms but + // telemetry never reported — yet no "does this ordinal have a row?" guard + // is needed, because the comparator already orders them last: the keys are + // `Option`, and `None` sorts below every `Some`, including `Some(0)`. + // So a rowless ordinal can only be the maximum when *no* candidate has a + // row; then every key is `None`, every comparison falls through to the + // index tie-break, and the winner is the lowest candidate index — which is + // precisely what Pass 3 would return from an ascending `reported`. Adding + // the guard back would therefore change no outcome, only skip a pass that + // already agrees. if let Some(&index) = candidate_indices.iter().max_by(|left, right| { let left_free = usage_for(**left).map(|usage| usage.free_mb()); let right_free = usage_for(**right).map(|usage| usage.free_mb()); @@ -30219,9 +30243,11 @@ install therock"; "an idle-looking but service-pinned GPU 0 must still lose to the free GPU 1" ); // The masking behaviour this rows-only construction was introduced for is - // untouched: under a `[2, 3]` mask the synthetic `0..count` range is - // dropped wholesale by the visible retain, so no unconfirmed ordinal is - // ever synthesised and the all-busy fallback still names a reported GPU. + // untouched: rows `[2, 3]` against `count == 2` put a row index at or above + // the count, which is the proof the ordinal space is re-indexed, so the + // union never happens and the rows stand alone. (The visible retain would + // also have dropped `0`/`1` here — this pins the row-index rule itself, so + // the masked case survives on a host where the mask is unknown too.) let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; assert_eq!( select_auto_gpu_index(Some(2), Some(&[2, 3]), &[2, 3], Some(&masked)), @@ -30230,6 +30256,55 @@ install therock"; ); } + #[test] + fn auto_selection_considers_an_untelemetried_gpu_when_the_visible_set_is_unknown() { + // Same shape as the test above, but `visible` is `None`. That is not an + // exotic host: `probe_usable_amd_gpu_indices` returns `None` + // *unconditionally* off Linux, and `serve` threads that straight into + // `visible`, so every Windows run takes this path — as does any Linux host + // whose KFD topology and DRM cards are both unreadable. Gating the + // `0..count` union on `visible.is_some()` therefore left the whole bug + // standing on a supported platform. + // + // Nothing about the mask is needed to justify GPU 1 here: the amd-smi + // `list` count asserts two devices and the rows name only ordinals below + // that count, so the rows are a subset of a dense range, not a re-indexed + // one. GPU 0 is pinned by a managed service, so the untelemetried GPU 1 is + // the only free ordinal and must win over the busy reported fallback. + assert_eq!( + select_auto_gpu_index(Some(2), None, &[0], Some(&[vram(0, 182_000, 192_000)])), + vec![1], + "an unprobeable host must still consider a counted GPU that reported no VRAM row" + ); + // A row index at or above the count is the only evidence available that the + // ordinal space is *not* a dense `0..count`, so there the rows stay + // authoritative even with no mask to retain against — otherwise an + // unprobeable masked host would have `[0, 1]` invented for it. + let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(Some(2), None, &[2, 3], Some(&masked)), + vec![2], + "re-indexed rows must not gain unconfirmed candidates 0/1 from the detected count" + ); + // The count must come from the `list` enumeration to confirm anything. + // With `detected` absent it is `rows.len()`, i.e. the rows restating their + // own size, and `parse_gpu_vram_usage` can repeat an ordinal: it falls back + // to the array position only when an entry has no `gpu` field, so a payload + // naming `"gpu": 0` twice yields two index-0 rows. That makes `rows.len()` + // 2 with every index below it, and dropping the `detected` half of the + // condition would invent a GPU 1 nothing ever enumerated. + assert_eq!( + select_auto_gpu_index( + None, + None, + &[0], + Some(&[vram(0, 182_000, 192_000), vram(0, 182_000, 192_000)]) + ), + vec![0], + "a row-derived count must not synthesise an ordinal no source reported" + ); + } + #[test] fn validate_pinned_gpu_index_rejects_out_of_range() { // Index equal to or beyond the detected count is rejected. From ce7a2340ffb1144e7d7d98c42d2f778b05a0c4ad Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 24 Sep 2026 16:24:47 +0300 Subject: [PATCH 37/39] fix(vllm): reject a mandatory line break from the quoted symptom too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `quotable_in_single_quotes` gates what is embedded in the `rocm diagnose --symptom '...'` command the OOM hint prints for the user to paste, and it tested each character with `is_control_or_format` alone. That predicate is `char::is_control` plus the enumerated `Cf` set, and `U+2028 LINE SEPARATOR` / `U+2029 PARAGRAPH SEPARATOR` are `Zl`/`Zp`, so both passed the gate. The value reaching it is raw: `candidate` is built from `raw_line`, not from the stripped `symptom_line` beside it, and `str::lines` does not split on `U+2028`, so the scalar survives inside a selected line. A tail line like `RuntimeError: HIP out of memory...` cleared the OOM scan, cleared this gate, scored above threshold and was printed verbatim inside the quotes — the bytes stay inside them, but the rest of the command renders on the next row. That is the same class of display control the guard's own documentation says it closes. `classify_char` already widened past `is_control_or_format` for exactly this pair, so the module stated one contract and the gate implemented a narrower one. Rather than repeat the `matches!` at the gate, export `is_control_or_line_separator` from `rocm_core::terminal` and use it at both sites: `is_control_or_format`'s own documentation advertises itself as the answer to this concern, so the next call site would reach for the narrower test again. One predicate removes the trap, and the rationale moves onto it instead of being restated at each user. The stripper table test pinned both scalars against `strip_terminal_control_sequences` only; it now pins them against the quoting guard as well, which is where they got through. Co-Authored-By: Claude Opus 4.7 Signed-off-by: Roman Sirokov --- crates/rocm-core/src/terminal.rs | 49 ++++++++++++++++++-------------- engines/vllm/src/lib.rs | 46 +++++++++++++++++------------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/crates/rocm-core/src/terminal.rs b/crates/rocm-core/src/terminal.rs index d59a5bec9..20eead507 100644 --- a/crates/rocm-core/src/terminal.rs +++ b/crates/rocm-core/src/terminal.rs @@ -152,6 +152,25 @@ pub fn is_control_or_format(c: char) -> bool { | '\u{e0020}'..='\u{e007f}') } +/// Whether `c` is a `Cc` control or one of the two Unicode separators that +/// mandate a line break. +/// +/// `U+2028 LINE SEPARATOR` and `U+2029 PARAGRAPH SEPARATOR` are `Zl`/`Zp`, so +/// neither `char::is_control` nor [`is_control_or_format`]'s enumerated `Cf` +/// set covers them, yet a terminal draws the text after either on the next row. +/// The two places that must not let that happen — this module's classifier and +/// the vLLM engine's guard on what may be quoted into the `rocm diagnose +/// --symptom '...'` command it prints — therefore share this predicate instead +/// of each deciding for itself what breaks a line. The classifier was widened +/// to the pair when this walk moved out of the vLLM engine (the commit that +/// moved it says "behaviour is unchanged", which is true of everything except +/// this); the guard was not, and a line separator went on riding into the +/// printed command. +#[must_use] +pub fn is_control_or_line_separator(c: char) -> bool { + c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') +} + /// Consumes one glyph, control character or escape sequence and says which of /// the three it was. /// @@ -260,32 +279,18 @@ fn next_token(chars: &mut std::iter::Peekable>) -> Option Token { if c == '\t' { return Token::Text(c); } - if c.is_control() || matches!(c, '\u{2028}' | '\u{2029}') { + if is_control_or_line_separator(c) { return Token::LineBreak; } if is_control_or_format(c) { diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index efa523a42..85efd860e 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -4,7 +4,9 @@ use anyhow::{Context, Result, anyhow, bail}; use clap::{Parser, Subcommand}; -use rocm_core::terminal::{is_control_or_format, strip_terminal_control_sequences}; +use rocm_core::terminal::{ + is_control_or_format, is_control_or_line_separator, strip_terminal_control_sequences, +}; use rocm_core::{ AppPaths, DEFAULT_LOCAL_PORT, DependencyViolation, check_dependencies, ensure_uv_binary, format_http_base_url, openai_models_endpoint_has_model, require_nonempty, split_local_version, @@ -2774,11 +2776,16 @@ fn oom_utilization_hint(log_tail: &str) -> String { /// human-readable sentence above the command (and in the log tail printed with /// it), so nothing is lost but the copy-paste convenience. /// -/// The character test is [`is_control_or_format`], not `char::is_control`: the -/// latter is Unicode `Cc` only, so a bidi override in the failing line survived -/// into the printed command and reordered how it renders. +/// The character test is [`is_control_or_format`] together with +/// [`is_control_or_line_separator`], because each covers scalars the other does +/// not and `symptom` is the *raw* log line, not the stripped one: a `Cf` bidi +/// override reordered how the printed command renders, and a `Zl` line +/// separator broke it across two rows. fn quotable_in_single_quotes(symptom: &str) -> bool { - !symptom.contains('\'') && !symptom.chars().any(is_control_or_format) + !symptom.contains('\'') + && !symptom + .chars() + .any(|c| is_control_or_format(c) || is_control_or_line_separator(c)) } /// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a @@ -3510,16 +3517,9 @@ mod tests { "RuntimeError: HIP out of memory", "a bidi override must not survive into the message", ), - // U+2028/U+2029 are `Zl`/`Zp`, not `Cc`, and are in neither the - // enumerated `Cf` set, so the engine-local stripper this one - // replaced let both through verbatim -- it classified every - // non-escape character with `is_control_or_format` alone. Moving the - // walk into `rocm-core::terminal` widened that one case on purpose: - // both are mandatory Unicode line breaks, so a terminal draws the - // rest of the message on the next row, which is exactly what this - // stripper exists to stop untrusted output doing. These two cases - // are the only thing pinning it -- drop the pair from - // `classify_char`'s boundary set and both go red. + // The `Zl`/`Zp` pair that `is_control_or_line_separator` adds to + // `char::is_control`: drop it from that predicate and both of these + // go red, as do the quoting assertions below. ( "RuntimeError: HIP\u{2028}out of memory", "RuntimeError: HIPout of memory", @@ -3544,14 +3544,22 @@ mod tests { ); } - // Cf characters must also be inadmissible in the quoted command, not - // merely stripped from the echoed sentence: the two tests are separate - // because they are separate call sites and only one of them used - // `char::is_control`. + // Every one of those scalars must also be inadmissible in the quoted + // command, not merely stripped from the echoed sentence: the candidate + // the guard sees is built from the raw line, not the stripped one, so + // the second call site needs its own pins. assert!( !quotable_in_single_quotes("vllm: \u{202e}HIP out of memory"), "a bidi override must make a line unquotable, not ride into the command" ); + assert!( + !quotable_in_single_quotes("vllm: HIP\u{2028}out of memory"), + "a line separator must make a line unquotable, not ride into the command" + ); + assert!( + !quotable_in_single_quotes("vllm: HIP\u{2029}out of memory"), + "a paragraph separator must make a line unquotable, not ride into the command" + ); assert!( quotable_in_single_quotes("vllm: HIP out of memory"), "the rejection must not be so broad that ordinary lines stop qualifying" From 5da11d5710e917eefaf1f63feda7b38ac0650a3d Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 2 Oct 2026 14:44:56 +0300 Subject: [PATCH 38/39] docs(skill): document fix-16-vllm-oom in the rocm-doctor reference `fix-16-vllm-oom` is added to the CLI's closed catalog by this branch, but `skills/rocm-doctor/reference.md` still described `fix-16` as "a reserved handle, not a missing row". The skill doc is asserted against the CLI catalog by three e2e scenarios, and all three failed on the merge: `rocm fix` offers fix-16-vllm-oom, which reference.md does not document. Add the catalog row (linux/wsl, not auto-applicable, matching the recipe's LINUX_AND_WSL and auto_applicable: false), raise the catalog count from 24 to 25, and move the reserved-handle note to `fix-18`, which is still the only gap in the numeric series. Signed-off-by: Roman Sirokov --- skills/rocm-doctor/reference.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/skills/rocm-doctor/reference.md b/skills/rocm-doctor/reference.md index be7179abb..f0d3540f3 100644 --- a/skills/rocm-doctor/reference.md +++ b/skills/rocm-doctor/reference.md @@ -74,7 +74,7 @@ non-interactive shell without `--yes`, and confirm first. **Two exceptions:** pinned. Run `rocm fix fix-9-igpu-dgpu --device-index N` (not the bare id) once you know N. -## Closed catalog (24 failure modes) +## Closed catalog (25 failure modes) The OS column is the platform family the CLI scopes an entry to, and WSL2 is a family of its own — not a flavour of `linux`. An entry reaches a WSL host only @@ -99,6 +99,7 @@ reporting confident nonsense. | `fix-13-hip-sdk-missing` | windows | HIP SDK not installed | no HIP SDK under Program Files, `hipInfo` not recognized | no | | `fix-14-adrenalin-too-old` | windows | Adrenalin / kernel-mode driver too old for the HIP SDK | `hipInfo` can't enumerate, "driver too old", HSA "no agents found" | no | | `fix-15-msvc-redist` | windows | MSVC runtime missing (HIP DLLs can't load) | `vcruntime140.dll` / `vcruntime140_1.dll` missing | no | +| `fix-16-vllm-oom` | linux/wsl | vLLM reserved ~90% of the GPU's TOTAL VRAM for its KV cache and ran out at startup | vLLM engine start fails with an out-of-memory error; the GPU is shared/busy, or the model does not fit | no | | `fix-17-torch-dlpack` | linux | `torch-c-dlpack-ext` loads its CUDA prebuilt on a ROCm torch, aborting vLLM's engine start at import time | vLLM engine start fails on import; error names `torch_c_dlpack_ext` or tvm_ffi's `_optional_torch_c_dlpack` | no | | `fix-19-shm-too-small` | linux/wsl | `/dev/shm` too small for a serving workload, which needs gigabytes where a container and WSL2 both default to 64 MB | reported under 1 GiB; a data-loader worker killed by a bus error, or a failed write to a temporary file, with nothing naming shared memory | no | | `fix-wsl-1-gpu-not-exposed` | wsl | `/dev/dxg` absent, so the distro cannot reach the GPU at all | no `/dev/dxg`; in a container, the device was never passed through | no | @@ -109,14 +110,14 @@ reporting confident nonsense. | `fix-wsl-6-host-driver-too-old` | wsl | Windows host driver too old or absent, with the distro side already complete | the Windows host reports no AMD display adapter | no | | `fix-wsl-7-wsl1` | wsl | Distro running under WSL 1, which exposes no GPU device at all | the running kernel is a WSL 1 kernel | no | -Two things the numbering does not tell you. `fix-16` is a reserved handle, not a -missing row — ids are stable handles rather than positions. And the `fix-wsl-N` -entries are a parallel series, not a continuation of the numeric one, because -they answer for a different platform family. +One thing the numbering does not tell you: the `fix-wsl-N` entries are a parallel +series, not a continuation of the numeric one, because they answer for a +different platform family. (`fix-18` is a reserved handle rather than a missing +row — ids are stable handles, not positions.) Linux-only: fix-3, -4, -5, -7, -10, -11, -12, -17. Windows-only: fix-13, -14, -15. WSL-only: fix-wsl-1 through fix-wsl-7. Linux + Windows: fix-9. -Linux + WSL: fix-19. Linux + Windows + WSL: fix-1, -2, -6, -8. +Linux + WSL: fix-16, -19. Linux + Windows + WSL: fix-1, -2, -6, -8. ## Framework routing From 31f2186684486463c398963deec9b9f7cf5ef416 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 6 Oct 2026 09:52:20 +0300 Subject: [PATCH 39/39] fix(diagnose): let the catalog fill fix-16's auto_applicable `check_16_vllm_oom` built its `Fix` with an inline `auto_applicable: false`. `no_checker_hand_sets_auto_applicable` forbids that: only `take_applicability_from_the_catalog` may set the field, because the catalog pass overwrites whatever a checker puts there. The output was right either way -- which is the point. A checker that disagreed with the catalog would be silently masked rather than caught, the way `fix-9-igpu-dgpu` once drifted. The field is now left at `Fix::default()` and filled from the recipe, whose `applies_on: PRINT_ON_LINUX_AND_WSL` already carries `FixClass::PrintOnly`. This was latent rather than new: the guard never ran on the previous head because the suite aborted at an unrelated flaky test first, so the offending line reached `gpu-out-of-memory` unnoticed. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/diagnose.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 281124671..81c9f7b69 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -1695,7 +1695,6 @@ fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { "# problem; pick a smaller or quantized model (single-GPU serving only).".to_owned(), ], fix_id: "fix-16-vllm-oom".to_owned(), - auto_applicable: false, verify: "rocm serve # re-run and watch for a clean startup".to_owned(), notes: vec![ "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.".to_owned(),