diff --git a/crates/rocm-core/src/diagnose.rs b/crates/rocm-core/src/diagnose.rs index 784cdf490..a4d94cd2d 100644 --- a/crates/rocm-core/src/diagnose.rs +++ b/crates/rocm-core/src/diagnose.rs @@ -288,6 +288,51 @@ const KEYWORDS_PAGE_FAULT: KeywordTable = &[ ("out_of_registers", 30, "compiler OUT_OF_REGISTERS"), ]; +// vLLM's startup OOM. The distinctive signature is a torch/HIP allocation +// failure while the engine is reserving KV-cache VRAM; a bare "out of memory" +// on its own is deliberately sub-threshold so this only claims the failure mode +// when the vLLM/HIP shape of the error is present. +// +// None of these tokens are vLLM-specific on their own: `torch.OutOfMemoryError: +// CUDA out of memory` is the identical shape any ROCm PyTorch job emits (ROCm's +// PyTorch build reports the CUDA-compat name), so this table alone cannot tell +// vLLM's OOM apart from an arbitrary training script hitting the same +// allocator error. `check_16_vllm_oom` therefore requires `VLLM_ANCHOR_PATTERN` +// to match before scoring this table at all. +const KEYWORDS_VLLM_OOM: KeywordTable = &[ + ( + r"torch\.outofmemoryerror", + 45, + "error mentions torch.OutOfMemoryError", + ), + ( + "hip out of memory", + 45, + "error mentions 'HIP out of memory'", + ), + ( + r"cuda out of memory", + 45, + "error mentions 'CUDA out of memory' (ROCm reports the CUDA-compat name)", + ), + ( + r"tried to allocate .*(?:gib|mib)", + 30, + "error names an allocation it could not satisfy", + ), + ( + r"\boutofmemory\b", + 30, + "error mentions standalone OutOfMemory", + ), + ("out of memory", 25, "error mentions 'out of memory'"), + ( + r"gpu[-_]memory[-_]utilization", + 20, + "log mentions gpu_memory_utilization (vLLM VRAM reservation)", + ), +]; + /// Score the strongest (top-2) keyword matches in `table` against `symptom`. fn keyword_score(symptom: &str, table: KeywordTable) -> (i32, Vec) { if symptom.is_empty() { @@ -1267,6 +1312,67 @@ fn check_15_msvc_redist(e: &Examination, symptom: &str) -> Diagnosis { ) } +/// A required anchor before [`KEYWORDS_VLLM_OOM`] is even scored: the word +/// "vllm" itself, or its distinctive VRAM-reservation flag. Without one of +/// these, a generic HIP/CUDA/PyTorch OOM string is not evidence of *this* +/// failure mode -- see the comment on [`KEYWORDS_VLLM_OOM`]. `tensor[-_]parallel` +/// is deliberately NOT an anchor: it is a Megatron/DeepSpeed term (rocm-cli does +/// not serve one model across GPUs), so anchoring on it would misattribute those +/// frameworks' OOMs to vLLM. +const VLLM_ANCHOR_PATTERN: &str = r"vllm|gpu[-_]memory[-_]utilization"; + +/// vLLM ran the GPU out of memory. Keyword-only: an [`Examination`] carries no +/// per-GPU VRAM or tenancy fields, so nothing structural can corroborate this — +/// the user must pass the error via `--symptom`, or arrive from the serve +/// failure note that points here. `e` is therefore unused. +fn check_16_vllm_oom(_e: &Examination, symptom: &str) -> Diagnosis { + if !symptom_matches(symptom, VLLM_ANCHOR_PATTERN) { + // No vLLM anchor: don't attribute a bare framework OOM to vLLM at all. + return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); + } + let (score, evidence) = keyword_score(symptom, KEYWORDS_VLLM_OOM); + if score <= 0 { + return zero("fix-16-vllm-oom", "vLLM GPU out of memory"); + } + // Two different faults share this error, and the remediation splits on which + // one it is: on a shared/busy GPU vLLM's fixed ~90% reservation collides + // with memory already in use, so lowering the reservation is the fix; for a + // model that genuinely does not fit, lowering it only trades an earlier OOM + // for a later one, so the wording must not present the knob as the answer in + // that case. + let summary = format!( + "{} If instead the model genuinely does not fit in this GPU's VRAM, lowering the \ + reservation will not help — use a smaller or quantized model (rocm-cli serves one \ + model on a single GPU; it does not shard a model across GPUs).", + crate::VLLM_GPU_MEMORY_UTILIZATION_HINT + ); + let fix = Fix { + summary, + commands: vec![ + "# If the GPU is shared/busy (tenancy collision), lower vLLM's reservation:".to_owned(), + "rocm serve --gpu-memory-utilization 0.5".to_owned(), + "# ...or steer the server onto a less-busy device:".to_owned(), + "rocm serve --gpu ".to_owned(), + "# If the model genuinely does not fit, the reservation is not the problem:".to_owned(), + "# pick a smaller or quantized model (single-GPU serving only).".to_owned(), + ], + fix_id: "fix-16-vllm-oom".to_owned(), + auto_applicable: false, + verify: "rocm serve # re-run and watch for a clean startup".to_owned(), + notes: vec![ + "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it does not create room a too-large model needs.".to_owned(), + ], + ..Fix::default() + }; + finalize( + "fix-16-vllm-oom", + "vLLM ran the GPU out of memory at startup", + score, + evidence, + fix, + ) +} + /// A checker plus the OS families it applies to. type Checker = (fn(&Examination, &str) -> Diagnosis, &'static [&'static str]); @@ -1286,12 +1392,15 @@ const CHECKERS: &[Checker] = &[ (check_13_hip_sdk_missing, &["windows"]), (check_14_adrenalin_too_old, &["windows"]), (check_15_msvc_redist, &["windows"]), + (check_16_vllm_oom, &["linux", "wsl"]), ]; /// Run every applicable checker, drop zero-score results, sort by score /// descending (stable, so ties keep catalog order). fn run_all_checks(e: &Examination, symptom: &str) -> Vec { - let os_family = if e.os_family.is_empty() { + let os_family = if e.is_wsl { + "wsl" + } else if e.os_family.is_empty() { "linux" } else { e.os_family.as_str() @@ -1330,11 +1439,25 @@ pub fn diagnose(e: &Examination, symptom: &str) -> DiagnoseReport { // it as out of scope (exit_code() == 2). Mirror that here: skip the // bare-metal Linux catalog entirely so we don't emit false positives like // fix-4-render-group / fix-5-amdgpu-load on a healthy WSL2 box. - let out_of_scope = wsl_out_of_scope_message(e); - let matched = if out_of_scope.is_some() { - Vec::new() + let (matched, out_of_scope) = if e.is_wsl { + // Most catalog checks inspect bare-metal Linux state that is irrelevant + // on WSL2. Keyword-only checks explicitly registered for `wsl` are safe + // to run there, however. `run_all_checks` already filters to just those + // (only fix-16-vllm-oom today), so an empty result here means no + // wsl-applicable checker fired at all -- that's the out-of-scope case. + // A *nonempty* result must be kept as-is, sub-threshold entries and all: + // `DiagnoseReport::matched`'s contract is every nonzero-score checker + // that fired, and dropping weak hits here would silently violate it (and + // would discard real signal -- e.g. a bare "out of memory" mention on + // WSL -- in favor of the generic out-of-scope routing message). + let wsl_matches = run_all_checks(e, symptom); + if wsl_matches.is_empty() { + (Vec::new(), wsl_out_of_scope_message(e)) + } else { + (wsl_matches, None) + } } else { - run_all_checks(e, symptom) + (run_all_checks(e, symptom), None) }; DiagnoseReport { has_match: any_cleared_threshold(&matched), @@ -1914,6 +2037,122 @@ mod tests { assert!(report.matched.iter().any(|d| d.id == "fix-11-iommu")); } + #[test] + fn vllm_oom_signature_is_a_high_confidence_match() { + // The distinctive vLLM startup OOM: torch/HIP allocation failure, with + // the vLLM anchor that tells this apart from an arbitrary PyTorch OOM. + let report = diagnose( + &linux_base(), + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + ); + let top = &report.matched[0]; + assert_eq!(top.id, "fix-16-vllm-oom"); + assert!(top.score >= HIGH_CONFIDENCE, "score was {}", top.score); + assert!(report.has_match()); + // The remediation must stay conditional: the reservation knob is right + // for a tenancy collision and wrong for a model that does not fit. + let fix = top.fix.as_ref().unwrap(); + assert!(!fix.auto_applicable, "OOM workaround must be print-only"); + assert!(fix.summary.contains("--gpu-memory-utilization")); + assert!( + fix.summary.contains("does not fit"), + "the 'genuinely does not fit' caveat must survive: {}", + fix.summary + ); + } + + #[test] + fn generic_pytorch_oom_without_a_vllm_signal_does_not_match() { + // The reviewed false positive: `torch.OutOfMemoryError: CUDA out of + // memory` scores 45+45=90 under the keyword table alone, but nothing + // about that string is vLLM-specific -- any ROCm PyTorch job emits the + // identical shape (ROCm's PyTorch build reports the CUDA-compat name). + // Without an explicit vLLM anchor, this failure mode must not fire. + let report = diagnose(&linux_base(), "torch.OutOfMemoryError: CUDA out of memory"); + assert!( + report.matched.iter().all(|d| d.id != "fix-16-vllm-oom"), + "a bare framework OOM with no vLLM signal must not be attributed to vLLM: {:?}", + report.matched + ); + assert!(!report.has_match()); + } + + #[test] + fn torch_oom_class_alone_is_not_a_vllm_match() { + // `outofmemory` is nested inside the exception class name. It must not + // count as a second, independent signal and turn one token into a + // high-confidence vLLM diagnosis for an arbitrary PyTorch workload. + let report = diagnose(&linux_base(), "vllm: torch.OutOfMemoryError"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the weak keyword signal remains visible"); + assert!(oom.score < MIN_SCORE_FOR_MATCH, "score was {}", oom.score); + assert!(!report.has_match()); + } + + #[test] + fn a_bare_out_of_memory_stays_below_the_match_threshold() { + // Without the vLLM/HIP shape, "out of memory" alone must not claim this + // failure mode -- it scores (the `vllm` anchor is present and "out of + // memory" is a keyword), but stays below MIN_SCORE_FOR_MATCH. + let report = diagnose(&linux_base(), "vllm: the process was killed: out of memory"); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("the weak `out of memory` keyword signal must remain visible in `matched`"); + assert!( + oom.score < MIN_SCORE_FOR_MATCH, + "a bare OOM must stay sub-threshold, got {}", + oom.score + ); + } + + #[test] + fn vllm_oom_is_not_available_on_native_windows() { + // vLLM is Linux/WSL-only (native Windows is unsupported), so the checker + // must not fire on a Windows examination even with the error text. + let e = Examination { + os_family: "windows".to_owned(), + ..Examination::default() + }; + let report = diagnose(&e, "vllm: torch.OutOfMemoryError: HIP out of memory."); + assert!(report.matched.iter().all(|d| d.id != "fix-16-vllm-oom")); + } + + #[test] + fn vllm_oom_keyword_diagnosis_is_available_on_wsl() { + let mut e = linux_base(); + e.is_wsl = true; + let report = diagnose( + &e, + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.", + ); + assert!(report.out_of_scope.is_none()); + assert!(report.has_match()); + assert_eq!(report.matched[0].id, "fix-16-vllm-oom"); + } + + #[test] + fn wsl_sub_threshold_vllm_signal_is_preserved_not_routed_out_of_scope() { + // A weak (sub-threshold) vLLM OOM signal on WSL must still surface in + // `matched` per the `DiagnoseReport::matched` contract -- it must not + // be silently dropped in favor of the out-of-scope routing message. + let mut e = linux_base(); + e.is_wsl = true; + let report = diagnose(&e, "vllm: out of memory"); + assert!(report.out_of_scope.is_none()); + assert!(!report.has_match()); + let oom = report + .matched + .iter() + .find(|d| d.id == "fix-16-vllm-oom") + .expect("a nonzero sub-threshold hit must stay in `matched`, not be dropped"); + assert!(oom.score < MIN_SCORE_FOR_MATCH, "score was {}", oom.score); + } + #[test] fn keyword_score_takes_top_two() { // INVALID_ISA: two hits (50 + 40) -> 90, not the sum of all. diff --git a/crates/rocm-core/src/fix.rs b/crates/rocm-core/src/fix.rs index 5e788dda6..a901b754e 100644 --- a/crates/rocm-core/src/fix.rs +++ b/crates/rocm-core/src/fix.rs @@ -364,6 +364,30 @@ const RECIPES: &[FixRecipe] = &[ applies_on: WINDOWS_ONLY, runner: None, }, + FixRecipe { + fix_id: "fix-16-vllm-oom", + title: "vLLM ran the GPU out of memory at startup", + rationale: "vLLM reserves a fixed fraction (~90%) of each GPU's TOTAL VRAM for its KV cache by default, regardless of the model size or how much is currently free. Two different faults surface the same OOM, and they need opposite responses. If the GPU is shared or already busy, that reservation collides with memory in use and lowering it (or moving to a less-busy GPU) is the fix. If the model genuinely does not fit in this GPU's VRAM, lowering the reservation only trades an earlier OOM for a later one -- use a smaller or quantized model instead (rocm-cli serves one model on a single GPU; it does not shard a model across GPUs).", + auto_applicable: false, + commands: &[ + "# Case 1 -- shared/busy GPU (tenancy collision): lower the reservation,", + "# or steer vLLM onto a less-busy device:", + "rocm serve --gpu-memory-utilization 0.5", + "rocm serve --gpu ", + "# Case 2 -- the model genuinely does not fit: the reservation is not the", + "# problem; pick a smaller or quantized model (single-GPU serving only).", + ], + needs_sudo: false, + needs_reboot: false, + needs_relogin: false, + verify: "rocm serve # re-run and watch for a clean startup", + notes: &[ + "Only lower --gpu-memory-utilization when the GPU is shared or already busy; on a GPU dedicated to this server it cannot create the room a too-large model needs.", + "This entry is keyword-matched from the error text: `rocm diagnose` cannot see per-GPU VRAM or tenancy, so pass the failure with --symptom (or arrive from the `rocm serve` failure note).", + ], + applies_on: LINUX_ONLY, + runner: None, + }, ]; fn find_recipe(fix_id: &str) -> Option<&'static FixRecipe> { @@ -459,7 +483,7 @@ pub fn apply(fix_id: &str, opts: &FixOptions) -> i32 { if looks_like_a_diagnosis_position(fix_id) { // `rocm diagnose` ranks findings `#1`, `#2`, and users reach for that // number here. It is a position in one report, not a name -- and it - // does not line up with the catalog's `fix-1 … fix-15` either, so a + // does not line up with the catalog's `fix-1 … fix-16` either, so a // bare "unknown id" left them with nothing to correct. eprintln!( "`{fix_id}` looks like a position in a `rocm diagnose` report, not a fix-id." @@ -1072,7 +1096,7 @@ mod tests { let count = ids.len(); ids.dedup(); assert_eq!(ids.len(), count, "duplicate fix-id in RECIPES"); - assert_eq!(count, 15, "expected 15 catalog entries"); + assert_eq!(count, 16, "expected 16 catalog entries"); } #[test] @@ -1103,6 +1127,12 @@ mod tests { "fix-9-igpu-dgpu" ] ); + // fix-16-vllm-oom carries a workaround the user must weigh (lowering the + // VRAM reservation is wrong for a model that genuinely does not fit), so + // it is deliberately PRINT-ONLY and must never join the AUTO set. + let oom = find_recipe("fix-16-vllm-oom").expect("OOM recipe is in the catalog"); + assert!(!oom.auto_applicable, "fix-16-vllm-oom must stay PRINT-ONLY"); + assert!(oom.runner.is_none(), "a PRINT-ONLY recipe has no runner"); } #[test] diff --git a/docs/vllm.md b/docs/vllm.md index 87f67e5ae..ea0c51578 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -135,7 +135,8 @@ a tiny model. rocm-cli helps in three ways: - **OOM failures hint the workaround.** When a startup failure log shows an out-of-memory error, the failure message suggests retrying with a smaller reservation, e.g. `--gpu-memory-utilization 0.1`, or targeting a less-busy GPU - with `--gpu `. + with `--gpu `, and points at `rocm diagnose --symptom ''` + for the full conditional remediation (busy GPU vs. a model that does not fit). Explicitly, the workaround for an OOM on a shared card is: @@ -143,6 +144,8 @@ Explicitly, the workaround for an OOM on a shared card is: rocm serve --gpu-memory-utilization 0.1 # optionally target a less-busy GPU rocm serve --gpu 3 --gpu-memory-utilization 0.1 +# for the full busy-GPU-vs-model-too-large breakdown, pass the error to diagnose +rocm diagnose --symptom 'vllm: torch.OutOfMemoryError: HIP out of memory' ``` ### Tool calling diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 73905a749..9cd6ceee6 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -1829,14 +1829,23 @@ fn startup_log_context(log_path: Option<&Path>) -> String { /// with the `rocm` CLI's pre-launch low-VRAM note so both surfaces point the /// user at the same fix rather than drifting into different phrasing. fn oom_utilization_hint(log_tail: &str) -> String { - if log_tail_shows_oom(log_tail) { - format!( - "\n\nDetected an out-of-memory failure. {}", - rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT - ) - } else { - String::new() + if !log_tail_shows_oom(log_tail) { + return String::new(); } + // Route the user's *actual* failing line into the `--symptom` example + // rather than a canned string, prefixed with a `vllm:` anchor so the + // diagnose checker attributes it to this failure mode. + let symptom_line = log_tail + .lines() + .rev() + .map(str::trim) + .find(|line| !line.is_empty() && log_tail_shows_oom(line)) + .unwrap_or("out of memory"); + format!( + "\n\nDetected an out-of-memory failure. {}\n\ + For conditional remediation, run `rocm diagnose --symptom 'vllm: {symptom_line}'`.", + rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT + ) } /// Case-insensitive scan for the out-of-memory signatures vLLM/PyTorch emit on a @@ -2241,6 +2250,16 @@ mod tests { "an OOM tail must surface the utilization workaround: {hint}" ); assert!(hint.contains("--gpu ")); + assert!( + hint.contains("rocm diagnose --symptom"), + "an OOM tail must route the user to the conditional catalog entry: {hint}" + ); + assert!( + hint.contains( + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB." + ), + "the user's real failing line must be routed into --symptom, not a canned string: {hint}" + ); // Detection is case-insensitive and also matches the spaced phrasing. assert!(log_tail_shows_oom("HIP OUT OF MEMORY")); diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 88b2f795f..a9d253df8 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -140,3 +140,13 @@ Feature: Diagnosing failures and listing fixes When the user asks the CLI which fixes it offers Then every fix the catalog documents is listed And only the fixes the CLI can carry out itself are marked as such + + # vLLM runs on Linux and WSL, but not native Windows. This scenario is + # GPU-independent: it supplies the captured startup error as symptom text and + # proves the public diagnosis output preserves both branches of the remedy. + @id:diagnose-vllm-oom-is-conditional @requires-os:linux + Scenario: 13 - A vLLM startup OOM receives conditional remediation + Given a user whose vLLM server ran out of GPU memory + When the user asks the CLI to diagnose that symptom in machine-readable form + Then the diagnosis identifies the vLLM startup OOM + And the OOM remedy distinguishes a busy GPU from a model that does not fit diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 1c617ad29..9c739b4a3 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -64,11 +64,14 @@ pub struct ScenarioDecl { /// in-tree amdgpu driver, so it does not hold under WSL2 — which uses /// `/dev/dxg` and the Windows host driver instead. /// - /// Two things stop short there. `rocm diagnose` skips its bare-metal catalog - /// *by design* rather than emitting Linux diagnoses that cannot apply, so a - /// scenario needing a catalog match has no premise. And `examine`'s probe - /// returns as soon as it recognises WSL2, before most of its steps run, so a - /// scenario asserting anything those steps populate has none either. + /// Two things stop short there. `rocm diagnose` skips its *bare-metal* + /// catalog *by design* rather than emitting Linux diagnoses that cannot + /// apply, so a scenario needing one of those bare-metal matches has no + /// premise. (Keyword-only checks explicitly registered for WSL — e.g. the + /// vLLM startup-OOM entry — do run there; those belong under + /// `@requires-os:linux`, not this tag.) And `examine`'s probe returns as + /// soon as it recognises WSL2, before most of its steps run, so a scenario + /// asserting anything those steps populate has none either. /// /// This is a SKIP and not an xfail row: the diagnose half is deliberate and /// separately unit-tested, and recording it as a known bug would tell every diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 74272ed19..87e25508d 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -57,6 +57,7 @@ const CATALOG_FIX_IDS: &[&str] = &[ "fix-13-hip-sdk-missing", "fix-14-adrenalin-too-old", "fix-15-msvc-redist", + "fix-16-vllm-oom", ]; /// The fixes the CLI carries out itself. Every other entry only prints a plan. @@ -117,6 +118,13 @@ async fn user_hit_unknown_failure(world: &mut E2eWorld) { world.model_name = Some("xyzzy totally unrelated gibberish".to_string()); } +#[given("a user whose vLLM server ran out of GPU memory")] +async fn user_hit_vllm_oom(world: &mut E2eWorld) { + world.model_name = Some( + "vllm: torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.".to_string(), + ); +} + #[given("a user who has chosen a known fix")] async fn user_chose_known_fix(world: &mut E2eWorld) { world.model_name = Some(PREVIEW_FIX_ID.to_string()); @@ -323,6 +331,70 @@ async fn assert_json_identifies_match(world: &mut E2eWorld) { ); } +/// Locate the `fix-16-vllm-oom` diagnosis in a parsed report's `matched` array, +/// panicking with the raw output if it is absent. Shared by the two `Then` +/// steps that assert on the vLLM OOM entry. +fn find_vllm_oom<'a>(report: &'a serde_json::Value, output: &str) -> &'a serde_json::Value { + report + .get("matched") + .and_then(serde_json::Value::as_array) + .and_then(|matches| { + matches.iter().find(|diagnosis| { + diagnosis.get("id").and_then(serde_json::Value::as_str) == Some("fix-16-vllm-oom") + }) + }) + .unwrap_or_else(|| panic!("expected fix-16-vllm-oom in diagnosis:\n{output}")) +} + +#[then("the diagnosis identifies the vLLM startup OOM")] +async fn assert_diagnosis_identifies_vllm_oom(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "diagnose should exit 0"); + let (report, output) = parsed_diagnosis(world); + let oom = find_vllm_oom(&report, &output); + let high_confidence = report + .get("high_confidence_threshold") + .and_then(serde_json::Value::as_i64) + .unwrap_or_else(|| panic!("report must publish its high_confidence_threshold:\n{output}")); + assert!( + oom.get("score") + .and_then(serde_json::Value::as_i64) + .is_some_and(|score| score >= high_confidence), + "the distinctive OOM signature should be high-confidence:\n{output}" + ); + assert_eq!( + oom.pointer("/fix/auto_applicable") + .and_then(serde_json::Value::as_bool), + Some(false), + "the OOM remedy must remain print-only:\n{output}" + ); +} + +#[then("the OOM remedy distinguishes a busy GPU from a model that does not fit")] +async fn assert_oom_remedy_is_conditional(world: &mut E2eWorld) { + let (report, output) = parsed_diagnosis(world); + let fix = find_vllm_oom(&report, &output) + .get("fix") + .unwrap_or_else(|| panic!("expected fix-16-vllm-oom remediation:\n{output}")); + let rendered = fix.to_string(); + assert!( + rendered.contains("shared/busy") + && rendered.contains("does not fit") + && rendered.contains("--gpu-memory-utilization") + && rendered.contains("smaller or quantized"), + "the remedy must retain both conditional branches:\n{output}" + ); + assert!( + !rendered.contains("--tensor-parallel-size"), + "the remedy must not prescribe the non-existent multi-GPU sharding flag:\n{output}" + ); + assert!( + !fix.get("verify") + .and_then(serde_json::Value::as_str) + .is_some_and(|verify| verify.contains("--gpu-memory-utilization")), + "verification must not unconditionally prescribe the tenancy workaround:\n{output}" + ); +} + /// Hold the report to its own arithmetic: `has_match` is true exactly when some /// cause cleared the threshold the report itself declares. Returns the ids that /// cleared it.