diff --git a/apps/rocm/src/comfyui.rs b/apps/rocm/src/comfyui.rs index 4e0707fae..5ae412265 100644 --- a/apps/rocm/src/comfyui.rs +++ b/apps/rocm/src/comfyui.rs @@ -2111,6 +2111,7 @@ mod tests { ], ..therock::RocmSdkPythonProbe::default() }), + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms: 100, @@ -2176,6 +2177,7 @@ mod tests { library_paths: vec![sdk_lib.clone()], ..Default::default() }), + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms: 100, @@ -2272,6 +2274,7 @@ mod tests { ], ..therock::RocmSdkPythonProbe::default() }), + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms: 100, diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 88b63864d..4f404b61d 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -94,9 +94,9 @@ fn chat_api_key_from_env() -> Option { /// unavailable store yields `None` (the dash still launches; switching to the /// Anthropic provider then surfaces an actionable error turn). fn anthropic_api_key_for_dash() -> Option { - crate::provider_keys::resolve_provider_api_key("anthropic", "ANTHROPIC_API_KEY") + crate::provider_keys::provider_credential("anthropic", "ANTHROPIC_API_KEY") .ok() - .map(|k| k.value) + .map(crate::provider_keys::ProviderCredential::into_value) } /// Adapt the built-in `rocm-core` model recipes into the TUI-local summaries the diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 375e851ca..75a75ffc3 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2444,8 +2444,8 @@ fn install(target: InstallTarget) -> Result<()> { finalize_successful_sdk_install(&paths)? }; print!("{output}"); - if let Some(finalized) = finalized { - print_sdk_install_success(&finalized); + if let Some(finalized) = &finalized { + print_sdk_install_success(finalized); // The SDK runtime wheel bundles PyTorch, whose ROCm build // links against libatomic.so.1 and the system numactl // runtime (libnuma.so.1 / libnuma_1.2). Ensure both are @@ -2453,41 +2453,22 @@ fn install(target: InstallTarget) -> Result<()> { // engine (if any) is auto-installed below. ensure_libatomic_for_torch(yes); ensure_libnuma_for_torch(yes); - if let Err(error) = - maybe_auto_install_sdk_preferred_engine(&paths, &finalized, yes) - { - record_cli_audit_event( - &paths, - "engine", - "engine_auto_install", - "error", - format!( - "auto-install failed engine=vllm runtime_id={} family={}: {error}", - finalized.runtime_key, finalized.family - ), - None, - ); - eprintln!("warning: automatic vLLM install failed: {error}"); - eprintln!( - "warning: SDK install completed; you can run `rocm engines install vllm --runtime-id {}` after vLLM is available in that runtime", - finalized.runtime_key - ); - } } - record_cli_audit_event( + finish_sdk_install( &paths, - "runtime", + finalized.as_ref(), if dry_run { "install_sdk_dry_run" } else { "install_sdk" }, - "info", format!( "sdk install completed channel={channel} format={format_name} prefix={prefix_display} version_selector={version_selector_display} dry_run={dry_run}" ), - None, - ); + |paths, finalized| { + maybe_auto_install_sdk_preferred_engine(paths, finalized, yes) + }, + )?; } Err(error) => { record_cli_audit_event( @@ -3812,7 +3793,7 @@ fn engines(command: EnginesCommand) -> Result<()> { println!(" reinstall: {reinstall}"); println!(" env_id: {}", response.env_id); println!(" env_path: {}", response.env_path); - for warning in response.warnings { + for warning in &response.warnings { println!(" warning: {warning}"); } if response.managed_env == Some(false) { @@ -3831,6 +3812,12 @@ fn engines(command: EnginesCommand) -> Result<()> { config.save(&paths)?; let _ = seeded_preference; } + // Settle last, matching `maybe_auto_install_sdk_preferred_engine`. The + // check blocks then print under the `engine:`/`runtime_id:`/`env_id:` + // lines they describe instead of above them, and the config bookkeeping + // above still lands when settling fails — the engine did install; it is + // the runtime it left behind that is being reported on. + settle_engine_install(&paths, &engine, &runtime_id, &response)?; record_cli_audit_event( &paths, "engine", @@ -3954,6 +3941,51 @@ fn runtime_manifest_for_selector<'a>( }) } +/// The `runtime_key` of the runtime whose install root contains `python`. +fn runtime_key_for_python(paths: &AppPaths, python: &Path) -> Option { + let manifests = therock::load_runtime_manifests(paths).ok()?; + runtime_key_owning_python(&manifests, python).map(str::to_owned) +} + +/// Which runtime owns an interpreter, decided by install root. +/// +/// Split from the registry read so the decision can be tested without a +/// registry on disk, matching `sdk_torch_build_from_manifest`. +/// +/// `runtime_id` cannot answer this: it is shared by every side-by-side install +/// of one channel and family, which is exactly the situation an engine install +/// has to be attributed in. An install root contains one runtime by +/// construction, so the interpreter's path settles it. +/// +/// Both sides are compared verbatim *and* canonicalized. The CLI writes +/// `install_root` canonicalized while an engine adapter reports back whatever +/// path it was handed, and comparing a single form makes ownership fail +/// silently on a symlinked runtimes directory. Roots can nest, so the longest +/// containing root wins. +fn runtime_key_owning_python<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + python: &Path, +) -> Option<&'a str> { + fn both_forms(path: &Path) -> Vec { + let verbatim = path.to_path_buf(); + match path.canonicalize() { + Ok(resolved) if resolved != verbatim => vec![verbatim, resolved], + _ => vec![verbatim], + } + } + + let pythons = both_forms(python); + manifests + .iter() + .filter(|manifest| { + both_forms(&manifest.install_root) + .iter() + .any(|root| pythons.iter().any(|python| python.starts_with(root))) + }) + .max_by_key(|manifest| manifest.install_root.as_os_str().len()) + .map(|manifest| manifest.runtime_key.as_str()) +} + fn env_root_for_service( paths: &AppPaths, engine: &str, @@ -4014,7 +4046,7 @@ fn ensure_self_managed_engine_ready( None } else { eprintln!("Preparing {engine} for GPU serving..."); - Some(engine_request_with_env_root::<_, InstallResponse>( + let response = engine_request_with_env_root::<_, InstallResponse>( Some(paths), engine, EngineMethod::Install, @@ -4025,7 +4057,14 @@ fn ensure_self_managed_engine_ready( env_root: env_root.clone(), }, env_root.as_deref(), - )?) + )?; + // No `settle_engine_install` here. This function returns at the top unless + // `engine_manages_own_runtime(engine)`, and that is exactly the case + // `settles_runtime_torch` declines: the runtime holds the engine's own + // binary, not an interpreter with a torch in it. Calling it would be inert + // at best, and a call that provably cannot act invites someone to "fix" the + // gate later. + Some(response) }; let engine_config = config.engine_config_mut(engine); @@ -4342,6 +4381,7 @@ fn resolve_engine_env( }, env_root.as_deref(), )?; + settle_engine_install(paths, engine, &runtime_id, &response)?; Ok(ResolvedEngineEnv { env_id: response.env_id, runtime_id, @@ -7394,6 +7434,9 @@ fn maybe_auto_install_sdk_preferred_engine( engine, runtime_python_for_key(paths, &finalized.runtime_key).as_deref(), &finalized.runtime_key, + // The engine install failed, so no alignment ran and any + // divergence found here is a real one. + None, ); return Err(error); } @@ -7401,7 +7444,7 @@ fn maybe_auto_install_sdk_preferred_engine( println!(" reinstall: false"); println!(" env_id: {}", response.env_id); println!(" env_path: {}", response.env_path); - for warning in response.warnings { + for warning in &response.warnings { println!(" warning: {warning}"); } @@ -7417,16 +7460,7 @@ fn maybe_auto_install_sdk_preferred_engine( } config.save(paths)?; - // The SDK and the engine share this environment, and the SDK's torch stack was - // just written into it. Say plainly whether the engine's own requirements - // survived that, so a runtime the engine cannot use is never reported only as a - // successful install. - report_engine_dependency_check( - paths, - engine, - Some(Path::new(&response.python_executable)), - &finalized.runtime_key, - ); + settle_engine_install(paths, engine, &finalized.runtime_key, &response)?; } record_cli_audit_event( @@ -7451,7 +7485,24 @@ enum EngineDependencyCheck { Satisfied, /// The engine declares requirements the environment does not meet, one line each, /// as the resolver reported them. - Violated(Vec), + /// + /// `expected` carries any divergence the install made on purpose that shares the + /// run with a genuine violation. It is reported separately rather than folded in + /// because the remedy differs: reinstalling the engine repairs `violations` and + /// destroys `expected`, so naming that remedy is only safe while `expected` is + /// empty. + Violated { + violations: Vec, + expected: Vec, + }, + /// The only unmet requirements are ones the install deliberately diverged from. + /// + /// Torch alignment leaves the runtime holding the SDK's build of the release + /// the engine pins, which the engine's exact-pin metadata cannot express. The + /// distinction matters because the remedy for a real violation — reinstalling + /// the engine — is precisely what would undo the alignment and restore a + /// runtime that cannot open a device. + ExpectedDivergence(Vec), /// The check itself could not run (no usable `uv`, unreadable environment). NotVerified(String), } @@ -7466,18 +7517,34 @@ fn report_engine_dependency_check( engine: &str, python: Option<&Path>, runtime_key: &str, + realigned_package: Option<&str>, ) { - let outcome = engine_dependency_check(paths, engine, python); + let outcome = engine_dependency_check(paths, engine, python, realigned_package); print!("{}", render_engine_dependency_check(engine, &outcome)); let (level, message) = match &outcome { EngineDependencyCheck::Satisfied => ( "info", format!("engine={engine} runtime_id={runtime_key} dependency_check=satisfied"), ), - EngineDependencyCheck::Violated(details) => ( + EngineDependencyCheck::Violated { + violations, + expected, + } => ( "error", format!( - "engine={engine} runtime_id={runtime_key} dependency_check=violated: {}", + "engine={engine} runtime_id={runtime_key} dependency_check=violated: {}{}", + violations.join("; "), + if expected.is_empty() { + String::new() + } else { + format!(" (expected_divergence: {})", expected.join("; ")) + } + ), + ), + EngineDependencyCheck::ExpectedDivergence(details) => ( + "info", + format!( + "engine={engine} runtime_id={runtime_key} dependency_check=expected_divergence: {}", details.join("; ") ), ), @@ -7507,663 +7574,1804 @@ fn runtime_python_for_key(paths: &AppPaths, runtime_key: &str) -> Option Vec { + let Ok(manifests) = therock::load_runtime_manifests(paths) else { + return Vec::new(); + }; + runtime_manifest_for_selector(&manifests, runtime_key) + .and_then(|manifest| manifest.rocm_sdk.as_ref()) + .map(|probe| probe.library_paths.clone()) + .unwrap_or_default() +} + +/// The wheel index the runtime was installed from. +fn runtime_index_url_for_key(paths: &AppPaths, runtime_key: &str) -> Option { + let manifests = therock::load_runtime_manifests(paths).ok()?; + let manifest = runtime_manifest_for_selector(&manifests, runtime_key)?; + manifest.index_url.clone() +} + +/// Which torch a runtime should hold once an engine has been installed into it. +/// +/// Two installers write torch into the same environment. The SDK install writes +/// TheRock's build; the engine then writes the build from its own index, pinned +/// to an exact version. Letting either side win unconditionally is wrong, and +/// both failures have been observed in the field: +/// +/// * If the engine's build always wins, the runtime can end up with a torch that +/// loads against the installed SDK and then enumerates no devices, so serving +/// fails with an unhelpful error long after the install reported success. +/// * If the SDK's build always wins, the runtime can end up on a torch that +/// enumerates a device and still has no kernel image for this exact target, +/// failing on the first tensor operation instead. +/// +/// So compatibility is tested rather than assumed. A torch already executing a +/// GPU kernel with this SDK is kept exactly as it is — that decision is +/// `TorchRetention`, and it runs first. This alignment is the repair for +/// everything else, and what it installs is the SDK's *build* of the *release* +/// the engine pins: the release comes from the engine, which is built against it, +/// and the build comes from the SDK, which the libraries belong to. +#[derive(Debug, Clone, PartialEq, Eq)] +enum TorchAlignment { + /// The runtime already holds the SDK build of the engine's torch release. + AlreadyAligned { version: String }, + /// The engine's build was replaced with the SDK build of the same release. + Realigned { from: String, to: String }, + /// The SDK publishes no build of that release; the engine's own is kept. + /// + /// Claimed only when the resolver actually said so. Any other failure is an + /// `InstallFailed`, because asserting "not published" over a network or disk + /// error sends the reader hunting for a missing wheel that exists. + Unavailable { wanted: String, kept: String }, + /// The realignment install failed for some other reason, carried verbatim. + InstallFailed { + wanted: String, + kept: String, + error: String, + }, + /// The user opted out, and a replacement was due: their torch is kept. + /// + /// Distinct from every other outcome because nothing was attempted. Reading + /// it as `NotApplicable` would say the rule had no opinion, when it had one + /// and was told not to act on it, and that difference is what the reader + /// needs: `wanted` is the build that was not installed, `kept` the one still + /// there. Only reachable when those two actually differ, so a runtime the + /// opt-out changed nothing about is never described as one it spared. + Disabled { wanted: String, kept: String }, + /// Nothing to decide — no exact pin, no engine metadata, or no SDK torch. + NotApplicable(String), +} + +/// What the alignment rule concludes, before any of it is acted on. +#[derive(Debug, Clone, PartialEq, Eq)] +enum TorchAlignmentPlan { + AlreadyAligned { version: String }, + Install { wanted: String, from: String }, + NotApplicable(String), +} + +/// The rule itself, kept free of I/O so both field failures can be tested. +/// +/// `wanted` is the engine's pinned *release* carrying the SDK's *build*. +fn plan_torch_alignment( + sdk_build: Option<&str>, + installed_torch: Option<&str>, + engine_requirement: Option<&str>, + engine: &str, +) -> TorchAlignmentPlan { + let Some(sdk_build) = sdk_build else { + return TorchAlignmentPlan::NotApplicable( + "the runtime manifest does not identify the SDK's torch build".to_owned(), + ); + }; + let Some(requirement) = engine_requirement else { + return TorchAlignmentPlan::NotApplicable(format!("{engine} does not pin torch")); + }; + let Some(pinned) = therock::requirement_pinned_version(requirement) else { + return TorchAlignmentPlan::NotApplicable(format!( + "{engine} does not pin torch to an exact version ({requirement})" + )); + }; + let wanted = format!("{}+{sdk_build}", therock::split_local_version(pinned).0); + let installed = installed_torch.unwrap_or_default().to_owned(); + if installed == wanted { + TorchAlignmentPlan::AlreadyAligned { version: wanted } + } else { + TorchAlignmentPlan::Install { + wanted, + from: installed, + } + } +} + +/// Whether a failed install means the resolver could not find that version. +/// +/// Deliberately narrow. Anything unmatched is reported as a plain failure with +/// its error, so an unrecognised message degrades to the honest answer rather +/// than to a confident wrong one. +fn install_error_reports_version_unavailable(error: &str) -> bool { + let error = error.to_ascii_lowercase(); + error.contains("no solution found") + || error.contains("were found for") + || error.contains("not found in the package registry") + || error.contains("has no version") +} + +/// Whether the user has opted out of realigning torch. +/// +/// The alignment rewrites a package the user may have installed deliberately, and +/// it runs on every path that installs an engine, so a hand-installed torch is +/// otherwise replaced again by the next `engines install`, `install sdk`, or +/// `engines shell`. The stack this resolves to — the SDK's build of the release +/// the engine pins — is not validated against the supported matrix, so "the SDK's +/// build does not work on this machine" is a case that can happen rather than a +/// hypothetical one, and it needs an exit that is not "stop using the CLI". +/// +/// The engine reads the same variable through the same helper, so the two sides +/// cannot drift; [`rocm_core::torch_alignment_disabled`] carries why that matters. +/// Suppressing the correction does not suppress the diagnosis: a runtime that +/// opens no device or cannot run a kernel on one is still reported as such — and +/// still fails the install on a host where a GPU was found. +fn torch_alignment_disabled() -> bool { + rocm_core::torch_alignment_disabled() +} + +/// Install the SDK's build of the release the engine pins, when that is needed. +/// +/// `torch` is the metadata probe the caller already took: the retention decision +/// that runs first needs the engine's pin too, and asking the runtime the same +/// question twice would be a second interpreter launch for an answer in hand. +/// +/// The opt-out is read where the install would run, not at the top. That is the +/// only place it changes anything, and reading it there is what lets the result +/// name the replacement it declined to make: a runtime with no replacement due +/// reports what it is, rather than reporting an opt-out that skipped nothing. +fn align_runtime_torch( paths: &AppPaths, + python: &Path, + index_url: Option<&str>, + sdk_build: Option<&str>, engine: &str, - python: Option<&Path>, -) -> EngineDependencyCheck { - let Some(python) = python else { - return EngineDependencyCheck::NotVerified( - "the runtime's Python environment could not be located".to_owned(), + torch: Result<&therock::TorchAlignmentProbe, &anyhow::Error>, +) -> TorchAlignment { + let probe = match torch { + Ok(probe) => probe, + Err(error) => return TorchAlignment::NotApplicable(error.to_string()), + }; + let plan = plan_torch_alignment( + sdk_build, + probe.installed_torch.as_deref(), + probe.engine_requires_torch.as_deref(), + engine, + ); + let (wanted, from) = match plan { + TorchAlignmentPlan::AlreadyAligned { version } => { + return TorchAlignment::AlreadyAligned { version }; + } + TorchAlignmentPlan::NotApplicable(reason) => { + return TorchAlignment::NotApplicable(reason); + } + TorchAlignmentPlan::Install { wanted, from } => (wanted, from), + }; + // Read only now that a replacement is actually due, so the opt-out reports + // the install it stopped rather than standing in for a runtime that needed + // nothing. Everything after this point is the rewrite itself. + if torch_alignment_disabled() { + return TorchAlignment::Disabled { wanted, kept: from }; + } + let Some(index_url) = index_url else { + return TorchAlignment::NotApplicable( + "the runtime manifest records no wheel index to install from".to_owned(), ); }; - match rocm_core::check_dependencies(paths, python) { - Ok(violations) => { - let owned = rocm_core::violations_requiring(&violations, engine); - if owned.is_empty() { - EngineDependencyCheck::Satisfied + match therock::install_pinned_package( + paths, + python, + index_url, + "torch", + &format!("torch=={wanted}"), + ) { + Ok(()) => TorchAlignment::Realigned { from, to: wanted }, + // The SDK index may not publish this release at all. That is a real + // possibility, not an error to abort on: the engine's own build is left + // in place and the device check that follows reports whether it works. + Err(error) => { + let error = format!("{error:#}"); + if install_error_reports_version_unavailable(&error) { + TorchAlignment::Unavailable { wanted, kept: from } } else { - EngineDependencyCheck::Violated( - owned - .iter() - .map(|violation| violation.detail.clone()) - .collect(), - ) + TorchAlignment::InstallFailed { + wanted, + kept: from, + error, + } } } - Err(error) => EngineDependencyCheck::NotVerified(error.to_string()), } } -fn render_engine_dependency_check(engine: &str, outcome: &EngineDependencyCheck) -> String { +/// Render one alignment outcome as the `torch_alignment:` check block. +/// +/// `engine` names the engine whose pin decided the release, so the realigned line +/// reads `... the release vllm pins` rather than an anonymous "the engine". It +/// arrives from the engine selection and is sanitized like every other +/// interpolated value. +/// +/// `before` is the verdict the runtime gave *prior* to the realignment. Only the +/// realigned outcome replaced anything, so only that arm prints it — and it must, +/// because the `device_check:` block further down reports the runtime as it is +/// now. Without the before verdict beside it, a realignment that ends in a +/// failing runtime does not say whether the alignment broke something that +/// worked or found something already broken, and answering that afterwards means +/// going to the machine to look. The value is already in hand at the call site; +/// dropping it only moves the cost onto whoever reads the output. +fn render_torch_alignment( + outcome: &TorchAlignment, + engine: &str, + before: Option<&RuntimeDeviceCheck>, +) -> String { let mut output = String::new(); match outcome { - EngineDependencyCheck::Satisfied => { - let _ = writeln!(output, " dependency_check: satisfied"); - } - EngineDependencyCheck::NotVerified(reason) => { + TorchAlignment::AlreadyAligned { version } => { let _ = writeln!( output, - " dependency_check: not_verified ({})", - sanitize_log_value(reason) + " torch_alignment: already_aligned ({})", + sanitize_log_value(version) ); } - EngineDependencyCheck::Violated(details) => { - let _ = writeln!(output, " dependency_check: violated"); - for detail in details { - let _ = writeln!(output, " violation: {}", sanitize_log_value(detail)); + TorchAlignment::Realigned { from, to } => { + let _ = writeln!(output, " torch_alignment: realigned"); + let _ = writeln!( + output, + " {} -> {} (the SDK's build of the release {} pins)", + sanitize_log_value(from), + sanitize_log_value(to), + sanitize_log_value(engine) + ); + if let Some(before) = before { + let _ = writeln!( + output, + " before this replacement: {} (device_check below is after it)", + sanitize_log_value(&device_check_verdict(before)) + ); } + } + TorchAlignment::Unavailable { wanted, kept } => { + let _ = writeln!(output, " torch_alignment: unavailable"); + let _ = writeln!( + output, + " the SDK index publishes no {}; keeping {}", + sanitize_log_value(wanted), + sanitize_log_value(kept) + ); + } + TorchAlignment::InstallFailed { + wanted, + kept, + error, + } => { + let _ = writeln!(output, " torch_alignment: install_failed"); + let _ = writeln!( + output, + " could not install {}: {}", + sanitize_log_value(wanted), + sanitize_log_value(error) + ); + let _ = writeln!(output, " keeping {}", sanitize_log_value(kept)); + } + TorchAlignment::Disabled { wanted, kept } => { + let _ = writeln!(output, " torch_alignment: disabled"); + let _ = writeln!( + output, + " ROCM_CLI_DISABLE_TORCH_ALIGNMENT is set; keeping {} rather than installing {}", + sanitize_log_value(kept), + sanitize_log_value(wanted) + ); + } + TorchAlignment::NotApplicable(reason) => { let _ = writeln!( output, - " action: rocm engines install {engine} --reinstall" + " torch_alignment: not_applicable ({})", + sanitize_log_value(reason) ); } } output } -fn render_sdk_install_success(finalized: &SdkInstallFinalization) -> String { - format!( - "ROCm SDK installed successfully.\n install folder: {}\n active runtime: {}\n next step: run `rocm help` to see how to use rocm-cli.\n", - finalized.install_root.display(), - finalized.runtime_key - ) -} - -fn finalize_successful_sdk_install(paths: &AppPaths) -> Result> { - let Some(manifest) = newest_installed_runtime_manifest(paths)? else { - return Ok(None); +/// Repair the runtime's torch, print the `torch_alignment:` block, and record it. +/// +/// Reached only when no torch in the runtime has proven it can run a GPU kernel. +/// The outcome is returned so the dependency check that follows can be read +/// against it: `deliberately_diverged_package` turns it into the one package +/// whose divergence is intended. +fn report_torch_alignment( + paths: &AppPaths, + engine: &str, + python: &Path, + runtime_key: &str, + sdk_build: Option<&str>, + torch: Result<&therock::TorchAlignmentProbe, &anyhow::Error>, + before: &RuntimeDeviceCheck, +) -> TorchAlignment { + let index_url = runtime_index_url_for_key(paths, runtime_key); + let outcome = align_runtime_torch( + paths, + python, + index_url.as_deref(), + sdk_build, + engine, + torch, + ); + print!("{}", render_torch_alignment(&outcome, engine, Some(before))); + let (level, message) = match &outcome { + TorchAlignment::AlreadyAligned { version } => ( + "info", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment=already_aligned version={version}" + ), + ), + // The before verdict rides along in the audit line too: the log is read + // long after the install, when the runtime on disk can no longer answer + // what it was like beforehand. + TorchAlignment::Realigned { from, to } => ( + "info", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment=realigned from={from} to={to} before={}", + device_check_verdict(before) + ), + ), + TorchAlignment::Unavailable { wanted, kept } => ( + "error", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment=unavailable wanted={wanted} kept={kept}" + ), + ), + TorchAlignment::InstallFailed { + wanted, + kept, + error, + } => ( + "error", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment=install_failed wanted={wanted} kept={kept}: {error}" + ), + ), + // The user's own decision, carried out as asked, so it is not an error. + // The device check that follows is what says whether the torch they kept + // works, and that verdict is recorded — and enforced — on its own. + TorchAlignment::Disabled { wanted, kept } => ( + "info", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment=disabled kept={kept} wanted={wanted}" + ), + ), + TorchAlignment::NotApplicable(reason) => ( + "info", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment=not_applicable: {reason}" + ), + ), }; - let mut config = RocmCliConfig::load(paths)?; - config.setup.completed = true; - config.setup.therock_venv = Some(manifest.install_root.clone()); - config.save(paths)?; + record_cli_audit_event(paths, "engine", "torch_alignment", level, message, None); + outcome +} - let activation_paths = paths - .clone() - .with_managed_root(manifest.install_root.clone(), false); - if paths.config_dir != activation_paths.config_dir - || paths.data_dir != activation_paths.data_dir - { - recover_setup_runtime_registration(paths, &config)?; - let mut current_config = RocmCliConfig::load(paths)?; - current_config.setup.completed = true; - current_config.setup.therock_venv = Some(manifest.install_root.clone()); - let _ = activate_runtime(paths, &mut current_config, &manifest.runtime_key)?; +/// The package the runtime now deliberately diverges on, if any, so the +/// dependency check can tell that divergence apart from a real violation. +/// +/// Both `Realigned` and `AlreadyAligned` diverge from the engine's exact pin — +/// the second is a rerun over a runtime already put right, which is the normal +/// state on every refresh after the first. +/// +/// `Disabled` counts for the same reason, from the other direction: the torch +/// that does not satisfy the pin is the one the user told us to leave alone, so +/// reporting it as a violation would answer their instruction with an error and +/// a `--reinstall` remedy that would undo it. It is only ever constructed over a +/// real mismatch, so this never suppresses a divergence that is not there. +/// +/// `Unavailable` and `InstallFailed` stay excluded. Nothing was replaced there +/// either, but nothing was intended either — the repair was attempted and did +/// not happen — so whatever the dependency check finds is a real finding. +const fn deliberately_diverged_package(outcome: &TorchAlignment) -> Option<&'static str> { + match outcome { + TorchAlignment::Realigned { .. } + | TorchAlignment::AlreadyAligned { .. } + | TorchAlignment::Disabled { .. } => Some("torch"), + TorchAlignment::Unavailable { .. } + | TorchAlignment::InstallFailed { .. } + | TorchAlignment::NotApplicable(_) => None, } +} - recover_setup_runtime_registration(&activation_paths, &config)?; - - let mut config = RocmCliConfig::load(&activation_paths)?; - config.setup.completed = true; - config.setup.therock_venv = Some(manifest.install_root.clone()); - let activation = activate_runtime(&activation_paths, &mut config, &manifest.runtime_key)?; +/// Whether the installed runtime can actually run work on a GPU. +/// +/// The dependency check answers whether the engine's declared requirements are +/// satisfied. That is a question about metadata, and it is not the same question +/// as whether the environment works. Two distinct answers matter here, because a +/// torch can fail at either step: one built against a different ROCm version than +/// the installed SDK loads cleanly and then reports no devices, which vLLM turns +/// into `Failed to infer device type` at first serve; one built without a kernel +/// image for this target reports a device and then dies on the first tensor +/// operation. Only a torch that gets past both has been shown to work. +#[derive(Debug, Clone, PartialEq, Eq)] +enum RuntimeDeviceCheck { + /// torch imported and reported at least one device. + Usable { + device_count: u32, + torch_version: String, + }, + /// torch imported but reported no devices — a torch built for another SDK. + NoDevices { + torch_version: String, + hip_version: String, + }, + /// torch found a device and then could not execute a kernel on it. + /// + /// Distinct from `NoDevices` because the remedy is different: the runtime is + /// not looking at the wrong SDK, it is holding a build with no code for this + /// GPU. + KernelFailed { + torch_version: String, + error: String, + }, + /// The question could not be answered — including when torch does not import. + /// + /// Never a reason to assume healthy, but on its own never fatal either: see + /// `install_left_runtime_unusable`, which acts only on the verdicts where the + /// runtime was asked and answered badly. This variant covers benign causes as + /// well as real ones — a runtime whose Python could not be located, or a probe + /// that could not launch — and failing a multi-gigabyte install because a + /// probe did not run is worse than reporting what was and was not seen. The + /// cost is that a runtime whose torch is present but unimportable is reported + /// rather than failed. + NotVerified(String), +} - Ok(Some(SdkInstallFinalization { - runtime_key: activation.runtime_key, - install_root: manifest.install_root, - family: manifest.family, - })) +fn runtime_device_check(python: Option<&Path>, library_paths: &[PathBuf]) -> RuntimeDeviceCheck { + let Some(python) = python else { + return RuntimeDeviceCheck::NotVerified( + "the runtime's Python environment could not be located".to_owned(), + ); + }; + match therock::probe_runtime_devices(python, library_paths) { + Ok(probe) => classify_runtime_device_probe(probe), + Err(error) => RuntimeDeviceCheck::NotVerified(error.to_string()), + } } -fn newest_installed_runtime_manifest( - paths: &AppPaths, -) -> Result> { - let mut manifests = therock::load_runtime_manifests(paths)?; - manifests.sort_by(|left, right| { - right - .installed_at_unix_ms - .cmp(&left.installed_at_unix_ms) - .then_with(|| left.runtime_key.cmp(&right.runtime_key)) - }); - Ok(manifests.into_iter().next()) +/// Read one probe as a verdict, kept free of I/O so every state can be tested. +fn classify_runtime_device_probe(probe: therock::RuntimeDeviceProbe) -> RuntimeDeviceCheck { + if !probe.import_ok { + return RuntimeDeviceCheck::NotVerified( + probe + .error + .unwrap_or_else(|| "torch did not import".to_owned()), + ); + } + let torch_version = probe.torch_version.unwrap_or_else(|| "unknown".to_owned()); + // A kernel that would not launch is definitive, so it is read before the + // count and never dropped: the probe only reaches that step after it has + // already enumerated a device, and the count alone would read as healthy. + if let Some(error) = probe.kernel_error { + return RuntimeDeviceCheck::KernelFailed { + torch_version, + error, + }; + } + match probe.device_count { + Some(0) => RuntimeDeviceCheck::NoDevices { + torch_version, + hip_version: probe.hip_version.unwrap_or_else(|| "unknown".to_owned()), + }, + Some(device_count) => RuntimeDeviceCheck::Usable { + device_count, + torch_version, + }, + // A `device_count()` that raises — HIP init failures are the live example — + // reports no count and puts the real exception in `error`. That exception is + // precisely the diagnostic this probe exists to capture, so prefer it over + // the generic line, which is only right when the probe returned nothing at + // all to explain itself. + None => RuntimeDeviceCheck::NotVerified( + probe + .error + .unwrap_or_else(|| "torch imported but did not report a device count".to_owned()), + ), + } } -#[derive(Debug, Clone)] -struct AdoptRuntimeRequest { - python_executable: PathBuf, - install_root: PathBuf, - runtime_id: String, - runtime_key: String, - replace: bool, +/// One device-check verdict on one line, for quoting inside another block. +/// +/// The full block explains the verdict and names a consequence, which is right +/// where it is the answer and wrong where it is context for something else. This +/// keeps the part that identifies the verdict — the name and the torch it was +/// asked about — so a before/after pair reads as a pair rather than as two +/// competing diagnoses. +fn device_check_verdict(outcome: &RuntimeDeviceCheck) -> String { + match outcome { + RuntimeDeviceCheck::Usable { + device_count, + torch_version, + } => format!("usable ({device_count} device(s), torch {torch_version})"), + RuntimeDeviceCheck::NoDevices { torch_version, .. } => { + format!("no_devices (torch {torch_version})") + } + RuntimeDeviceCheck::KernelFailed { torch_version, .. } => { + format!("kernel_failed (torch {torch_version})") + } + RuntimeDeviceCheck::NotVerified(reason) => format!("not_verified ({reason})"), + } } -#[derive(Debug, Clone)] -struct AdoptRuntimeOptions { - python_input: PathBuf, - install_root: Option, - runtime_id: Option, - runtime_key: Option, - channel: Option, - replace: bool, +fn render_runtime_device_check(outcome: &RuntimeDeviceCheck) -> String { + let mut output = String::new(); + match outcome { + RuntimeDeviceCheck::Usable { + device_count, + torch_version, + } => { + let _ = writeln!( + output, + " device_check: usable ({device_count} device(s), torch {})", + sanitize_log_value(torch_version) + ); + } + RuntimeDeviceCheck::NotVerified(reason) => { + let _ = writeln!( + output, + " device_check: not_verified ({})", + sanitize_log_value(reason) + ); + } + RuntimeDeviceCheck::NoDevices { + torch_version, + hip_version, + } => { + let _ = writeln!(output, " device_check: no_devices"); + let _ = writeln!( + output, + " torch {} (hip {}) imported but reports 0 devices", + sanitize_log_value(torch_version), + sanitize_log_value(hip_version) + ); + // Deliberately no remedy: the reinstall that would satisfy the + // engine's pin is what produces this state, so naming it here would + // send people in a circle. Say what is wrong and let them choose. + let _ = writeln!( + output, + " serving will fail with `Failed to infer device type`; this torch is built \ + for a different ROCm version than the installed SDK" + ); + } + RuntimeDeviceCheck::KernelFailed { + torch_version, + error, + } => { + let _ = writeln!(output, " device_check: kernel_failed"); + let _ = writeln!( + output, + " torch {} found a GPU but could not run a kernel on it: {}", + sanitize_log_value(torch_version), + sanitize_log_value(error) + ); + let _ = writeln!( + output, + " serving will fail on the first tensor operation; this torch has no kernel \ + image for this GPU" + ); + } + } + output } -fn adopt_runtime_from_python_options( +/// Print one device-check verdict and record it. +/// +/// The verdict is taken as an argument because the settle path probes the runtime +/// before it decides what to do with it, and the block is printed in its usual +/// place afterwards rather than where the probe happened to run. +fn report_runtime_device_check( paths: &AppPaths, - options: AdoptRuntimeOptions, -) -> Result { - let (python_executable, inferred_root) = resolve_adopt_python_input(&options.python_input)?; - let probe = therock::probe_rocm_sdk_runtime(&python_executable) - .with_context(|| format!("failed to probe {}", python_executable.display()))?; - let request = infer_adopt_runtime_request( - python_executable, - options.install_root.or(inferred_root), - options.runtime_id, - options.runtime_key, - options.channel, - options.replace, - &probe, - )?; - adopt_runtime_from_probe(paths, request, probe) + engine: &str, + runtime_key: &str, + outcome: RuntimeDeviceCheck, +) -> RuntimeDeviceCheck { + print!("{}", render_runtime_device_check(&outcome)); + let (level, message) = match &outcome { + RuntimeDeviceCheck::Usable { device_count, .. } => ( + "info", + format!( + "engine={engine} runtime_id={runtime_key} device_check=usable devices={device_count}" + ), + ), + RuntimeDeviceCheck::NoDevices { + torch_version, + hip_version, + } => ( + "error", + format!( + "engine={engine} runtime_id={runtime_key} device_check=no_devices torch={torch_version} hip={hip_version}" + ), + ), + RuntimeDeviceCheck::KernelFailed { + torch_version, + error, + } => ( + "error", + format!( + "engine={engine} runtime_id={runtime_key} device_check=kernel_failed torch={torch_version}: {error}" + ), + ), + RuntimeDeviceCheck::NotVerified(reason) => ( + "info", + format!("engine={engine} runtime_id={runtime_key} device_check=not_verified: {reason}"), + ), + }; + record_cli_audit_event( + paths, + "engine", + "runtime_device_check", + level, + message, + None, + ); + outcome } -fn infer_adopt_runtime_request( - python_executable: PathBuf, - install_root: Option, - runtime_id: Option, - runtime_key: Option, - channel: Option, - replace: bool, - probe: &therock::RocmSdkPythonProbe, -) -> Result { - let install_root = install_root.with_context(|| { - format!( - "could not infer the Python environment folder from {}; pass --root", - python_executable.display() - ) - })?; - let runtime_id = match runtime_id { - Some(value) if !value.trim().is_empty() => { - if let Some(channel) = channel.as_deref() { - let (parsed_channel, _) = parse_therock_runtime_id(&value)?; - let requested_channel = normalize_adopt_channel(channel)?; - if parsed_channel != requested_channel { - bail!( - "--channel {requested_channel} does not match runtime id channel {parsed_channel}" - ); - } - } - value - } - Some(_) => bail!("runtime_id must not be empty"), - None => { - let channel = normalize_adopt_channel(channel.as_deref().unwrap_or("release"))?; - let family = probe - .resolved_target_family - .as_deref() - .or(probe.default_target_family.as_deref()) - .filter(|value| !value.trim().is_empty()) - .context( - "rocm_sdk probe did not report a GPU package; pass --runtime-id explicitly", - )?; - format!("therock-{channel}:{family}") - } +/// The build identifier of the torch that belongs to this runtime's SDK. +/// +/// Taken from the manifest, never from whatever torch happens to be installed. +/// The environment tells you what is there now, which after an engine install is +/// the engine's build — trusting that would let a runtime already holding the +/// wrong torch declare itself correct and never recover. +/// +/// Older manifests predate the recorded value, so fall back to deriving it from +/// the SDK version, which is how TheRock names these builds. +fn sdk_torch_build_for_key(paths: &AppPaths, runtime_key: &str) -> Option { + let manifests = therock::load_runtime_manifests(paths).ok()?; + let manifest = runtime_manifest_for_selector(&manifests, runtime_key)?; + sdk_torch_build_from_manifest(manifest) +} + +/// The build identifier one manifest names, with the fallback for older manifests. +/// +/// Split out from the lookup so the decision can be tested without a registry on +/// disk: the lookup is a directory scan, but this is the part that has to be right +/// for a runtime already stuck on the engine's build to recover. +fn sdk_torch_build_from_manifest(manifest: &therock::InstalledRuntimeManifest) -> Option { + if let Some(recorded) = manifest.sdk_torch.as_deref() + && let Some(build) = therock::split_local_version(recorded).1 + { + return Some(build.to_owned()); + } + let version = manifest + .rocm_sdk + .as_ref() + .and_then(|probe| probe.rocm_sdk_version.clone()) + .unwrap_or_else(|| manifest.version.clone()); + (!version.trim().is_empty()).then(|| format!("rocm{version}")) +} + +/// Which torch a runtime keeps once one of them has run a GPU kernel. +/// +/// Exactly two builds are allowed to stand: the engine's own exact pin, and the +/// SDK's build of the release it pins. Both are states this tool produces on +/// purpose, and each is a fixed point — a rerun over either one keeps it and +/// installs nothing, which is what stops `--reinstall` from oscillating between +/// the two package sources. Anything else is repaired by `TorchAlignment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum TorchRetention { + /// The engine's exact pin ran a kernel. It satisfies the pin, so it diverges + /// from nothing and the dependency check is read straight. + EngineBuild { version: String }, + /// The SDK's build of the pinned release ran. The engine's pin stays + /// deliberately unsatisfied, which the dependency check is told to expect. + SdkBuild { version: String }, + /// Nothing here has been shown to work: align, then look again. + Realign, +} + +/// Decide retention from the torch that actually executed a kernel. +/// +/// The version compared is the one the running interpreter reported, not the one +/// distribution metadata claims: the build that ran is the build being kept. The +/// engine's pin is checked first, so a pin that already names the SDK's build is +/// reported as satisfied rather than as an intended divergence from itself. +fn classify_retained_torch( + sdk_build: Option<&str>, + devices: &RuntimeDeviceCheck, + engine_requirement: Option<&str>, +) -> TorchRetention { + // Only a kernel that ran earns retention. Every other verdict — no devices, + // a failed launch, or no answer at all — goes to the repair path, which is + // also what this tool did before it could tell those apart. + let RuntimeDeviceCheck::Usable { torch_version, .. } = devices else { + return TorchRetention::Realign; }; - let runtime_key = match runtime_key { - Some(value) if !value.trim().is_empty() => value, - Some(_) => bail!("runtime_key must not be empty"), - None => { - let (channel, family) = parse_therock_runtime_id(&runtime_id)?; - let version = probe - .rocm_sdk_version - .as_deref() - .filter(|value| !value.trim().is_empty()) - .context("rocm_sdk probe did not report a version; cannot name adopted runtime")?; - format!( - "adopted-{channel}-pip-{}-{}", - runtime_key_component(&family), - runtime_key_component(version) - ) - } + let Some(pinned) = engine_requirement.and_then(therock::requirement_pinned_version) else { + return TorchRetention::Realign; }; - Ok(AdoptRuntimeRequest { - python_executable, - install_root, - runtime_id, - runtime_key, - replace, - }) + if torch_version.as_str() == pinned { + return TorchRetention::EngineBuild { + version: torch_version.clone(), + }; + } + let Some(sdk_build) = sdk_build else { + return TorchRetention::Realign; + }; + // The same version `plan_torch_alignment` would install, so the state that + // repair produces is the state recognised here on the next run. + let sdk_torch = format!("{}+{sdk_build}", therock::split_local_version(pinned).0); + if torch_version.as_str() == sdk_torch { + return TorchRetention::SdkBuild { + version: torch_version.clone(), + }; + } + TorchRetention::Realign } -fn normalize_adopt_channel(channel: &str) -> Result { - match channel.trim().to_ascii_lowercase().as_str() { - "release" => Ok("release".to_owned()), - "nightly" => Ok("nightly".to_owned()), - other => bail!("adopt channel must be release or nightly, got `{other}`"), +/// The package a retained torch deliberately diverges on, if any. +/// +/// The SDK's build does not satisfy the engine's exact pin and is kept anyway; +/// the engine's own build satisfies it, so claiming a divergence there would +/// suppress a violation that has not happened. +const fn retained_diverged_package(retention: &TorchRetention) -> Option<&'static str> { + match retention { + TorchRetention::SdkBuild { .. } => Some("torch"), + TorchRetention::EngineBuild { .. } | TorchRetention::Realign => None, } } -fn runtime_key_component(value: &str) -> String { +/// Render a retention as the `torch_alignment:` block, in place of an alignment. +/// +/// `Realign` renders nothing: nothing was retained, and `render_torch_alignment` +/// prints the block for the repair that runs instead. +fn render_torch_retention(retention: &TorchRetention, engine: &str) -> String { let mut output = String::new(); - let mut last_dash = false; - for ch in value.chars() { - if ch.is_ascii_alphanumeric() { - output.push(ch.to_ascii_lowercase()); - last_dash = false; - } else if !last_dash { - output.push('-'); - last_dash = true; + match retention { + TorchRetention::EngineBuild { version } => { + let _ = writeln!( + output, + " torch_alignment: retained_engine_build ({})", + sanitize_log_value(version) + ); + let _ = writeln!( + output, + " the torch {} pins ran a GPU kernel with this SDK", + sanitize_log_value(engine) + ); } + TorchRetention::SdkBuild { version } => { + let _ = writeln!( + output, + " torch_alignment: retained_sdk_build ({})", + sanitize_log_value(version) + ); + let _ = writeln!( + output, + " the SDK's build of the release {} pins ran a GPU kernel", + sanitize_log_value(engine) + ); + } + TorchRetention::Realign => {} } - let trimmed = output.trim_matches('-').to_owned(); - if trimmed.is_empty() { - "runtime".to_owned() - } else { - trimmed - } + output } -fn resolve_adopt_python_input(input: &Path) -> Result<(PathBuf, Option)> { - let absolute = if input.is_absolute() { - input.to_path_buf() - } else { - std::env::current_dir() - .context("failed to resolve current directory")? - .join(input) +/// Print the retention block and record it. `Realign` records nothing. +fn report_torch_retention( + paths: &AppPaths, + engine: &str, + runtime_key: &str, + retention: &TorchRetention, +) { + print!("{}", render_torch_retention(retention, engine)); + let (state, version) = match retention { + TorchRetention::EngineBuild { version } => ("retained_engine_build", version), + TorchRetention::SdkBuild { version } => ("retained_sdk_build", version), + TorchRetention::Realign => return, }; - if absolute.is_dir() { - let env_root = absolute.canonicalize().with_context(|| { - format!( - "failed to resolve Python environment folder {}", - absolute.display() - ) - })?; - let python = runtime_python_executable_in_env(&env_root); - if !python.is_file() { - bail!( - "Python executable is missing in {}", - python.parent().unwrap_or(env_root.as_path()).display() - ); - } - return Ok((python, Some(env_root))); - } - if absolute.is_file() { - let inferred_root = infer_python_env_root(&absolute); - return Ok((absolute, inferred_root)); - } - bail!( - "Python executable or folder is missing: {}", - absolute.display() + record_cli_audit_event( + paths, + "engine", + "torch_alignment", + "info", + format!( + "engine={engine} runtime_id={runtime_key} torch_alignment={state} version={version}" + ), + None, ); } -fn infer_python_env_root(python_executable: &Path) -> Option { - let bin_dir = python_executable.parent()?; - let bin_name = bin_dir.file_name()?.to_string_lossy(); - if bin_name.eq_ignore_ascii_case("Scripts") || bin_name == "bin" { - return bin_dir.parent().map(Path::to_path_buf); +/// Whether this host has a GPU at all, answered by inspecting the host itself. +/// +/// Deliberately independent of the runtime under test: the runtime reporting no +/// devices is the symptom being judged, so it cannot also be the evidence. The +/// third state is the one that matters — a host that could not be examined is not +/// a host without a GPU, and failing a multi-gigabyte install on that guess would +/// be worse than saying what was seen. +#[derive(Debug, Clone, PartialEq, Eq)] +enum HostGpu { + Detected, + Absent, + NotVerified(String), +} + +fn detect_host_gpu() -> HostGpu { + match ExamineSummary::gather() { + Ok(summary) if summary.detected_gfx_target.is_some() => HostGpu::Detected, + Ok(_) => HostGpu::Absent, + Err(error) => HostGpu::NotVerified(format!("{error:#}")), } - bin_dir.parent().map(Path::to_path_buf) } -fn adopt_runtime_from_probe( - paths: &AppPaths, - request: AdoptRuntimeRequest, - probe: therock::RocmSdkPythonProbe, -) -> Result { - if request.runtime_key.trim().is_empty() { - bail!("runtime_key must not be empty"); - } - if request.runtime_id.trim().is_empty() { - bail!("runtime_id must not be empty"); - } - let python_executable = absolute_existing_file_path_preserving_symlink( - &request.python_executable, - "runtime Python executable", - )?; - let install_root = request.install_root.canonicalize().with_context(|| { - format!( - "runtime install root is missing: {}", - request.install_root.display() - ) - })?; - if !install_root.is_dir() { - bail!( - "runtime install root is missing: {}", - install_root.display() - ); - } - let (channel, family) = parse_therock_runtime_id(&request.runtime_id)?; - therock::validate_rocm_sdk_runtime_probe(&probe)?; - let version = probe - .rocm_sdk_version - .as_deref() - .filter(|value| !value.trim().is_empty()) - .context("rocm_sdk probe did not report a version; cannot adopt runtime explicitly")? - .to_owned(); +/// Say why a runtime that cannot serve did not fail the install. +fn report_unverified_host_gpu(paths: &AppPaths, engine: &str, runtime_key: &str, reason: &str) { + println!(" host_gpu: not_verified ({})", sanitize_log_value(reason)); + println!(" the install is not failed on a host whose GPUs could not be inspected"); + record_cli_audit_event( + paths, + "engine", + "host_gpu", + "info", + format!("engine={engine} runtime_id={runtime_key} host_gpu=not_verified: {reason}"), + None, + ); +} - let manifest = therock::InstalledRuntimeManifest { - runtime_key: request.runtime_key, - runtime_id: request.runtime_id, - channel, - format: "wheel".to_owned(), - family, - family_source: "runtime_id".to_owned(), - version, - install_root: install_root.clone(), - selected_artifact_url: "adopted-read-only".to_owned(), - index_url: None, - tarball_file_name: None, - python_launcher: None, - python_executable: Some(python_executable.display().to_string()), - pip_cache_dir: None, - rocm_sdk: Some(probe), - read_only: true, - imported_from: Some(install_root), - installed_at_unix_ms: rocm_core::unix_time_millis(), - }; - validate_runtime_manifest_for_activation(&manifest) - .with_context(|| format!("adopted runtime `{}` is not usable", manifest.runtime_key))?; - write_runtime_registry_manifest(paths, &manifest, request.replace)?; - Ok(manifest) +/// Whether the runtime this install produced is one that cannot serve. +/// +/// Both bad verdicts count: a runtime that opens no device and one that opens a +/// device it cannot run a kernel on both fail at first serve. Neither says +/// anything about the alignment that preceded it — a repair that could not run +/// may still leave a working environment, and one that ran may still not have +/// helped, so what the runtime does now is the only thing read. +const fn runtime_cannot_serve(devices: &RuntimeDeviceCheck) -> bool { + matches!( + devices, + RuntimeDeviceCheck::NoDevices { .. } | RuntimeDeviceCheck::KernelFailed { .. } + ) } -fn absolute_existing_file_path_preserving_symlink(path: &Path, label: &str) -> Result { - let absolute = if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir() - .context("failed to resolve current directory")? - .join(path) - }; - if !absolute.is_file() { - bail!("{label} is missing: {}", absolute.display()); - } - Ok(absolute) +/// Whether an install finished having produced a runtime that cannot serve. +/// +/// Gated on the host, because the same verdict means different things on +/// different machines: no device is the correct answer on a machine with no GPU, +/// and a host that could not be examined has not told us which machine this is. +/// Only a GPU found independently of the runtime turns a runtime that cannot +/// serve into a failed install. +const fn install_left_runtime_unusable(devices: &RuntimeDeviceCheck, host_gpu: &HostGpu) -> bool { + matches!(host_gpu, HostGpu::Detected) && runtime_cannot_serve(devices) } -fn parse_therock_runtime_id(runtime_id: &str) -> Result<(String, String)> { - let (prefix, family) = runtime_id.split_once(':').with_context(|| { - format!("runtime_id `{runtime_id}` must include a TheRock family suffix after ':'") - })?; - let family = family.trim(); - if family.is_empty() { - bail!("runtime_id `{runtime_id}` has an empty TheRock family suffix"); +/// An engine install finished having left a runtime that cannot run GPU work. +/// +/// A distinct type rather than a plain message so `install sdk` can tell this +/// apart from an ordinary engine-install failure. It deliberately tolerates the +/// latter — a failed engine install still leaves a good multi-gigabyte SDK +/// behind, and discarding that would be worse — but a runtime it has just left +/// unable to serve is the exact silent success this change exists to end, so +/// that one has to fail the command. +#[derive(Debug)] +struct UnusableRuntimeAfterInstall(String); + +impl std::fmt::Display for UnusableRuntimeAfterInstall { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) } - let channel = match prefix.trim() { - "therock-release" => "release", - "therock-nightly" => "nightly", - other => bail!( - "runtime_id `{runtime_id}` must start with therock-release: or therock-nightly:, got `{other}`" - ), - }; - Ok((channel.to_owned(), family.to_owned())) } -fn write_runtime_registry_manifest( +impl std::error::Error for UnusableRuntimeAfterInstall {} + +fn unusable_runtime_error(engine: &str, runtime_key: &str) -> anyhow::Error { + anyhow::Error::new(UnusableRuntimeAfterInstall(format!( + "the {engine} install left runtime `{runtime_key}` on a torch that cannot run GPU work \ + with this SDK, on a host where a GPU was found. The ROCm SDK itself is installed; read \ + the device check above and install a torch build that suits both this SDK and this GPU." + ))) +} + +/// Whether a failed engine auto-install should fail `rocm install sdk` itself. +/// +/// Only the unusable-runtime case does. Everything else — an unreachable engine +/// index, a resolver error, a missing build tool — leaves the SDK installed and +/// usable, so it warns and keeps the successful exit rather than throwing away +/// the install that did succeed. +fn engine_auto_install_failure_is_fatal(error: &anyhow::Error) -> bool { + error.is::() +} + +/// Everything `install sdk` does once the SDK itself is on disk: auto-install the +/// family's engine, record the install, and decide the exit code. +/// +/// The engine auto-install is a parameter so that decision is reachable from a +/// test without a multi-gigabyte install behind it. It is the part worth pinning: +/// a failed engine install must keep the successful exit, because the SDK is +/// installed and usable and only a separately retryable step is missing, while a +/// runtime this install left unable to open a device must not — reporting success +/// for that is the defect being fixed. The SDK's own audit event is recorded +/// either way, before the command fails, because the SDK install did complete. +fn finish_sdk_install( paths: &AppPaths, - manifest: &therock::InstalledRuntimeManifest, - replace: bool, + finalized: Option<&SdkInstallFinalization>, + audit_action: &str, + audit_detail: String, + auto_install: impl FnOnce(&AppPaths, &SdkInstallFinalization) -> Result<()>, ) -> Result<()> { - let manifest = manifest.clone().normalize_storage_paths(); - let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); - if registry_path.exists() && !replace { - bail!( - "runtime registry entry already exists: {}; pass --replace to overwrite it", - registry_path.display() + let mut unusable_runtime = None; + if let Some(finalized) = finalized + && let Err(error) = auto_install(paths, finalized) + { + record_cli_audit_event( + paths, + "engine", + "engine_auto_install", + "error", + format!( + "auto-install failed engine=vllm runtime_id={} family={}: {error}", + finalized.runtime_key, finalized.family + ), + None, ); + if engine_auto_install_failure_is_fatal(&error) { + unusable_runtime = Some(error); + } else { + eprintln!("warning: automatic vLLM install failed: {error}"); + eprintln!( + "warning: SDK install completed; you can run `rocm engines install vllm --runtime-id {}` after vLLM is available in that runtime", + finalized.runtime_key + ); + } + } + record_cli_audit_event(paths, "runtime", audit_action, "info", audit_detail, None); + if let Some(error) = unusable_runtime { + return Err(error); } - fs::create_dir_all( - registry_path - .parent() - .context("runtime registry path has no parent directory")?, - )?; - fs::write( - ®istry_path, - serde_json::to_vec_pretty(&manifest) - .context("failed to serialize runtime registry manifest")?, - ) - .with_context(|| format!("failed to write {}", registry_path.display()))?; Ok(()) } -fn recover_setup_runtime_registration( +/// Whether rocm-cli owns the torch in the runtime this install just produced. +/// +/// Two kinds of environment are left alone, because there is no torch here that +/// rocm-cli put in place. External environments are the obvious one. The subtler +/// one is an engine that manages its own runtime: those report +/// `managed_env: Some(true)` — rocm-cli did create the runtime — but their +/// `python_executable` is the engine's native binary, not an interpreter, and no +/// torch ever lived in there. Probing one anyway spawns that binary with a +/// generated `.py` path as `argv[1]` and prints check blocks about a package the +/// runtime never had. +fn settles_runtime_torch(engine: &str, managed_env: Option) -> bool { + managed_env != Some(false) && !engine_manages_own_runtime(engine) +} + +/// Settle which torch a managed runtime keeps after an engine install, then report. +/// +/// Every path that installs an engine into a managed runtime must call this. +/// Skipping it anywhere lets the engine's own torch win silently and can leave a +/// runtime that cannot serve — including after an explicit +/// `rocm engines install --reinstall`, which is exactly what someone +/// reaches for when a runtime already looks wrong. +/// +/// The runtime is asked what it can do before anything is written to it, and a +/// torch that already runs a GPU kernel is kept, whichever of the two intended +/// builds it is. Only when nothing has proven itself does the alignment install +/// run, and then the runtime is asked again, because that answer is now about a +/// different torch. +/// +/// See `settles_runtime_torch` for the environments this deliberately skips. +fn settle_engine_install( paths: &AppPaths, - config: &RocmCliConfig, -) -> Result> { - let Some(setup_root) = config - .setup - .therock_venv - .as_deref() - .filter(|path| !path.as_os_str().is_empty()) - else { - return Ok(None); - }; - if !setup_root.is_dir() { - return Ok(None); + engine: &str, + selector: &str, + response: &InstallResponse, +) -> Result<()> { + if !settles_runtime_torch(engine, response.managed_env) { + return Ok(()); } - - let local_manifest_path = setup_root.join(".rocm-cli-runtime.json"); - if !local_manifest_path.is_file() { - return Ok(None); + let python = Path::new(&response.python_executable); + // Which runtime is being settled has to be asked of the environment, not of + // the caller. Two of the three callers pass a `runtime_id`, and side-by-side + // installs of one channel and family share it, so the selector can name two + // runtimes at once; `runtime_manifest_for_selector` then correctly declines + // to guess and every lookup below silently comes back empty. It can also + // name the *active* runtime while the engine was installed into an older + // one, which is worse than empty — it settles the wrong tree's torch. + // The interpreter is unambiguous, so let it name its own runtime. + let owned = runtime_key_for_python(paths, python); + let runtime_key = owned.as_deref().unwrap_or(selector); + let library_paths = runtime_library_paths_for_key(paths, runtime_key); + let sdk_build = sdk_torch_build_for_key(paths, runtime_key); + let torch = therock::probe_torch_alignment(python, engine); + let probed = runtime_device_check(Some(python), &library_paths); + + let retention = classify_retained_torch( + sdk_build.as_deref(), + &probed, + torch + .as_ref() + .ok() + .and_then(|probe| probe.engine_requires_torch.as_deref()), + ); + let (diverged, devices) = match retention { + TorchRetention::Realign => { + let alignment = report_torch_alignment( + paths, + engine, + python, + runtime_key, + sdk_build.as_deref(), + torch.as_ref(), + &probed, + ); + // Only a realignment replaced torch. After every other outcome the + // environment is the one already probed, and asking it again would + // spend a second interpreter launch to be told the same thing. + let devices = match alignment { + TorchAlignment::Realigned { .. } => { + runtime_device_check(Some(python), &library_paths) + } + _ => probed, + }; + (deliberately_diverged_package(&alignment), devices) + } + retention => { + report_torch_retention(paths, engine, runtime_key, &retention); + (retained_diverged_package(&retention), probed) + } + }; + report_engine_dependency_check(paths, engine, Some(python), runtime_key, diverged); + let devices = report_runtime_device_check(paths, engine, runtime_key, devices); + + // Only a runtime that cannot serve raises the question the host answers, so + // the host is inspected only then. Both readers below already require this, + // so asking earlier would spend a full host scan on every healthy install to + // reach an answer nothing goes on to read. + if !runtime_cannot_serve(&devices) { + return Ok(()); } - let bytes = fs::read(&local_manifest_path) - .with_context(|| format!("failed to read {}", local_manifest_path.display()))?; - let manifest: therock::InstalledRuntimeManifest = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", local_manifest_path.display()))?; - if manifest.runtime_key.trim().is_empty() { - bail!( - "setup runtime manifest {} has an empty runtime_key", - local_manifest_path.display() - ); + // A runtime that cannot serve is only this install's failure where the host + // has a GPU to serve with. Where the host could not be examined, say so + // rather than failing a multi-gigabyte install on a guess. + let host_gpu = detect_host_gpu(); + if let HostGpu::NotVerified(reason) = &host_gpu { + report_unverified_host_gpu(paths, engine, runtime_key, reason); } - if !paths_equivalent(&manifest.install_root, setup_root) { - bail!( - "setup runtime manifest {} points at {}, but setup is configured for {}", - local_manifest_path.display(), - manifest.install_root.display(), - setup_root.display() - ); + if install_left_runtime_unusable(&devices, &host_gpu) { + return Err(unusable_runtime_error(engine, runtime_key)); } - validate_runtime_manifest_for_activation(&manifest).with_context(|| { - format!( - "setup runtime `{}` from {} is not usable", - manifest.runtime_key, - local_manifest_path.display() - ) - })?; + Ok(()) +} - if !runtime_manifest_path(paths, &manifest.runtime_key).is_file() { - write_runtime_registry_manifest(paths, &manifest, false).with_context(|| { - format!( - "failed to restore setup runtime `{}` into {}", - manifest.runtime_key, - runtime_registry_dir(paths).display() - ) - })?; +fn engine_dependency_check( + paths: &AppPaths, + engine: &str, + python: Option<&Path>, + realigned_package: Option<&str>, +) -> EngineDependencyCheck { + let Some(python) = python else { + return EngineDependencyCheck::NotVerified( + "the runtime's Python environment could not be located".to_owned(), + ); + }; + match rocm_core::check_dependencies(paths, python) { + Ok(violations) => { + let owned = rocm_core::violations_requiring(&violations, engine); + let details: Vec = owned + .iter() + .map(|violation| violation.detail.clone()) + .collect(); + classify_dependency_details(details, realigned_package) + } + Err(error) => EngineDependencyCheck::NotVerified(error.to_string()), } - Ok(Some(manifest.runtime_key)) } -fn current_runtime_manifest<'a>( - config: &RocmCliConfig, - manifests: &'a [therock::InstalledRuntimeManifest], -) -> Option<&'a therock::InstalledRuntimeManifest> { - if let Some(active_key) = config.active_runtime_key.as_deref() - && let Some(manifest) = manifests - .iter() - .find(|manifest| manifest.runtime_key.eq_ignore_ascii_case(active_key)) - { - return Some(manifest); +/// Separate a deliberate divergence from a genuine violation. +/// +/// Each line is judged on its own subject: a genuine violation elsewhere in the +/// same run says nothing about the package the install deliberately diverged on, +/// and must not drag it along. Lumping the two together loses the distinction +/// exactly where it matters most — the remedy for the genuine violation would +/// reinstall the engine and undo the alignment. +fn classify_dependency_details( + details: Vec, + realigned_package: Option<&str>, +) -> EngineDependencyCheck { + if details.is_empty() { + return EngineDependencyCheck::Satisfied; } - - let matches = default_runtime_id_matches(config, manifests); - match matches.as_slice() { - [manifest] => Some(*manifest), - _ => None, + let Some(package) = realigned_package else { + return EngineDependencyCheck::Violated { + violations: details, + expected: Vec::new(), + }; + }; + // A line whose subject cannot be parsed is not evidence of a divergence, so it + // stays a violation: the conservative side keeps saying something is wrong. + let (expected, violations): (Vec, Vec) = + details.into_iter().partition(|detail| { + rocm_core::violation_subject(detail) + .is_some_and(|subject| subject.package.eq_ignore_ascii_case(package)) + }); + if violations.is_empty() { + EngineDependencyCheck::ExpectedDivergence(expected) + } else { + EngineDependencyCheck::Violated { + violations, + expected, + } } } -fn default_runtime_id_matches<'a>( - config: &RocmCliConfig, - manifests: &'a [therock::InstalledRuntimeManifest], -) -> Vec<&'a therock::InstalledRuntimeManifest> { - let Some(default_runtime_id) = config.default_runtime_id.as_deref() else { - return Vec::new(); - }; - manifests - .iter() - .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(default_runtime_id)) - .collect() -} - -fn runtime_keys_text(manifests: &[&therock::InstalledRuntimeManifest]) -> String { - manifests - .iter() - .map(|manifest| manifest.runtime_key.as_str()) - .collect::>() - .join(", ") -} - -fn select_runtime_manifest<'a>( - manifests: &'a [therock::InstalledRuntimeManifest], - selector: &str, -) -> Result<&'a therock::InstalledRuntimeManifest> { - let selector = selector.trim(); - if selector.is_empty() { - bail!("runtime selector must not be empty"); - } - - if let Some(manifest) = manifests - .iter() - .find(|manifest| manifest.runtime_key.eq_ignore_ascii_case(selector)) - { - return Ok(manifest); - } - - let matches = manifests - .iter() - .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(selector)) - .collect::>(); - match matches.as_slice() { - [manifest] => Ok(manifest), - [] => bail!("installed runtime not found: {selector}"), - _ => { - let keys = matches - .iter() - .map(|manifest| manifest.runtime_key.as_str()) - .collect::>() - .join(", "); - bail!( - "runtime selector `{selector}` matches multiple installed runtimes; activate one by runtime_key: {keys}" +fn render_engine_dependency_check(engine: &str, outcome: &EngineDependencyCheck) -> String { + let mut output = String::new(); + match outcome { + EngineDependencyCheck::Satisfied => { + let _ = writeln!(output, " dependency_check: satisfied"); + } + EngineDependencyCheck::NotVerified(reason) => { + let _ = writeln!( + output, + " dependency_check: not_verified ({})", + sanitize_log_value(reason) + ); + } + EngineDependencyCheck::Violated { + violations, + expected, + } => { + let _ = writeln!(output, " dependency_check: violated"); + for detail in violations { + let _ = writeln!(output, " violation: {}", sanitize_log_value(detail)); + } + for detail in expected { + let _ = writeln!(output, " divergence: {}", sanitize_log_value(detail)); + } + if expected.is_empty() { + let _ = writeln!( + output, + " action: rocm engines install {engine} --reinstall" + ); + } else { + // The reinstall would repair the violations above and reinstate the + // engine's own build of the diverged package, restoring a runtime + // that cannot open a device. Naming a remedy that trades one defect + // for a worse one is not worth the convenience. + let _ = writeln!( + output, + " action: repair the violations above without reinstalling {engine}; a reinstall would undo the divergence kept on purpose (see torch_alignment above)" + ); + } + } + EngineDependencyCheck::ExpectedDivergence(details) => { + let _ = writeln!(output, " dependency_check: expected_divergence"); + for detail in details { + let _ = writeln!(output, " divergence: {}", sanitize_log_value(detail)); + } + // Deliberately not the reinstall remedy. It does not apply to + // either state that reaches here: a reinstall realigns torch again + // and reproduces the SDK build, and where the user has opted out of + // that realignment it would undo the decision they made. Which of + // the two this is, the block above has already said. + let _ = writeln!( + output, + " action: none; this torch is kept on purpose (see torch_alignment above)" ); } } + output } -pub(crate) fn runtime_usability_status(manifest: &therock::InstalledRuntimeManifest) -> String { - match validate_runtime_manifest_for_activation(manifest) { - Ok(()) => "ready".to_owned(), - Err(error) => format!("unusable ({error})"), - } +fn render_sdk_install_success(finalized: &SdkInstallFinalization) -> String { + format!( + "ROCm SDK installed successfully.\n install folder: {}\n active runtime: {}\n next step: run `rocm help` to see how to use rocm-cli.\n", + finalized.install_root.display(), + finalized.runtime_key + ) } -fn validate_runtime_manifest_for_activation( - manifest: &therock::InstalledRuntimeManifest, -) -> Result<()> { - if manifest.runtime_key.trim().is_empty() { - bail!("manifest runtime_key is empty"); - } - if manifest.runtime_id.trim().is_empty() { - bail!("manifest runtime_id is empty"); - } - if !manifest.install_root.is_dir() { - bail!( - "install root is missing: {}", - manifest.install_root.display() - ); - } - let local_manifest = manifest.install_root.join(".rocm-cli-runtime.json"); - if !manifest.read_only && !local_manifest.is_file() { - bail!( - "local runtime manifest is missing: {}", - local_manifest.display() - ); - } +fn finalize_successful_sdk_install(paths: &AppPaths) -> Result> { + let Some(manifest) = newest_installed_runtime_manifest(paths)? else { + return Ok(None); + }; + let mut config = RocmCliConfig::load(paths)?; + config.setup.completed = true; + config.setup.therock_venv = Some(manifest.install_root.clone()); + config.save(paths)?; - match manifest.format.as_str() { - "wheel" => validate_wheel_runtime_manifest(manifest), - "tarball" => validate_tarball_runtime_manifest(manifest), - other => bail!("unsupported runtime format in manifest: {other}"), + let activation_paths = paths + .clone() + .with_managed_root(manifest.install_root.clone(), false); + if paths.config_dir != activation_paths.config_dir + || paths.data_dir != activation_paths.data_dir + { + recover_setup_runtime_registration(paths, &config)?; + let mut current_config = RocmCliConfig::load(paths)?; + current_config.setup.completed = true; + current_config.setup.therock_venv = Some(manifest.install_root.clone()); + let _ = activate_runtime(paths, &mut current_config, &manifest.runtime_key)?; } -} -fn validate_wheel_runtime_manifest(manifest: &therock::InstalledRuntimeManifest) -> Result<()> { - let python_executable = manifest - .python_executable - .as_deref() - .context("pip runtime manifest is missing python_executable")?; - if !Path::new(python_executable).is_file() { - bail!("runtime Python executable is missing: {python_executable}"); - } - let probe = manifest - .rocm_sdk - .as_ref() - .context("pip runtime manifest is missing rocm_sdk probe data")?; - therock::validate_rocm_sdk_runtime_probe(probe)?; - Ok(()) -} + recover_setup_runtime_registration(&activation_paths, &config)?; -fn validate_tarball_runtime_manifest(manifest: &therock::InstalledRuntimeManifest) -> Result<()> { - if runtime_install_root_has_payload(&manifest.install_root)? { - Ok(()) - } else { - bail!( - "tarball runtime install root has no payload files: {}", - manifest.install_root.display() - ) - } -} + let mut config = RocmCliConfig::load(&activation_paths)?; + config.setup.completed = true; + config.setup.therock_venv = Some(manifest.install_root.clone()); + let activation = activate_runtime(&activation_paths, &mut config, &manifest.runtime_key)?; -fn runtime_install_root_has_payload(path: &Path) -> Result { - for entry in fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))? { - let entry = entry?; - if !entry.file_name().to_string_lossy().starts_with('.') { - return Ok(true); - } - } - Ok(false) + Ok(Some(SdkInstallFinalization { + runtime_key: activation.runtime_key, + install_root: manifest.install_root, + family: manifest.family, + })) } -fn runtime_registry_dir(paths: &AppPaths) -> PathBuf { - paths.data_dir.join("runtimes").join("registry") +fn newest_installed_runtime_manifest( + paths: &AppPaths, +) -> Result> { + let mut manifests = therock::load_runtime_manifests(paths)?; + manifests.sort_by(|left, right| { + right + .installed_at_unix_ms + .cmp(&left.installed_at_unix_ms) + .then_with(|| left.runtime_key.cmp(&right.runtime_key)) + }); + Ok(manifests.into_iter().next()) } -fn runtime_manifest_path(paths: &AppPaths, runtime_key: &str) -> PathBuf { - runtime_registry_dir(paths).join(format!("{runtime_key}.json")) +#[derive(Debug, Clone)] +struct AdoptRuntimeRequest { + python_executable: PathBuf, + install_root: PathBuf, + runtime_id: String, + runtime_key: String, + replace: bool, } -fn active_runtime_marker_path(paths: &AppPaths) -> PathBuf { - paths.data_dir.join("runtimes").join("active.json") +#[derive(Debug, Clone)] +struct AdoptRuntimeOptions { + python_input: PathBuf, + install_root: Option, + runtime_id: Option, + runtime_key: Option, + channel: Option, + replace: bool, } -fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { - let path = active_runtime_marker_path(paths); - fs::create_dir_all( - path.parent() - .context("active runtime marker path has no parent directory")?, +fn adopt_runtime_from_python_options( + paths: &AppPaths, + options: AdoptRuntimeOptions, +) -> Result { + let (python_executable, inferred_root) = resolve_adopt_python_input(&options.python_input)?; + let probe = therock::probe_rocm_sdk_runtime(&python_executable) + .with_context(|| format!("failed to probe {}", python_executable.display()))?; + let request = infer_adopt_runtime_request( + python_executable, + options.install_root.or(inferred_root), + options.runtime_id, + options.runtime_key, + options.channel, + options.replace, + &probe, )?; - let tmp_path = path.with_extension(format!("json.tmp-{}", rocm_core::unix_time_millis())); - fs::write( - &tmp_path, - serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?, - ) - .with_context(|| format!("failed to write {}", tmp_path.display()))?; - if path.exists() { - let _ = fs::remove_file(&path); - } - fs::rename(&tmp_path, &path).with_context(|| { + adopt_runtime_from_probe(paths, request, probe) +} + +fn infer_adopt_runtime_request( + python_executable: PathBuf, + install_root: Option, + runtime_id: Option, + runtime_key: Option, + channel: Option, + replace: bool, + probe: &therock::RocmSdkPythonProbe, +) -> Result { + let install_root = install_root.with_context(|| { format!( - "failed to move active runtime marker {} into {}", - tmp_path.display(), - path.display() + "could not infer the Python environment folder from {}; pass --root", + python_executable.display() ) })?; - Ok(()) -} - -fn config(command: ConfigCommand) -> Result<()> { + let runtime_id = match runtime_id { + Some(value) if !value.trim().is_empty() => { + if let Some(channel) = channel.as_deref() { + let (parsed_channel, _) = parse_therock_runtime_id(&value)?; + let requested_channel = normalize_adopt_channel(channel)?; + if parsed_channel != requested_channel { + bail!( + "--channel {requested_channel} does not match runtime id channel {parsed_channel}" + ); + } + } + value + } + Some(_) => bail!("runtime_id must not be empty"), + None => { + let channel = normalize_adopt_channel(channel.as_deref().unwrap_or("release"))?; + let family = probe + .resolved_target_family + .as_deref() + .or(probe.default_target_family.as_deref()) + .filter(|value| !value.trim().is_empty()) + .context( + "rocm_sdk probe did not report a GPU package; pass --runtime-id explicitly", + )?; + format!("therock-{channel}:{family}") + } + }; + let runtime_key = match runtime_key { + Some(value) if !value.trim().is_empty() => value, + Some(_) => bail!("runtime_key must not be empty"), + None => { + let (channel, family) = parse_therock_runtime_id(&runtime_id)?; + let version = probe + .rocm_sdk_version + .as_deref() + .filter(|value| !value.trim().is_empty()) + .context("rocm_sdk probe did not report a version; cannot name adopted runtime")?; + format!( + "adopted-{channel}-pip-{}-{}", + runtime_key_component(&family), + runtime_key_component(version) + ) + } + }; + Ok(AdoptRuntimeRequest { + python_executable, + install_root, + runtime_id, + runtime_key, + replace, + }) +} + +fn normalize_adopt_channel(channel: &str) -> Result { + match channel.trim().to_ascii_lowercase().as_str() { + "release" => Ok("release".to_owned()), + "nightly" => Ok("nightly".to_owned()), + other => bail!("adopt channel must be release or nightly, got `{other}`"), + } +} + +fn runtime_key_component(value: &str) -> String { + let mut output = String::new(); + let mut last_dash = false; + for ch in value.chars() { + if ch.is_ascii_alphanumeric() { + output.push(ch.to_ascii_lowercase()); + last_dash = false; + } else if !last_dash { + output.push('-'); + last_dash = true; + } + } + let trimmed = output.trim_matches('-').to_owned(); + if trimmed.is_empty() { + "runtime".to_owned() + } else { + trimmed + } +} + +fn resolve_adopt_python_input(input: &Path) -> Result<(PathBuf, Option)> { + let absolute = if input.is_absolute() { + input.to_path_buf() + } else { + std::env::current_dir() + .context("failed to resolve current directory")? + .join(input) + }; + if absolute.is_dir() { + let env_root = absolute.canonicalize().with_context(|| { + format!( + "failed to resolve Python environment folder {}", + absolute.display() + ) + })?; + let python = runtime_python_executable_in_env(&env_root); + if !python.is_file() { + bail!( + "Python executable is missing in {}", + python.parent().unwrap_or(env_root.as_path()).display() + ); + } + return Ok((python, Some(env_root))); + } + if absolute.is_file() { + let inferred_root = infer_python_env_root(&absolute); + return Ok((absolute, inferred_root)); + } + bail!( + "Python executable or folder is missing: {}", + absolute.display() + ); +} + +fn infer_python_env_root(python_executable: &Path) -> Option { + let bin_dir = python_executable.parent()?; + let bin_name = bin_dir.file_name()?.to_string_lossy(); + if bin_name.eq_ignore_ascii_case("Scripts") || bin_name == "bin" { + return bin_dir.parent().map(Path::to_path_buf); + } + bin_dir.parent().map(Path::to_path_buf) +} + +fn adopt_runtime_from_probe( + paths: &AppPaths, + request: AdoptRuntimeRequest, + probe: therock::RocmSdkPythonProbe, +) -> Result { + if request.runtime_key.trim().is_empty() { + bail!("runtime_key must not be empty"); + } + if request.runtime_id.trim().is_empty() { + bail!("runtime_id must not be empty"); + } + let python_executable = absolute_existing_file_path_preserving_symlink( + &request.python_executable, + "runtime Python executable", + )?; + let install_root = request.install_root.canonicalize().with_context(|| { + format!( + "runtime install root is missing: {}", + request.install_root.display() + ) + })?; + if !install_root.is_dir() { + bail!( + "runtime install root is missing: {}", + install_root.display() + ); + } + let (channel, family) = parse_therock_runtime_id(&request.runtime_id)?; + therock::validate_rocm_sdk_runtime_probe(&probe)?; + let version = probe + .rocm_sdk_version + .as_deref() + .filter(|value| !value.trim().is_empty()) + .context("rocm_sdk probe did not report a version; cannot adopt runtime explicitly")? + .to_owned(); + + let manifest = therock::InstalledRuntimeManifest { + runtime_key: request.runtime_key, + runtime_id: request.runtime_id, + channel, + format: "wheel".to_owned(), + family, + family_source: "runtime_id".to_owned(), + version, + install_root: install_root.clone(), + selected_artifact_url: "adopted-read-only".to_owned(), + index_url: None, + tarball_file_name: None, + python_launcher: None, + python_executable: Some(python_executable.display().to_string()), + pip_cache_dir: None, + rocm_sdk: Some(probe), + // Adoption does not install torch, so the build is derived from the SDK + // version instead. + sdk_torch: None, + read_only: true, + imported_from: Some(install_root), + installed_at_unix_ms: rocm_core::unix_time_millis(), + }; + validate_runtime_manifest_for_activation(&manifest) + .with_context(|| format!("adopted runtime `{}` is not usable", manifest.runtime_key))?; + write_runtime_registry_manifest(paths, &manifest, request.replace)?; + Ok(manifest) +} + +fn absolute_existing_file_path_preserving_symlink(path: &Path, label: &str) -> Result { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .context("failed to resolve current directory")? + .join(path) + }; + if !absolute.is_file() { + bail!("{label} is missing: {}", absolute.display()); + } + Ok(absolute) +} + +fn parse_therock_runtime_id(runtime_id: &str) -> Result<(String, String)> { + let (prefix, family) = runtime_id.split_once(':').with_context(|| { + format!("runtime_id `{runtime_id}` must include a TheRock family suffix after ':'") + })?; + let family = family.trim(); + if family.is_empty() { + bail!("runtime_id `{runtime_id}` has an empty TheRock family suffix"); + } + let channel = match prefix.trim() { + "therock-release" => "release", + "therock-nightly" => "nightly", + other => bail!( + "runtime_id `{runtime_id}` must start with therock-release: or therock-nightly:, got `{other}`" + ), + }; + Ok((channel.to_owned(), family.to_owned())) +} + +fn write_runtime_registry_manifest( + paths: &AppPaths, + manifest: &therock::InstalledRuntimeManifest, + replace: bool, +) -> Result<()> { + let manifest = manifest.clone().normalize_storage_paths(); + let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); + if registry_path.exists() && !replace { + bail!( + "runtime registry entry already exists: {}; pass --replace to overwrite it", + registry_path.display() + ); + } + fs::create_dir_all( + registry_path + .parent() + .context("runtime registry path has no parent directory")?, + )?; + fs::write( + ®istry_path, + serde_json::to_vec_pretty(&manifest) + .context("failed to serialize runtime registry manifest")?, + ) + .with_context(|| format!("failed to write {}", registry_path.display()))?; + Ok(()) +} + +fn recover_setup_runtime_registration( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result> { + let Some(setup_root) = config + .setup + .therock_venv + .as_deref() + .filter(|path| !path.as_os_str().is_empty()) + else { + return Ok(None); + }; + if !setup_root.is_dir() { + return Ok(None); + } + + let local_manifest_path = setup_root.join(".rocm-cli-runtime.json"); + if !local_manifest_path.is_file() { + return Ok(None); + } + + let bytes = fs::read(&local_manifest_path) + .with_context(|| format!("failed to read {}", local_manifest_path.display()))?; + let manifest: therock::InstalledRuntimeManifest = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", local_manifest_path.display()))?; + if manifest.runtime_key.trim().is_empty() { + bail!( + "setup runtime manifest {} has an empty runtime_key", + local_manifest_path.display() + ); + } + if !paths_equivalent(&manifest.install_root, setup_root) { + bail!( + "setup runtime manifest {} points at {}, but setup is configured for {}", + local_manifest_path.display(), + manifest.install_root.display(), + setup_root.display() + ); + } + validate_runtime_manifest_for_activation(&manifest).with_context(|| { + format!( + "setup runtime `{}` from {} is not usable", + manifest.runtime_key, + local_manifest_path.display() + ) + })?; + + if !runtime_manifest_path(paths, &manifest.runtime_key).is_file() { + write_runtime_registry_manifest(paths, &manifest, false).with_context(|| { + format!( + "failed to restore setup runtime `{}` into {}", + manifest.runtime_key, + runtime_registry_dir(paths).display() + ) + })?; + } + Ok(Some(manifest.runtime_key)) +} + +fn current_runtime_manifest<'a>( + config: &RocmCliConfig, + manifests: &'a [therock::InstalledRuntimeManifest], +) -> Option<&'a therock::InstalledRuntimeManifest> { + if let Some(active_key) = config.active_runtime_key.as_deref() + && let Some(manifest) = manifests + .iter() + .find(|manifest| manifest.runtime_key.eq_ignore_ascii_case(active_key)) + { + return Some(manifest); + } + + let matches = default_runtime_id_matches(config, manifests); + match matches.as_slice() { + [manifest] => Some(*manifest), + _ => None, + } +} + +fn default_runtime_id_matches<'a>( + config: &RocmCliConfig, + manifests: &'a [therock::InstalledRuntimeManifest], +) -> Vec<&'a therock::InstalledRuntimeManifest> { + let Some(default_runtime_id) = config.default_runtime_id.as_deref() else { + return Vec::new(); + }; + manifests + .iter() + .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(default_runtime_id)) + .collect() +} + +fn runtime_keys_text(manifests: &[&therock::InstalledRuntimeManifest]) -> String { + manifests + .iter() + .map(|manifest| manifest.runtime_key.as_str()) + .collect::>() + .join(", ") +} + +fn select_runtime_manifest<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + selector: &str, +) -> Result<&'a therock::InstalledRuntimeManifest> { + let selector = selector.trim(); + if selector.is_empty() { + bail!("runtime selector must not be empty"); + } + + if let Some(manifest) = manifests + .iter() + .find(|manifest| manifest.runtime_key.eq_ignore_ascii_case(selector)) + { + return Ok(manifest); + } + + let matches = manifests + .iter() + .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(selector)) + .collect::>(); + match matches.as_slice() { + [manifest] => Ok(manifest), + [] => bail!("installed runtime not found: {selector}"), + _ => { + let keys = matches + .iter() + .map(|manifest| manifest.runtime_key.as_str()) + .collect::>() + .join(", "); + bail!( + "runtime selector `{selector}` matches multiple installed runtimes; activate one by runtime_key: {keys}" + ); + } + } +} + +pub(crate) fn runtime_usability_status(manifest: &therock::InstalledRuntimeManifest) -> String { + match validate_runtime_manifest_for_activation(manifest) { + Ok(()) => "ready".to_owned(), + Err(error) => format!("unusable ({error})"), + } +} + +fn validate_runtime_manifest_for_activation( + manifest: &therock::InstalledRuntimeManifest, +) -> Result<()> { + if manifest.runtime_key.trim().is_empty() { + bail!("manifest runtime_key is empty"); + } + if manifest.runtime_id.trim().is_empty() { + bail!("manifest runtime_id is empty"); + } + if !manifest.install_root.is_dir() { + bail!( + "install root is missing: {}", + manifest.install_root.display() + ); + } + let local_manifest = manifest.install_root.join(".rocm-cli-runtime.json"); + if !manifest.read_only && !local_manifest.is_file() { + bail!( + "local runtime manifest is missing: {}", + local_manifest.display() + ); + } + + match manifest.format.as_str() { + "wheel" => validate_wheel_runtime_manifest(manifest), + "tarball" => validate_tarball_runtime_manifest(manifest), + other => bail!("unsupported runtime format in manifest: {other}"), + } +} + +fn validate_wheel_runtime_manifest(manifest: &therock::InstalledRuntimeManifest) -> Result<()> { + let python_executable = manifest + .python_executable + .as_deref() + .context("pip runtime manifest is missing python_executable")?; + if !Path::new(python_executable).is_file() { + bail!("runtime Python executable is missing: {python_executable}"); + } + let probe = manifest + .rocm_sdk + .as_ref() + .context("pip runtime manifest is missing rocm_sdk probe data")?; + therock::validate_rocm_sdk_runtime_probe(probe)?; + Ok(()) +} + +fn validate_tarball_runtime_manifest(manifest: &therock::InstalledRuntimeManifest) -> Result<()> { + if runtime_install_root_has_payload(&manifest.install_root)? { + Ok(()) + } else { + bail!( + "tarball runtime install root has no payload files: {}", + manifest.install_root.display() + ) + } +} + +fn runtime_install_root_has_payload(path: &Path) -> Result { + for entry in fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))? { + let entry = entry?; + if !entry.file_name().to_string_lossy().starts_with('.') { + return Ok(true); + } + } + Ok(false) +} + +fn runtime_registry_dir(paths: &AppPaths) -> PathBuf { + paths.data_dir.join("runtimes").join("registry") +} + +fn runtime_manifest_path(paths: &AppPaths, runtime_key: &str) -> PathBuf { + runtime_registry_dir(paths).join(format!("{runtime_key}.json")) +} + +fn active_runtime_marker_path(paths: &AppPaths) -> PathBuf { + paths.data_dir.join("runtimes").join("active.json") +} + +fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { + let path = active_runtime_marker_path(paths); + fs::create_dir_all( + path.parent() + .context("active runtime marker path has no parent directory")?, + )?; + let tmp_path = path.with_extension(format!("json.tmp-{}", rocm_core::unix_time_millis())); + fs::write( + &tmp_path, + serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?, + ) + .with_context(|| format!("failed to write {}", tmp_path.display()))?; + if path.exists() { + let _ = fs::remove_file(&path); + } + fs::rename(&tmp_path, &path).with_context(|| { + format!( + "failed to move active runtime marker {} into {}", + tmp_path.display(), + path.display() + ) + })?; + Ok(()) +} + +fn config(command: ConfigCommand) -> Result<()> { let paths = AppPaths::discover()?; let mut config = RocmCliConfig::load(&paths)?; @@ -8286,7 +9494,7 @@ fn config(command: ConfigCommand) -> Result<()> { bail!("local provider does not use a cloud API key"); } let key = read_provider_key_from_user(provider)?; - let status = provider_keys::set_provider_api_key(provider, &key)?; + let status = provider_keys::store_provider_credential(provider, &key)?; println!("{provider} API key saved"); println!( " key: {}", @@ -8309,7 +9517,7 @@ fn config(command: ConfigCommand) -> Result<()> { if provider == "local" { bail!("local provider does not use a cloud API key"); } - let status = provider_keys::clear_provider_api_key(provider)?; + let status = provider_keys::remove_provider_credential(provider)?; println!("{provider} API key cleared"); println!( " key: {}", @@ -21882,4125 +23090,5278 @@ install therock"; } #[test] - fn chat_how_to_setup_question_opens_install_folder_flow() { - assert!( - install_sdk_without_prefix_chat_approval("How do I setup TheRock?").is_some(), - "a setup question should ask for the install folder" - ); - assert!( - install_sdk_without_prefix_chat_approval("install therock").is_some(), - "a direct install command should open the folder picker" - ); + fn chat_how_to_setup_question_opens_install_folder_flow() { + assert!( + install_sdk_without_prefix_chat_approval("How do I setup TheRock?").is_some(), + "a setup question should ask for the install folder" + ); + assert!( + install_sdk_without_prefix_chat_approval("install therock").is_some(), + "a direct install command should open the folder picker" + ); + } + + #[test] + fn chat_install_intent_preserves_bare_folder_path() { + let approval = + install_sdk_chat_approval_for_prompt("install therock D:\\ROCm\\therock_venvs") + .expect("direct install prompt should be recognized"); + + assert_eq!( + approval.args, + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + ] + ); + } + + #[test] + fn fallback_tool_call_routes_requested_therock_exact_version_install() { + let call = fallback_rocm_tool_call_for_prompt( + "Install the TheRock ROCm wheel version 7.13.0a20260605 into D:\\ROCm\\therock_venvs", + ) + .unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + "--version".to_owned(), + "7.13.0a20260605".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + + #[test] + fn path_exists_chat_tool_is_read_only() { + let call = providers::ChatToolCall { + id: Some("path-check".to_owned()), + name: "path_exists".to_owned(), + arguments: serde_json::json!({ "path": "D:\\ROCm\\temp" }), + }; + + validate_chat_tool_call(&call).unwrap(); + assert!(chat_tool_call_is_read_only(&call)); + } + + #[test] + fn port_status_chat_tool_is_read_only_and_loopback_only() { + let call = providers::ChatToolCall { + id: Some("port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "127.0.0.1", "port": 8188 }), + }; + + validate_chat_tool_call(&call).unwrap(); + assert!(chat_tool_call_is_read_only(&call)); + + let public = providers::ChatToolCall { + id: Some("public-port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "192.168.1.10", "port": 8188 }), + }; + let error = validate_chat_tool_call(&public).unwrap_err().to_string(); + assert!(error.contains("non-local host"), "{error}"); + } + + #[test] + fn port_status_matches_loopback_managed_services() -> Result<()> { + let (root, paths) = test_paths("port-status-loopback"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-comfyui", + "comfyui", + "ComfyUI", + "ComfyUI", + "127.0.0.1", + 18188, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + record.write()?; + + let call = providers::ChatToolCall { + id: Some("port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "localhost", "port": 18188 }), + }; + let result = run_chat_port_status_tool(&paths, &call)?; + let text = mcp_tool_result_text(&result); + let managed_service_count = result + .get("structuredContent") + .and_then(|content| content.get("managed_services")) + .and_then(serde_json::Value::as_array) + .map_or(0, Vec::len); + let _ = fs::remove_dir_all(root); + + assert_eq!(managed_service_count, 1); + assert!(text.contains("managed_services:"), "{text}"); + assert!(text.contains("service_id=svc-comfyui"), "{text}"); + assert!(text.contains("running_state=starting"), "{text}"); + Ok(()) + } + + #[test] + fn fallback_tool_call_routes_simple_config_changes() { + let show = fallback_rocm_tool_call_for_prompt("Show current ROCm CLI config").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&show).unwrap(), + vec!["config".to_owned(), "show".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&show)); + + let engine = fallback_rocm_tool_call_for_prompt("Set the default engine to vllm").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&engine).unwrap(), + vec![ + "config".to_owned(), + "set-default-engine".to_owned(), + "vllm".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&engine)); + + let telemetry = + fallback_rocm_tool_call_for_prompt("Disable telemetry in settings").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&telemetry).unwrap(), + vec![ + "config".to_owned(), + "set-telemetry".to_owned(), + "off".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&telemetry)); + } + + #[test] + fn fallback_tool_call_routes_comfyui_support_and_actions() { + let status = fallback_rocm_tool_call_for_prompt("How do I setup ComfyUI?").unwrap(); + assert_eq!(status.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&status).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&status)); + + let install = fallback_rocm_tool_call_for_prompt("Can you setup ComfyUI for me?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&install).unwrap(), + vec!["comfyui".to_owned(), "install".to_owned()] + ); + assert!(!chat_tool_call_is_read_only(&install)); + let approval = + chat_tool_approval_request(&install, Some("Install ComfyUI after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Install ComfyUI"); + + let start = fallback_rocm_tool_call_for_prompt("Can you start ComfyUI?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&start).unwrap(), + vec!["comfyui".to_owned(), "start".to_owned()] + ); + assert!(!chat_tool_call_is_read_only(&start)); + let approval = + chat_tool_approval_request(&start, Some("Start ComfyUI after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Start ComfyUI"); + } + + #[test] + fn fallback_tool_call_routes_local_llm_serve_requests() { + let call = + fallback_rocm_tool_call_for_prompt("Can you setup and serve an LLM for me?").unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "serve".to_owned(), + "qwen".to_owned(), + "--engine".to_owned(), + "lemonade".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + let approval = + chat_tool_approval_request(&call, Some("Start qwen after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Start local model server"); + assert_eq!( + rocm_chat_tool_requested_command(&call).as_deref(), + Some("rocm serve qwen --engine lemonade --device gpu_required --managed") + ); + } + + #[test] + fn local_chat_tool_call_content_is_treated_as_intermediate() { + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "Qwen/Qwen3-0.6B".to_owned(), + content: "The active runtime root is /opt/rocm.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: Some("call-1".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }], + }; + + assert!(local_tool_call_content_is_intermediate( + "local", true, &response + )); + assert!(!local_tool_call_content_is_intermediate( + "openai", true, &response + )); + assert!(!local_tool_call_content_is_intermediate( + "local", false, &response + )); + + let without_tools = providers::ChatResponse { + tool_calls: Vec::new(), + ..response + }; + assert!(!local_tool_call_content_is_intermediate( + "local", + true, + &without_tools + )); + } + + #[test] + fn local_chat_follow_up_with_tool_call_is_not_final_answer() { + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "Qwen/Qwen3-0.6B".to_owned(), + content: "The runtime root is /opt/rocml.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: Some("call-2".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }], + }; + + assert!(!local_follow_up_content_is_final(&response)); + + let final_answer = providers::ChatResponse { + tool_calls: Vec::new(), + content: "The runtime root is D:\\ROCm\\therock_venvs.".to_owned(), + ..response + }; + assert!(local_follow_up_content_is_final(&final_answer)); } #[test] - fn chat_install_intent_preserves_bare_folder_path() { - let approval = - install_sdk_chat_approval_for_prompt("install therock D:\\ROCm\\therock_venvs") - .expect("direct install prompt should be recognized"); - + fn visible_chat_content_removes_reasoning_blocks() { assert_eq!( - approval.args, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - ] + visible_chat_content( + "\nchecking the tool output\n\nThe runtime root is D:\\ROCm\\temp." + ), + "The runtime root is D:\\ROCm\\temp." ); + assert_eq!( + visible_chat_content("Before\nhidden\nAfter"), + "Before\n\nAfter" + ); + assert_eq!(visible_chat_content("unfinished"), ""); } #[test] - fn fallback_tool_call_routes_requested_therock_exact_version_install() { - let call = fallback_rocm_tool_call_for_prompt( - "Install the TheRock ROCm wheel version 7.13.0a20260605 into D:\\ROCm\\therock_venvs", - ) - .unwrap(); - assert_eq!(call.name, "rocm_command"); + fn chat_tool_result_errors_use_plain_failure_wording() { + assert_eq!(chat_read_only_tool_status_label(false), "done"); + assert_eq!(chat_read_only_tool_status_label(true), "reported an error"); + assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - "--version".to_owned(), - "7.13.0a20260605".to_owned(), - ] + chat_tool_display_label("gpu_snapshot"), + "Checked GPU status" ); - assert!(!chat_tool_call_is_read_only(&call)); + assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); + assert!(mcp_tool_result_is_error(&serde_json::json!({ + "isError": true + }))); + assert!(!mcp_tool_result_is_error(&serde_json::json!({ + "isError": false + }))); + assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); } #[test] - fn path_exists_chat_tool_is_read_only() { - let call = providers::ChatToolCall { - id: Some("path-check".to_owned()), - name: "path_exists".to_owned(), - arguments: serde_json::json!({ "path": "D:\\ROCm\\temp" }), - }; + fn local_chat_without_service_explains_serve_before_chat_without_llm_setup() { + let (_root, paths) = test_paths("local-chat-no-service-guidance"); + let result = + render_chat_prompt_result(&paths, "local", None, "Check this ROCm setup", true) + .expect("missing local assistant should render guidance"); + assert!(result.approval.is_none()); + let rendered = result.rendered; - validate_chat_tool_call(&call).unwrap(); - assert!(chat_tool_call_is_read_only(&call)); + assert!(rendered.contains("No local assistant is running yet.")); + assert!(rendered.contains("First-time ROCm setup does not need an LLM")); + assert!(rendered.contains("Recommended path:")); + assert!(rendered.contains("Advanced manual command")); + assert!(rendered.contains( + "rocm serve Qwen3-4B-Instruct-2507-GGUF --engine lemonade --device gpu_required --managed" + )); + assert!(!rendered.contains("sshleifer/tiny-gpt2")); + assert!(rendered.contains("rocm chat --tools --provider local --prompt")); + assert!(rendered.contains("Nothing was changed.")); + assert!(!rendered.contains("install sdk")); + assert!(!rendered.contains("setup TheRock with an LLM")); } #[test] - fn port_status_chat_tool_is_read_only_and_loopback_only() { - let call = providers::ChatToolCall { - id: Some("port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "127.0.0.1", "port": 8188 }), - }; + fn local_chat_status_prompts_use_read_only_tools_without_assistant() -> Result<()> { + let (_root, paths) = test_paths("local-chat-status-fallback"); - validate_chat_tool_call(&call).unwrap(); - assert!(chat_tool_call_is_read_only(&call)); + let running = + render_chat_prompt_result(&paths, "local", None, "Is vLLM running?", true)?.rendered; + assert!(!running.contains("No local assistant is running yet.")); + assert!(running.contains("Checked model servers: done"), "{running}"); + assert!(running.contains("ROCm CLI summary"), "{running}"); + assert!( + running.contains("Local model servers: none running under ROCm CLI."), + "{running}" + ); + assert!(running.contains("Nothing was changed.")); - let public = providers::ChatToolCall { - id: Some("public-port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "192.168.1.10", "port": 8188 }), - }; - let error = validate_chat_tool_call(&public).unwrap_err().to_string(); - assert!(error.contains("non-local host"), "{error}"); - } + let installed = + render_chat_prompt_result(&paths, "local", None, "Is vLLM installed?", true)?.rendered; + assert!(!installed.contains("No local assistant is running yet.")); + assert!(installed.contains("Engine runtimes:"), "{installed}"); + assert!(installed.contains("vLLM:"), "{installed}"); - #[test] - fn port_status_matches_loopback_managed_services() -> Result<()> { - let (root, paths) = test_paths("port-status-loopback"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( + let installed_and_running = render_chat_prompt_result( &paths, - "svc-comfyui", - "comfyui", - "ComfyUI", - "ComfyUI", - "127.0.0.1", - 18188, - "managed", - std::process::id(), - Some("therock-release".to_owned()), + "local", None, - Some("gpu_required".to_owned()), + "Is vLLM installed and is it running?", + true, + )? + .rendered; + assert!( + installed_and_running.contains("Checked local engines: done"), + "{installed_and_running}" + ); + assert!( + installed_and_running.contains("Checked model servers: done"), + "{installed_and_running}" + ); + assert!( + installed_and_running.contains("Engine runtimes:"), + "{installed_and_running}" + ); + assert!( + installed_and_running.contains("Local model servers: none running under ROCm CLI."), + "{installed_and_running}" ); - record.status = "ready".to_owned(); - record.write()?; - let call = providers::ChatToolCall { - id: Some("port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "localhost", "port": 18188 }), - }; - let result = run_chat_port_status_tool(&paths, &call)?; - let text = mcp_tool_result_text(&result); - let managed_service_count = result - .get("structuredContent") - .and_then(|content| content.get("managed_services")) - .and_then(serde_json::Value::as_array) - .map_or(0, Vec::len); - let _ = fs::remove_dir_all(root); + let port = render_chat_prompt_result( + &paths, + "local", + None, + "What is running on port 8188?", + true, + )? + .rendered; + assert!(!port.contains("No local assistant is running yet.")); + assert!(port.contains("Checked local port: done"), "{port}"); + assert!(port.contains("Port 8188:"), "{port}"); + Ok(()) + } - assert_eq!(managed_service_count, 1); - assert!(text.contains("managed_services:"), "{text}"); - assert!(text.contains("service_id=svc-comfyui"), "{text}"); - assert!(text.contains("running_state=starting"), "{text}"); + #[test] + fn chat_tools_anthropic_reaches_provider_opt_in_boundary() { + let (_root, paths) = test_paths("anthropic-chat-tools-opt-in"); + + let error = render_chat_prompt_result( + &paths, + "anthropic", + Some("claude-test"), + "Check this ROCm setup", + true, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("cloud provider `anthropic` is disabled")); + assert!(error.contains("rocm config enable-provider anthropic")); + assert!(!error.contains("OpenAI-compatible provider")); + } + + #[test] + fn freeform_execution_validation_rejects_provider_assisted_plans() -> Result<()> { + let content = r#"{ + "intent": "serve", + "tool_call": { + "tool": "rocm", + "args": ["serve", "sshleifer/tiny-gpt2", "--engine", "vllm", "--managed"] + } + }"#; + let plan = provider_planner_response_to_plan("start a local model", "local", content)?; + let action = plan_next_action(plan).expect("provider plan should have an action"); + + let error = validate_freeform_execution_action(&action) + .unwrap_err() + .to_string(); + + assert!(action.provider_assisted); + assert!(error.contains("reviewed interactively")); Ok(()) } #[test] - fn fallback_tool_call_routes_simple_config_changes() { - let show = fallback_rocm_tool_call_for_prompt("Show current ROCm CLI config").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&show).unwrap(), - vec!["config".to_owned(), "show".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&show)); + fn render_update_text_reports_all_update_surfaces() -> Result<()> { + let (root, paths) = test_paths("update-surfaces"); - let engine = fallback_rocm_tool_call_for_prompt("Set the default engine to vllm").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&engine).unwrap(), - vec![ - "config".to_owned(), - "set-default-engine".to_owned(), - "vllm".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&engine)); + let rendered = render_update_text(&paths)?; + fs::remove_dir_all(root).ok(); - let telemetry = - fallback_rocm_tool_call_for_prompt("Disable telemetry in settings").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&telemetry).unwrap(), - vec![ - "config".to_owned(), - "set-telemetry".to_owned(), - "off".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&telemetry)); + assert!(rendered.contains("update_surfaces:")); + assert!(rendered.contains("cli: installed=")); + assert!(rendered.contains("status=not_configured")); + assert!(rendered.contains("engines: status=package_managed")); + assert!(rendered.contains("model_recipes: status=")); + assert!(rendered.contains("runtimes: status=none_configured")); + assert!(rendered.contains("`rocm update --apply` applies runtime updates only")); + Ok(()) } #[test] - fn fallback_tool_call_routes_comfyui_support_and_actions() { - let status = fallback_rocm_tool_call_for_prompt("How do I setup ComfyUI?").unwrap(); - assert_eq!(status.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&status).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&status)); - - let install = fallback_rocm_tool_call_for_prompt("Can you setup ComfyUI for me?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&install).unwrap(), - vec!["comfyui".to_owned(), "install".to_owned()] - ); - assert!(!chat_tool_call_is_read_only(&install)); - let approval = - chat_tool_approval_request(&install, Some("Install ComfyUI after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Install ComfyUI"); + fn render_logs_text_preserves_directory_summary() { + let (_root, paths) = test_paths("logs-summary"); + let rendered = render_logs_text(&paths); - let start = fallback_rocm_tool_call_for_prompt("Can you start ComfyUI?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&start).unwrap(), - vec!["comfyui".to_owned(), "start".to_owned()] - ); - assert!(!chat_tool_call_is_read_only(&start)); - let approval = - chat_tool_approval_request(&start, Some("Start ComfyUI after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Start ComfyUI"); + assert!(rendered.contains("Logs")); + assert!(rendered.contains("File locations: shown")); + assert!(rendered.contains(&format!( + " Folder: {}", + paths.data_dir.join("logs").display() + ))); + assert!(rendered.contains(&format!( + " Activity log: {}", + cli_lifecycle_log_path(&paths).display() + ))); + assert!(rendered.contains(" Command logs:")); + assert!(rendered.contains(" Screen command logs:")); + assert!(rendered.contains(&format!( + " Audit events: {}", + paths.audit_events_path().display() + ))); + assert!(rendered.contains(" Recent command files: none yet")); + assert!(rendered.contains("Recent activity: no activity yet")); + assert!(rendered.contains("Matching lines")); + assert!(rendered.contains(" Search: none")); + assert!(rendered.contains(" No logs found yet.")); } #[test] - fn fallback_tool_call_routes_local_llm_serve_requests() { - let call = - fallback_rocm_tool_call_for_prompt("Can you setup and serve an LLM for me?").unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "serve".to_owned(), - "qwen".to_owned(), - "--engine".to_owned(), - "lemonade".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - let approval = - chat_tool_approval_request(&call, Some("Start qwen after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Start local model server"); - assert_eq!( - rocm_chat_tool_requested_command(&call).as_deref(), - Some("rocm serve qwen --engine lemonade --device gpu_required --managed") - ); + fn render_logs_text_lists_action_logs_and_recent_lifecycle_tail() -> Result<()> { + let (root, paths) = test_paths("logs-navigation"); + fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; + fs::write( + cli_lifecycle_log_path(&paths), + (0..10).fold(String::new(), |mut acc, index| { + let _ = writeln!( + acc, + "{index} level=info category=runtime action=install_sdk message=event-{index}" + ); + acc + }), + )?; + fs::write( + paths + .data_dir + .join("logs") + .join("cli") + .join("runtime-install_sdk.log"), + "install event\n", + )?; + fs::write( + paths + .data_dir + .join("logs") + .join("cli") + .join("update-update_check.log"), + "update event\n", + )?; + + let rendered = render_logs_text(&paths); + + assert!(rendered.contains(" Recent command files:")); + assert!(rendered.contains("runtime-install_sdk.log")); + assert!(rendered.contains("update-update_check.log")); + assert!(rendered.contains("Recent activity: last 8 line(s)")); + assert!(!rendered.contains("event-0")); + assert!(!rendered.contains("event-1")); + assert!(rendered.contains("Install: event-2")); + assert!(rendered.contains("event-2")); + assert!(rendered.contains("event-9")); + assert!(rendered.contains(" Lines: 10 of 10 recent line(s)")); + assert!(rendered.contains(" command log runtime-install_sdk.log: install event")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn local_chat_tool_call_content_is_treated_as_intermediate() { - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "Qwen/Qwen3-0.6B".to_owned(), - content: "The active runtime root is /opt/rocm.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: Some("call-1".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }], - }; + fn render_logs_text_lists_screen_command_logs() -> Result<()> { + let (root, paths) = test_paths("logs-screen-command"); + let screen_dir = paths.data_dir.join("logs").join("tui"); + fs::create_dir_all(&screen_dir)?; + let screen_log = screen_dir.join("12345-install-the-rock-sdk.log"); + fs::write( + &screen_log, + "title: Install TheRock SDK\n\ + recent_live_output:\n\ + Output: resolving torch wheels\n\ + command_output:\n\ + stdout:\n\ + resolved torch\n", + )?; - assert!(local_tool_call_content_is_intermediate( - "local", true, &response - )); - assert!(!local_tool_call_content_is_intermediate( - "openai", true, &response - )); - assert!(!local_tool_call_content_is_intermediate( - "local", false, &response - )); + let rendered = render_logs_text(&paths); - let without_tools = providers::ChatResponse { - tool_calls: Vec::new(), - ..response - }; - assert!(!local_tool_call_content_is_intermediate( - "local", - true, - &without_tools - )); + assert!(rendered.contains(" Screen command logs:")); + assert!(rendered.contains("screen/12345-install-the-rock-sdk.log")); + assert!(rendered.contains("screen command log 12345-install-the-rock-sdk.log")); + assert!(rendered.contains("Output: resolving torch wheels")); + assert!(rendered.contains(" Lines: 6 of 6 recent line(s)")); + let filtered = render_logs_browser_text(&paths, Some("torch wheels")); + assert!(filtered.contains("Search: torch wheels")); + assert!(filtered.contains("Output: resolving torch wheels")); + assert!(filtered.contains("screen command log 12345-install-the-rock-sdk.log")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn local_chat_follow_up_with_tool_call_is_not_final_answer() { - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "Qwen/Qwen3-0.6B".to_owned(), - content: "The runtime root is /opt/rocml.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: Some("call-2".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }], - }; + fn render_logs_browser_text_filters_lifecycle_and_action_logs() -> Result<()> { + let (root, paths) = test_paths("logs-browser-search"); + fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; + fs::write( + cli_lifecycle_log_path(&paths), + "1 level=info category=runtime action=install_sdk message=installed sdk\n\ + 2 level=info category=service action=serve message=server ready\n", + )?; + fs::write( + paths + .data_dir + .join("logs") + .join("cli") + .join("service-serve.log"), + "server ready\nmodel warmed\n", + )?; + + let rendered = render_logs_browser_text(&paths, Some("server")); + + assert!(rendered.contains(" Search: server")); + assert!(rendered.contains(" Lines: 2 of 4 recent line(s)")); + assert!(rendered.contains(" recent activity: Service event: server ready")); + assert!(rendered.contains(" command log service-serve.log: server ready")); + assert!(!rendered.contains("installed sdk")); + let _ = fs::remove_dir_all(root); + Ok(()) + } - assert!(!local_follow_up_content_is_final(&response)); + #[test] + fn render_logs_browser_page_text_paginates_matching_lines() -> Result<()> { + let (root, paths) = test_paths("logs-browser-pages"); + let action_dir = paths.data_dir.join("logs").join("cli"); + fs::create_dir_all(&action_dir)?; + fs::write(action_dir.join("a.log"), "alpha-1\nalpha-2\nalpha-3\n")?; + fs::write(action_dir.join("b.log"), "alpha-4\nalpha-5\nalpha-6\n")?; - let final_answer = providers::ChatResponse { - tool_calls: Vec::new(), - content: "The runtime root is D:\\ROCm\\therock_venvs.".to_owned(), - ..response - }; - assert!(local_follow_up_content_is_final(&final_answer)); + let rendered = render_logs_browser_page_text(&paths, Some("alpha"), 1, 4); + + assert!(rendered.contains(" Page: 2 of 2")); + assert!(rendered.contains(" Showing: 5-6 of 6")); + assert!(!rendered.contains("alpha-1")); + assert!(rendered.contains("alpha-5")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn visible_chat_content_removes_reasoning_blocks() { - assert_eq!( - visible_chat_content( - "\nchecking the tool output\n\nThe runtime root is D:\\ROCm\\temp." - ), - "The runtime root is D:\\ROCm\\temp." - ); - assert_eq!( - visible_chat_content("Before\nhidden\nAfter"), - "Before\n\nAfter" + fn cli_lifecycle_tail_lines_render_compactly() { + let rendered = format_cli_lifecycle_tail_line( + "42 level=error category=runtime action=install_sdk service_id= message=line one", ); - assert_eq!(visible_chat_content("unfinished"), ""); + + assert_eq!(rendered, "Install (error): line one"); } #[test] - fn chat_tool_result_errors_use_plain_failure_wording() { - assert_eq!(chat_read_only_tool_status_label(false), "done"); - assert_eq!(chat_read_only_tool_status_label(true), "reported an error"); - assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); - assert_eq!( - chat_tool_display_label("gpu_snapshot"), - "Checked GPU status" + fn render_service_logs_text_tails_manifest_log() -> Result<()> { + let (root, paths) = test_paths("service-logs"); + paths.ensure()?; + + let mut record = ManagedServiceRecord::new( + &paths, + "svc_qwen35_primary", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_preferred".to_owned()), ); - assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); - assert!(mcp_tool_result_is_error(&serde_json::json!({ - "isError": true - }))); - assert!(!mcp_tool_result_is_error(&serde_json::json!({ - "isError": false - }))); - assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); - } + record.status = "ready".to_owned(); + record.write()?; - #[test] - fn local_chat_without_service_explains_serve_before_chat_without_llm_setup() { - let (_root, paths) = test_paths("local-chat-no-service-guidance"); - let result = - render_chat_prompt_result(&paths, "local", None, "Check this ROCm setup", true) - .expect("missing local assistant should render guidance"); - assert!(result.approval.is_none()); - let rendered = result.rendered; + let mut log = String::new(); + for index in 1..=90 { + let _ = writeln!(log, "entry-{index:03}"); + } + fs::write(&record.log_path, log)?; - assert!(rendered.contains("No local assistant is running yet.")); - assert!(rendered.contains("First-time ROCm setup does not need an LLM")); - assert!(rendered.contains("Recommended path:")); - assert!(rendered.contains("Advanced manual command")); - assert!(rendered.contains( - "rocm serve Qwen3-4B-Instruct-2507-GGUF --engine lemonade --device gpu_required --managed" - )); - assert!(!rendered.contains("sshleifer/tiny-gpt2")); - assert!(rendered.contains("rocm chat --tools --provider local --prompt")); - assert!(rendered.contains("Nothing was changed.")); - assert!(!rendered.contains("install sdk")); - assert!(!rendered.contains("setup TheRock with an LLM")); + let rendered = render_service_logs_text(&paths, "svc_qwen35_primary")?; + assert!(rendered.contains("Service Log")); + assert!(rendered.contains("Service: svc_qwen35_primary")); + assert!(rendered.contains("Engine: vllm")); + assert!(rendered.contains("Status: starting")); + assert!(rendered.contains("File locations: shown")); + assert!(rendered.contains(&format!( + " Details file: {}", + record.manifest_path.display() + ))); + assert!(rendered.contains(&format!(" Log file: {}", record.log_path.display()))); + assert!(!rendered.contains("entry-010")); + assert!(rendered.contains("entry-011")); + assert!(rendered.contains("entry-090")); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn local_chat_status_prompts_use_read_only_tools_without_assistant() -> Result<()> { - let (_root, paths) = test_paths("local-chat-status-fallback"); + fn render_services_text_lists_live_services_by_default_and_all_on_request() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; - let running = - render_chat_prompt_result(&paths, "local", None, "Is vLLM running?", true)?.rendered; - assert!(!running.contains("No local assistant is running yet.")); - assert!(running.contains("Checked model servers: done"), "{running}"); - assert!(running.contains("ROCm CLI summary"), "{running}"); - assert!( - running.contains("Local model servers: none running under ROCm CLI."), - "{running}" - ); - assert!(running.contains("Nothing was changed.")); + let (root, paths) = test_paths("services-list"); + paths.ensure()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let ready_port = listener.local_addr()?.port(); + // The ready service is listed and then asked to complete a request: + // readiness is not granted on the model listing alone. + let server = thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = [0_u8; 512]; + let read = stream.read(&mut request)?; + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let current_pid = std::process::id(); - let installed = - render_chat_prompt_result(&paths, "local", None, "Is vLLM installed?", true)?.rendered; - assert!(!installed.contains("No local assistant is running yet.")); - assert!(installed.contains("Engine runtimes:"), "{installed}"); - assert!(installed.contains("vLLM:"), "{installed}"); + for (service_id, status, port) in [ + ("svc-ready", "ready", ready_port), + ("svc-starting", "starting", 11436_u16), + ("svc-failed", "failed", 11437_u16), + ] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + current_pid, + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = status.to_owned(); + record.write()?; + } - let installed_and_running = render_chat_prompt_result( + let rendered = render_services_text(&paths, false)?; + server + .join() + .expect("fake models server should not panic")?; + let all = render_services_text(&paths, true)?; + let _ = fs::remove_dir_all(root); + + assert!(rendered.contains("Local Servers")); + assert!(rendered.contains("Status: 1 ready, 1 starting")); + assert!(!rendered.contains("Past attempts")); + assert!(rendered.contains("- svc-ready")); + assert!(rendered.contains(" stop: rocm services stop svc-ready --yes")); + assert!(!rendered.contains("- svc-failed")); + assert!(all.contains("- svc-failed")); + assert!(all.contains(" restart: rocm services restart svc-failed --yes")); + assert!(!rendered.contains("running servers")); + Ok(()) + } + + #[test] + fn render_services_text_demotes_stale_ready_record() -> Result<()> { + let (root, paths) = test_paths("services-stale-ready"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( &paths, - "local", + "svc-stale-ready", + "lemonade", + "qwen", + providers::BUILTIN_ASSISTANT_MODEL_ID, + "127.0.0.1", + 9, + "managed", + 999_999_999, + Some("therock-release".to_owned()), None, - "Is vLLM installed and is it running?", - true, - )? - .rendered; - assert!( - installed_and_running.contains("Checked local engines: done"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Checked model servers: done"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Engine runtimes:"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Local model servers: none running under ROCm CLI."), - "{installed_and_running}" + Some("gpu_required".to_owned()), ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; - let port = render_chat_prompt_result( - &paths, - "local", - None, - "What is running on port 8188?", - true, - )? - .rendered; - assert!(!port.contains("No local assistant is running yet.")); - assert!(port.contains("Checked local port: done"), "{port}"); - assert!(port.contains("Port 8188:"), "{port}"); + let rendered = render_services_text(&paths, false)?; + let all = render_services_text(&paths, true)?; + let reloaded = load_managed_service(&paths, "svc-stale-ready")?; + let _ = fs::remove_dir_all(root); + + assert!(rendered.contains("No local servers are running.")); + assert!(all.contains("- svc-stale-ready")); + assert!(all.contains(" status: stopped")); + assert_eq!(reloaded.status, "stopped"); Ok(()) } #[test] - fn chat_tools_anthropic_reaches_provider_opt_in_boundary() { - let (_root, paths) = test_paths("anthropic-chat-tools-opt-in"); + fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { + // `generate_service_id` embeds a timestamp, so a second launch for the + // same engine+model has a DIFFERENT service_id. The guard must still + // detect the live service by (engine, canonical_model_id), and return + // the newest live match. `starting` skips the endpoint probe; the + // current process id is a guaranteed-live PID. + let (root, paths) = test_paths("dup-managed-distinct-ids"); + paths.ensure()?; - let error = render_chat_prompt_result( + // Older, dead manifest for the same engine+model (distinct service_id). + let mut dead = ManagedServiceRecord::new( &paths, - "anthropic", - Some("claude-test"), - "Check this ROCm setup", - true, - ) - .unwrap_err() - .to_string(); + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, + None, + None, + ); + dead.status = "ready".to_owned(); + dead.engine_pid = Some(999_999_999); + dead.created_at_unix_ms = 1000; + dead.write()?; - assert!(error.contains("cloud provider `anthropic` is disabled")); - assert!(error.contains("rocm config enable-provider anthropic")); - assert!(!error.contains("OpenAI-compatible provider")); + // Newer, live manifest for the same engine+model (distinct service_id). + let mut live = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-2000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11501, + "managed", + std::process::id(), + None, + None, + None, + ); + live.status = "starting".to_owned(); + live.engine_pid = Some(std::process::id()); + live.created_at_unix_ms = 2000; + live.write()?; + + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + let found = found.expect("a live managed service should be detected by engine+model"); + assert_eq!( + found.service_id, "lemonade-qwen-2000", + "should return the newest live match" + ); + assert!(managed_service_is_live(&found)); + Ok(()) } #[test] - fn freeform_execution_validation_rejects_provider_assisted_plans() -> Result<()> { - let content = r#"{ - "intent": "serve", - "tool_call": { - "tool": "rocm", - "args": ["serve", "sshleifer/tiny-gpt2", "--engine", "vllm", "--managed"] - } - }"#; - let plan = provider_planner_response_to_plan("start a local model", "local", content)?; - let action = plan_next_action(plan).expect("provider plan should have an action"); + fn dead_managed_service_allows_relaunch() -> Result<()> { + // A stale manifest with dead PIDs must NOT block a relaunch: liveness + // refresh demotes it to "stopped", so the guard returns None. + let (root, paths) = test_paths("dup-managed-dead"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; - let error = validate_freeform_execution_action(&action) - .unwrap_err() - .to_string(); + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); - assert!(action.provider_assisted); - assert!(error.contains("reviewed interactively")); + assert!( + found.is_none(), + "a dead managed service must not block relaunch" + ); Ok(()) } #[test] - fn render_update_text_reports_all_update_surfaces() -> Result<()> { - let (root, paths) = test_paths("update-surfaces"); + fn live_service_for_other_model_does_not_block() -> Result<()> { + // A live service for a DIFFERENT model must not match — the guard keys + // on the model, not just the engine. + let (root, paths) = test_paths("dup-managed-other-model"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-other-1", + "lemonade", + "other", + "other-canonical", + "127.0.0.1", + 11503, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; - let rendered = render_update_text(&paths)?; - fs::remove_dir_all(root).ok(); + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); - assert!(rendered.contains("update_surfaces:")); - assert!(rendered.contains("cli: installed=")); - assert!(rendered.contains("status=not_configured")); - assert!(rendered.contains("engines: status=package_managed")); - assert!(rendered.contains("model_recipes: status=")); - assert!(rendered.contains("runtimes: status=none_configured")); - assert!(rendered.contains("`rocm update --apply` applies runtime updates only")); + assert!( + found.is_none(), + "a live service for a different model must not match" + ); Ok(()) } #[test] - fn render_logs_text_preserves_directory_summary() { - let (_root, paths) = test_paths("logs-summary"); - let rendered = render_logs_text(&paths); - - assert!(rendered.contains("Logs")); - assert!(rendered.contains("File locations: shown")); - assert!(rendered.contains(&format!( - " Folder: {}", - paths.data_dir.join("logs").display() - ))); - assert!(rendered.contains(&format!( - " Activity log: {}", - cli_lifecycle_log_path(&paths).display() - ))); - assert!(rendered.contains(" Command logs:")); - assert!(rendered.contains(" Screen command logs:")); - assert!(rendered.contains(&format!( - " Audit events: {}", - paths.audit_events_path().display() - ))); - assert!(rendered.contains(" Recent command files: none yet")); - assert!(rendered.contains("Recent activity: no activity yet")); - assert!(rendered.contains("Matching lines")); - assert!(rendered.contains(" Search: none")); - assert!(rendered.contains(" No logs found yet.")); + fn missing_manifest_allows_launch() { + // No services dir / manifests → nothing to detect, launch proceeds. + let (root, paths) = test_paths("dup-managed-missing"); + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!(found.is_none()); } #[test] - fn render_logs_text_lists_action_logs_and_recent_lifecycle_tail() -> Result<()> { - let (root, paths) = test_paths("logs-navigation"); - fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; - fs::write( - cli_lifecycle_log_path(&paths), - (0..10).fold(String::new(), |mut acc, index| { - let _ = writeln!( - acc, - "{index} level=info category=runtime action=install_sdk message=event-{index}" - ); - acc - }), - )?; - fs::write( - paths - .data_dir - .join("logs") - .join("cli") - .join("runtime-install_sdk.log"), - "install event\n", - )?; - fs::write( - paths - .data_dir - .join("logs") - .join("cli") - .join("update-update_check.log"), - "update event\n", - )?; + fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { + // A live service recorded with one recipe (e.g. a tool-call parser flag) + // must reject a relaunch requesting a different recipe rather than + // silently reusing the old server, and the error must not claim the + // mismatch is specifically about generation defaults when it could stem + // from any recipe field. + let (root, paths) = test_paths("dup-managed-recipe-mismatch"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], + ..EngineRecipeHint::default() + })?); + existing.write()?; - let rendered = render_logs_text(&paths); + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let requested_recipe = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], + ..EngineRecipeHint::default() + }; - assert!(rendered.contains(" Recent command files:")); - assert!(rendered.contains("runtime-install_sdk.log")); - assert!(rendered.contains("update-update_check.log")); - assert!(rendered.contains("Recent activity: last 8 line(s)")); - assert!(!rendered.contains("event-0")); - assert!(!rendered.contains("event-1")); - assert!(rendered.contains("Install: event-2")); - assert!(rendered.contains("event-2")); - assert!(rendered.contains("event-9")); - assert!(rendered.contains(" Lines: 10 of 10 recent line(s)")); - assert!(rendered.contains(" command log runtime-install_sdk.log: install event")); + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + 11511, + &resolve.device_policy, + &[], + None, + None, + Some(&requested_recipe), + ); let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("mismatched recipe on a live service must be rejected") + }; + let message = error.to_string(); + assert!( + message.contains("different serve options"), + "message should describe the mismatch generically: {message}" + ); + assert!( + message.contains("recipe hint, tool-call parser, or generation defaults"), + "message should not single out generation defaults as the sole cause: {message}" + ); Ok(()) } #[test] - fn render_logs_text_lists_screen_command_logs() -> Result<()> { - let (root, paths) = test_paths("logs-screen-command"); - let screen_dir = paths.data_dir.join("logs").join("tui"); - fs::create_dir_all(&screen_dir)?; - let screen_log = screen_dir.join("12345-install-the-rock-sdk.log"); - fs::write( - &screen_log, - "title: Install TheRock SDK\n\ - recent_live_output:\n\ - Output: resolving torch wheels\n\ - command_output:\n\ - stdout:\n\ - resolved torch\n", - )?; + fn services_tool_result_text_includes_running_interpretation() { + let (_root, paths) = test_paths("services-tool-text"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-vllm", + "vllm", + "Qwen/Qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); - let rendered = render_logs_text(&paths); + let rendered = render_services_tool_result_text(&[record]); - assert!(rendered.contains(" Screen command logs:")); - assert!(rendered.contains("screen/12345-install-the-rock-sdk.log")); - assert!(rendered.contains("screen command log 12345-install-the-rock-sdk.log")); - assert!(rendered.contains("Output: resolving torch wheels")); - assert!(rendered.contains(" Lines: 6 of 6 recent line(s)")); - let filtered = render_logs_browser_text(&paths, Some("torch wheels")); - assert!(filtered.contains("Search: torch wheels")); - assert!(filtered.contains("Output: resolving torch wheels")); - assert!(filtered.contains("screen command log 12345-install-the-rock-sdk.log")); - let _ = fs::remove_dir_all(root); - Ok(()) + assert!(rendered.contains("status_meaning: ready/running = running")); + assert!(rendered.contains("engine=vllm")); + assert!(rendered.contains("running_state=running")); } #[test] - fn render_logs_browser_text_filters_lifecycle_and_action_logs() -> Result<()> { - let (root, paths) = test_paths("logs-browser-search"); - fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; - fs::write( - cli_lifecycle_log_path(&paths), - "1 level=info category=runtime action=install_sdk message=installed sdk\n\ - 2 level=info category=service action=serve message=server ready\n", - )?; - fs::write( - paths - .data_dir - .join("logs") - .join("cli") - .join("service-serve.log"), - "server ready\nmodel warmed\n", - )?; + fn service_actions_require_yes_and_render_sandbox_result() { + let (_root, paths) = test_paths("services-action-approval"); + let error = run_approved_service_action(&paths, "stop_server", "svc-qwen", false) + .unwrap_err() + .to_string(); + assert!(error.contains("requires --yes")); + assert!(error.contains("rocm services stop svc-qwen --yes")); - let rendered = render_logs_browser_text(&paths, Some("server")); + let rendered = render_service_action_result( + "stop_server", + &serde_json::json!({ + "output": { + "status": "stopped", + "result": { + "service": { + "service_id": "svc-qwen", + "status": "stopped", + "endpoint_url": "http://127.0.0.1:11435/v1" + }, + "signaled_pids": [1234, 5678] + } + } + }), + ); - assert!(rendered.contains(" Search: server")); - assert!(rendered.contains(" Lines: 2 of 4 recent line(s)")); - assert!(rendered.contains(" recent activity: Service event: server ready")); - assert!(rendered.contains(" command log service-serve.log: server ready")); - assert!(!rendered.contains("installed sdk")); - let _ = fs::remove_dir_all(root); - Ok(()) + assert!(rendered.contains("Local server stopped")); + assert!(rendered.contains("service: svc-qwen")); + assert!(rendered.contains("status: stopped")); + assert!(rendered.contains("stopped processes: 2")); } #[test] - fn render_logs_browser_page_text_paginates_matching_lines() -> Result<()> { - let (root, paths) = test_paths("logs-browser-pages"); - let action_dir = paths.data_dir.join("logs").join("cli"); - fs::create_dir_all(&action_dir)?; - fs::write(action_dir.join("a.log"), "alpha-1\nalpha-2\nalpha-3\n")?; - fs::write(action_dir.join("b.log"), "alpha-4\nalpha-5\nalpha-6\n")?; + fn services_is_structured_not_freeform() { + let invocation = parse_freeform_invocation(&["services".to_owned()]); + assert!(!should_treat_as_freeform(&invocation)); + Cli::try_parse_from(["rocm", "services"]).expect("services should be a real command"); + Cli::try_parse_from(["rocm", "services", "list"]) + .expect("services list should be a real command"); + Cli::try_parse_from(["rocm", "services", "logs", "svc-qwen"]) + .expect("services logs should be a real command"); + Cli::try_parse_from(["rocm", "services", "stop", "svc-qwen", "--yes"]) + .expect("services stop should accept --yes"); + Cli::try_parse_from(["rocm", "services", "restart", "svc-qwen", "--yes"]) + .expect("services restart should accept --yes"); + } - let rendered = render_logs_browser_page_text(&paths, Some("alpha"), 1, 4); + #[test] + fn install_sdk_accepts_family_override() { + Cli::try_parse_from([ + "rocm", + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--prefix", + "D:\\ROCm\\therock_venvs", + "--family", + "gfx110X-all", + ]) + .expect("install sdk should accept a TheRock family override"); + } - assert!(rendered.contains(" Page: 2 of 2")); - assert!(rendered.contains(" Showing: 5-6 of 6")); - assert!(!rendered.contains("alpha-1")); - assert!(rendered.contains("alpha-5")); - let _ = fs::remove_dir_all(root); - Ok(()) + #[test] + fn top_level_cli_commands_are_not_treated_as_freeform() { + for command in [ + "examine", + "bootstrap", + "version", + "setup", + "chat", + "install", + "update", + "runtimes", + "storage", + "engines", + "model", + "models", + "serve", + "comfyui", + "comfy", + "services", + "automations", + "config", + "logs", + "daemon", + "dash", + "bench", + "uninstall", + "completions", + "help", + ] { + let invocation = parse_freeform_invocation(&[command.to_owned()]); + assert!( + !should_treat_as_freeform(&invocation), + "{command} should parse as a structured CLI command, not natural language" + ); + } + Cli::try_parse_from(["rocm", "setup"]).expect("setup should parse"); + Cli::try_parse_from(["rocm", "bootstrap"]).expect("bootstrap setup should parse"); + Cli::try_parse_from(["rocm", "setup", "status"]).expect("setup status should parse"); + Cli::try_parse_from(["rocm", "setup", "reset"]).expect("setup reset should parse"); + Cli::try_parse_from(["rocm", "models"]).expect("models alias should parse"); + Cli::try_parse_from(["rocm", "comfyui", "status"]).expect("comfyui status should parse"); + Cli::try_parse_from(["rocm", "comfyui", "logs", "--lines", "3"]) + .expect("comfyui logs should parse"); + Cli::try_parse_from(["rocm", "comfyui", "stop"]).expect("comfyui stop should parse"); + Cli::try_parse_from(["rocm", "comfy", "logs"]).expect("comfy alias should parse"); + } + + #[test] + fn t5_bench_load_clap_parse_smoke() { + // T5: verify BenchCommand::Load parses correctly including comma-separated concurrency. + let cli = Cli::try_parse_from([ + "rocm", + "bench", + "load", + "--endpoint", + "http://x", + "--concurrency", + "1,8,32,64", + ]) + .expect("rocm bench load should parse"); + match cli.command { + Some(Command::Bench { + command: BenchCommand::Load { concurrency, .. }, + }) => { + assert_eq!(concurrency, vec![1u32, 8, 32, 64]); + } + other => panic!("expected Bench/Load, got {other:?}"), + } } #[test] - fn cli_lifecycle_tail_lines_render_compactly() { - let rendered = format_cli_lifecycle_tail_line( - "42 level=error category=runtime action=install_sdk service_id= message=line one", - ); + fn setup_reset_cli_output_is_plain_and_persists_first_time_prompt() -> Result<()> { + let (_root, paths) = test_paths("setup-reset-cli"); + let mut config = RocmCliConfig { + onboarding_dismissed: true, + setup: rocm_core::SetupConfig { + completed: true, + therock_venv: Some(paths.data_dir.join("envs").join("default")), + cli_install_dir: None, + }, + ..Default::default() + }; + config.provider_config_mut("openai").enabled = true; + config.save(&paths)?; - assert_eq!(rendered, "Install (error): line one"); + let rendered = reset_setup_prompt_state(&paths, &mut config)?; + + assert!(rendered.contains("Setup will show again")); + assert!(rendered.contains("ROCm installs were not deleted")); + assert!(rendered.contains("API keys")); + assert!(!rendered.contains("request plan")); + assert!(!rendered.contains("planner:")); + assert!(!rendered.contains("tool_schema")); + + let saved = RocmCliConfig::load(&paths)?; + assert!(!saved.onboarding_dismissed); + assert!(!saved.setup.completed); + assert!(saved.setup.therock_venv.is_some()); + assert!(saved.provider_enabled("openai")); + Ok(()) } #[test] - fn render_service_logs_text_tails_manifest_log() -> Result<()> { - let (root, paths) = test_paths("service-logs"); - paths.ensure()?; - - let mut record = ManagedServiceRecord::new( + fn setup_status_reports_completed_active_runtime() -> Result<()> { + let (root, paths) = test_paths("setup-status-completed-runtime"); + let manifest = write_test_pip_runtime( &paths, - "svc_qwen35_primary", - "vllm", - "qwen3.5", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - std::process::id(), - Some("therock-release".to_owned()), - None, - Some("gpu_preferred".to_owned()), - ); - record.status = "ready".to_owned(); - record.write()?; + "release-pip-gfx120x-all-status", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + let config = RocmCliConfig { + default_runtime_id: Some(manifest.runtime_id.clone()), + active_runtime_key: Some(manifest.runtime_key.clone()), + setup: rocm_core::SetupConfig { + completed: true, + therock_venv: Some(manifest.install_root.clone()), + cli_install_dir: None, + }, + ..Default::default() + }; - let mut log = String::new(); - for index in 1..=90 { - let _ = writeln!(log, "entry-{index:03}"); - } - fs::write(&record.log_path, log)?; + let rendered = render_setup_status_text(&paths, &config)?; - let rendered = render_service_logs_text(&paths, "svc_qwen35_primary")?; - assert!(rendered.contains("Service Log")); - assert!(rendered.contains("Service: svc_qwen35_primary")); - assert!(rendered.contains("Engine: vllm")); - assert!(rendered.contains("Status: starting")); - assert!(rendered.contains("File locations: shown")); - assert!(rendered.contains(&format!( - " Details file: {}", - record.manifest_path.display() - ))); - assert!(rendered.contains(&format!(" Log file: {}", record.log_path.display()))); - assert!(!rendered.contains("entry-010")); - assert!(rendered.contains("entry-011")); - assert!(rendered.contains("entry-090")); + assert!(rendered.contains("status: completed"), "{rendered}"); + assert!( + rendered.contains(&format!( + "install folder: {}", + manifest.install_root.display() + )), + "{rendered}" + ); + assert!( + rendered.contains("active_runtime_key: release-pip-gfx120x-all-status"), + "{rendered}" + ); + assert!( + rendered.contains("active_runtime_id: therock-release:gfx120X-all"), + "{rendered}" + ); + assert!( + rendered.contains("active_runtime_status: ready"), + "{rendered}" + ); + assert!(rendered.contains("rocm help"), "{rendered}"); let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn render_services_text_lists_live_services_by_default_and_all_on_request() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - let (root, paths) = test_paths("services-list"); - paths.ensure()?; - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let ready_port = listener.local_addr()?.port(); - // The ready service is listed and then asked to complete a request: - // readiness is not granted on the model listing alone. - let server = thread::spawn(move || -> Result<()> { - for _ in 0..2 { - let (mut stream, _) = listener.accept()?; - stream.set_read_timeout(Some(Duration::from_secs(2)))?; - let mut request = [0_u8; 512]; - let read = stream.read(&mut request)?; - let request = String::from_utf8_lossy(&request[..read]).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# - }; - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - )?; - } - Ok(()) - }); - let current_pid = std::process::id(); - - for (service_id, status, port) in [ - ("svc-ready", "ready", ready_port), - ("svc-starting", "starting", 11436_u16), - ("svc-failed", "failed", 11437_u16), - ] { - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - port, - "managed", - current_pid, - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = status.to_owned(); - record.write()?; - } + fn setup_status_reports_first_time_when_not_completed() -> Result<()> { + let (_root, paths) = test_paths("setup-status-first-time"); + let config = RocmCliConfig::default(); - let rendered = render_services_text(&paths, false)?; - server - .join() - .expect("fake models server should not panic")?; - let all = render_services_text(&paths, true)?; - let _ = fs::remove_dir_all(root); + let rendered = render_setup_status_text(&paths, &config)?; - assert!(rendered.contains("Local Servers")); - assert!(rendered.contains("Status: 1 ready, 1 starting")); - assert!(!rendered.contains("Past attempts")); - assert!(rendered.contains("- svc-ready")); - assert!(rendered.contains(" stop: rocm services stop svc-ready --yes")); - assert!(!rendered.contains("- svc-failed")); - assert!(all.contains("- svc-failed")); - assert!(all.contains(" restart: rocm services restart svc-failed --yes")); - assert!(!rendered.contains("running servers")); + assert!(rendered.contains("status: first-time setup will show")); + assert!(rendered.contains("active_runtime_status: ")); Ok(()) } #[test] - fn render_services_text_demotes_stale_ready_record() -> Result<()> { - let (root, paths) = test_paths("services-stale-ready"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-stale-ready", - "lemonade", - "qwen", - providers::BUILTIN_ASSISTANT_MODEL_ID, - "127.0.0.1", - 9, - "managed", - 999_999_999, - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), + fn serve_bind_validation_requires_public_ack() { + validate_bind_host("127.0.0.1", false).unwrap(); + validate_bind_host("localhost", false).unwrap(); + validate_bind_host("::1", false).unwrap(); + let error = validate_bind_host("0.0.0.0", false).unwrap_err(); + assert!( + error.to_string().contains("--allow-public-bind"), + "{error:#}" ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; + validate_bind_host("0.0.0.0", true).unwrap(); + } - let rendered = render_services_text(&paths, false)?; - let all = render_services_text(&paths, true)?; - let reloaded = load_managed_service(&paths, "svc-stale-ready")?; - let _ = fs::remove_dir_all(root); + #[test] + fn resolve_endpoint_auth_loopback_stays_credential_free() { + // Loopback binds never require auth, even if a key is supplied. + for host in ["127.0.0.1", "localhost", "::1"] { + assert_eq!(resolve_endpoint_auth(host, None).unwrap(), None); + assert_eq!(resolve_endpoint_auth(host, Some("ignored")).unwrap(), None); + } + } - assert!(rendered.contains("No local servers are running.")); - assert!(all.contains("- svc-stale-ready")); - assert!(all.contains(" status: stopped")); - assert_eq!(reloaded.status, "stopped"); - Ok(()) + #[test] + fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { + let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key ")) + .unwrap() + .expect("public bind must have a key"); + assert_eq!(key, "my-key"); + } + + #[test] + fn resolve_endpoint_auth_public_generates_key_when_absent() { + let key = resolve_endpoint_auth("0.0.0.0", None) + .unwrap() + .expect("public bind must generate a key"); + assert_eq!(key.len(), 48); + assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); + } + + #[test] + fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { + let error = resolve_endpoint_auth("0.0.0.0", Some(" ")).unwrap_err(); + assert!(error.to_string().contains("non-empty"), "{error:#}"); } #[test] - fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { - // `generate_service_id` embeds a timestamp, so a second launch for the - // same engine+model has a DIFFERENT service_id. The guard must still - // detect the live service by (engine, canonical_model_id), and return - // the newest live match. `starting` skips the endpoint probe; the - // current process id is a guaranteed-live PID. - let (root, paths) = test_paths("dup-managed-distinct-ids"); - paths.ensure()?; + fn resolve_endpoint_auth_public_rejects_embedded_crlf() { + // A supplied key survives `trim()` with embedded CR/LF intact and would + // otherwise be interpolated into a raw `Authorization: Bearer` header, + // injecting an extra header line. It must be rejected at input validation. + for supplied in [ + "good-key\r\nX-Injected: value", + "good-key\nmore", + "line\rreturn", + ] { + let error = resolve_endpoint_auth("0.0.0.0", Some(supplied)).unwrap_err(); + assert!(error.to_string().contains("control character"), "{error:#}"); + } + } - // Older, dead manifest for the same engine+model (distinct service_id). - let mut dead = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, - ); - dead.status = "ready".to_owned(); - dead.engine_pid = Some(999_999_999); - dead.created_at_unix_ms = 1000; - dead.write()?; + #[test] + fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { + let (root, paths) = test_paths("drop-orphaned-key-stored"); + let service_id = "svc-orphaned"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); - // Newer, live manifest for the same engine+model (distinct service_id). - let mut live = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-2000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11501, - "managed", - std::process::id(), - None, - None, - None, - ); - live.status = "starting".to_owned(); - live.engine_pid = Some(std::process::id()); - live.created_at_unix_ms = 2000; - live.write()?; + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); let _ = fs::remove_dir_all(root); - - let found = found.expect("a live managed service should be detected by engine+model"); - assert_eq!( - found.service_id, "lemonade-qwen-2000", - "should return the newest live match" - ); - assert!(managed_service_is_live(&found)); - Ok(()) } #[test] - fn dead_managed_service_allows_relaunch() -> Result<()> { - // A stale manifest with dead PIDs must NOT block a relaunch: liveness - // refresh demotes it to "stopped", so the guard returns None. - let (root, paths) = test_paths("dup-managed-dead"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; + fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { + // A loopback attempt never stores a key (`freshly_stored == None`), so the + // helper must not panic or error, and no file must appear. + let (root, paths) = test_paths("drop-orphaned-key-loopback"); + let service_id = "svc-loopback"; - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); - assert!( - found.is_none(), - "a dead managed service must not block relaunch" - ); - Ok(()) + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); } #[test] - fn live_service_for_other_model_does_not_block() -> Result<()> { - // A live service for a DIFFERENT model must not match — the guard keys - // on the model, not just the engine. - let (root, paths) = test_paths("dup-managed-other-model"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-other-1", - "lemonade", - "other", - "other-canonical", - "127.0.0.1", - 11503, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "starting".to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; + fn public_bind_fails_closed_for_windows_lemonade_only() { + // Windows + Lemonade + public bind: refuse (cannot enforce the key). + let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); + assert!(error.to_string().contains("lemonade"), "{error:#}"); + // Every other combination is allowed: + ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows + ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows + ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key + } - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); + #[test] + fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { + // A stop deletes the key file, so a later restart of a public service + // would otherwise respawn it with no auth at all. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("0.0.0.0"), "{error:#}"); + assert!(message.contains("without authentication"), "{error:#}"); + // Actionable: name the command that mints a fresh key. + assert!(message.contains("--allow-public-bind"), "{error:#}"); - assert!( - found.is_none(), - "a live service for a different model must not match" - ); - Ok(()) + // A public service that still has its key restarts normally. + ensure_public_service_has_endpoint_key("0.0.0.0", true).unwrap(); } #[test] - fn missing_manifest_allows_launch() { - // No services dir / manifests → nothing to detect, launch proceeds. - let (root, paths) = test_paths("dup-managed-missing"); - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - assert!(found.is_none()); + fn respawn_allows_loopback_services_without_an_endpoint_key() { + // Loopback stays credential-free, so every accepted spelling must pass + // the guard with no key present. + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } } #[test] - fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { - // A live service recorded with one recipe (e.g. a tool-call parser flag) - // must reject a relaunch requesting a different recipe rather than - // silently reusing the old server, and the error must not claim the - // mismatch is specifically about generation defaults when it could stem - // from any recipe field. - let (root, paths) = test_paths("dup-managed-recipe-mismatch"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( + fn restart_refuses_a_public_service_without_a_key_before_stopping_it() { + // The guard runs before the stop, so a refused restart must leave the + // record exactly as it was rather than taking down a running service. + let (root, paths) = test_paths("restart-public-no-key"); + let service_id = "svc-public-nokey"; + let mut record = ManagedServiceRecord::new( &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11510, + service_id, + "vllm", + "model-ref", + "canonical/model", + "0.0.0.0", + 11435, "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], - ..EngineRecipeHint::default() - })?); - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let requested_recipe = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], - ..EngineRecipeHint::default() - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-2000", - "qwen", - &resolve, - "127.0.0.1", - 11511, - &resolve.device_policy, - &[], + std::process::id(), None, None, - Some(&requested_recipe), + Some("gpu_required".to_owned()), ); - let _ = fs::remove_dir_all(root); + record.status = "running".to_owned(); + record.write().unwrap(); - let Err(error) = result else { - panic!("mismatched recipe on a live service must be rejected") - }; - let message = error.to_string(); + let error = restart_internal_managed_service(&paths, service_id).unwrap_err(); assert!( - message.contains("different serve options"), - "message should describe the mismatch generically: {message}" + error.to_string().contains("without authentication"), + "{error:#}" ); - assert!( - message.contains("recipe hint, tool-call parser, or generation defaults"), - "message should not single out generation defaults as the sole cause: {message}" + + // Proves the *ordering*, not just the refusal: had the guard run after + // `stop_internal_managed_service`, the stop would have written + // "stopped". It reaches that state here because the record's only pid is + // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` + // skips the caller's pid), so the stop confirms termination trivially. + let after = load_managed_service(&paths, service_id).unwrap(); + assert_ne!( + after.status, "stopped", + "a refused restart must not stop the service" ); - Ok(()) + let _ = fs::remove_dir_all(root); } - #[test] - fn services_tool_result_text_includes_running_interpretation() { - let (_root, paths) = test_paths("services-tool-text"); + /// A public record with a dead pid and a stored endpoint key, for the + /// liveness-refresh cases below. The port is one nothing listens on, so the + /// refresh's endpoint probe fails and it falls through to the pid check. + fn dead_public_service_with_key( + paths: &AppPaths, + service_id: &str, + port: u16, + ) -> ManagedServiceRecord { + paths.ensure().unwrap(); let mut record = ManagedServiceRecord::new( - &paths, - "svc-vllm", + paths, + service_id, "vllm", - "Qwen/Qwen3.5", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, + "model-ref", + "canonical/model", + "0.0.0.0", + port, "managed", - std::process::id(), - Some("therock-release".to_owned()), + // A pid far above any plausible live process, as in + // `dead_managed_service_allows_relaunch`. + 999_999_999, + None, + None, None, - Some("gpu_required".to_owned()), ); - record.status = "ready".to_owned(); + record.status = "running".to_owned(); + record.write().unwrap(); + endpoint_keys::store_endpoint_api_key(paths, service_id, "secret-key").unwrap(); + record + } - let rendered = render_services_tool_result_text(&[record]); + #[test] + fn crashed_public_service_keeps_its_endpoint_key() { + // A crash (OOM kill, host reboot, panic) leaves the record at "running" + // with dead pids and no stop marker. Dropping the key here would make + // the fail-closed respawn guards refuse every later `rocm services + // restart` and every daemon recovery attempt — permanently, because + // nothing can re-mint the key. The refresh happens on every read + // (`rocm services list`), so this must survive it. + let (root, paths) = test_paths("liveness-crash-keeps-key"); + let service_id = "svc-crashed-public"; + let mut record = dead_public_service_with_key(&paths, service_id, 11982); - assert!(rendered.contains("status_meaning: ready/running = running")); - assert!(rendered.contains("engine=vllm")); - assert!(rendered.contains("running_state=running")); + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + assert!(changed, "a dead service must be demoted to stopped"); + assert_eq!(record.status, "stopped"); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, service_id).as_deref(), + Some("secret-key"), + "a crashed public service must stay restartable" + ); + let _ = fs::remove_dir_all(root); } #[test] - fn service_actions_require_yes_and_render_sandbox_result() { - let (_root, paths) = test_paths("services-action-approval"); - let error = run_approved_service_action(&paths, "stop_server", "svc-qwen", false) - .unwrap_err() - .to_string(); - assert!(error.contains("requires --yes")); - assert!(error.contains("rocm services stop svc-qwen --yes")); + fn unconfirmed_stop_clears_the_endpoint_key_once_the_processes_are_gone() { + // The other half: a stop that could not confirm termination leaves the + // key in place (the engine may still be alive and enforcing it) and + // records the intent. Once the processes are observed gone, the deferred + // cleanup runs, so no plaintext secret is stranded for a service the + // operator did ask to stop. + let (root, paths) = test_paths("liveness-pending-stop-clears-key"); + let service_id = "svc-pending-stop"; + let mut record = dead_public_service_with_key(&paths, service_id, 11983); + record.stop_requested_unix_ms = Some(1); - let rendered = render_service_action_result( - "stop_server", - &serde_json::json!({ - "output": { - "status": "stopped", - "result": { - "service": { - "service_id": "svc-qwen", - "status": "stopped", - "endpoint_url": "http://127.0.0.1:11435/v1" - }, - "signaled_pids": [1234, 5678] - } - } - }), - ); + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); - assert!(rendered.contains("Local server stopped")); - assert!(rendered.contains("service: svc-qwen")); - assert!(rendered.contains("status: stopped")); - assert!(rendered.contains("stopped processes: 2")); + assert!(changed); + assert_eq!(record.status, "stopped"); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, service_id), + None, + "a requested stop must still drop the key" + ); + assert_eq!( + record.stop_requested_unix_ms, None, + "the marker is consumed, so the cleanup does not run again" + ); + let _ = fs::remove_dir_all(root); } #[test] - fn services_is_structured_not_freeform() { - let invocation = parse_freeform_invocation(&["services".to_owned()]); - assert!(!should_treat_as_freeform(&invocation)); - Cli::try_parse_from(["rocm", "services"]).expect("services should be a real command"); - Cli::try_parse_from(["rocm", "services", "list"]) - .expect("services list should be a real command"); - Cli::try_parse_from(["rocm", "services", "logs", "svc-qwen"]) - .expect("services logs should be a real command"); - Cli::try_parse_from(["rocm", "services", "stop", "svc-qwen", "--yes"]) - .expect("services stop should accept --yes"); - Cli::try_parse_from(["rocm", "services", "restart", "svc-qwen", "--yes"]) - .expect("services restart should accept --yes"); + fn pending_stop_cleanup_runs_even_once_the_record_reads_stopped() { + // Proves the cleanup sits *before* the `managed_service_is_live` gate: + // a record that reached "stopped" by another route (an engine state + // refresh, a concurrent writer) would otherwise early-return and strand + // the key of a service the operator stopped. + let (root, paths) = test_paths("liveness-pending-stop-when-stopped"); + let service_id = "svc-pending-stop-stopped"; + let mut record = dead_public_service_with_key(&paths, service_id, 11984); + record.status = "stopped".to_owned(); + record.stop_requested_unix_ms = Some(1); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + assert!( + changed, + "consuming the marker is a record change worth writing" + ); + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert_eq!(record.stop_requested_unix_ms, None); + let _ = fs::remove_dir_all(root); } #[test] - fn install_sdk_accepts_family_override() { - Cli::try_parse_from([ - "rocm", - "install", - "sdk", - "--channel", - "release", - "--format", - "wheel", - "--prefix", - "D:\\ROCm\\therock_venvs", - "--family", - "gfx110X-all", - ]) - .expect("install sdk should accept a TheRock family override"); + fn endpoint_client_config_shows_key_once_with_bearer_guidance() { + let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); + assert!(rendered.contains("secret-123"), "{rendered}"); + assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); + assert!(rendered.contains("shown only now"), "{rendered}"); } #[test] - fn top_level_cli_commands_are_not_treated_as_freeform() { - for command in [ - "examine", - "bootstrap", - "version", - "setup", - "chat", - "install", - "update", - "runtimes", - "storage", - "engines", - "model", - "models", - "serve", - "comfyui", - "comfy", - "services", - "automations", - "config", - "logs", - "daemon", - "dash", - "bench", - "uninstall", - "completions", - "help", - ] { - let invocation = parse_freeform_invocation(&[command.to_owned()]); - assert!( - !should_treat_as_freeform(&invocation), - "{command} should parse as a structured CLI command, not natural language" - ); - } - Cli::try_parse_from(["rocm", "setup"]).expect("setup should parse"); - Cli::try_parse_from(["rocm", "bootstrap"]).expect("bootstrap setup should parse"); - Cli::try_parse_from(["rocm", "setup", "status"]).expect("setup status should parse"); - Cli::try_parse_from(["rocm", "setup", "reset"]).expect("setup reset should parse"); - Cli::try_parse_from(["rocm", "models"]).expect("models alias should parse"); - Cli::try_parse_from(["rocm", "comfyui", "status"]).expect("comfyui status should parse"); - Cli::try_parse_from(["rocm", "comfyui", "logs", "--lines", "3"]) - .expect("comfyui logs should parse"); - Cli::try_parse_from(["rocm", "comfyui", "stop"]).expect("comfyui stop should parse"); - Cli::try_parse_from(["rocm", "comfy", "logs"]).expect("comfy alias should parse"); + fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { + let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); + + let selection = select_serve_engine(None, None, Some(&recipe), None); + + assert_eq!( + selection, + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + } + ); + assert_eq!( + serve_engine_selection_line(&selection), + " engine_selection: recipe preferred engine; pass --engine to override; no automatic fallback" + ); + assert_eq!( + serve_model_ref_for_engine("qwen32b", Some(&recipe), "vllm"), + "Qwen/Qwen3-32B-FP8" + ); } #[test] - fn t5_bench_load_clap_parse_smoke() { - // T5: verify BenchCommand::Load parses correctly including comma-separated concurrency. - let cli = Cli::try_parse_from([ - "rocm", - "bench", - "load", - "--endpoint", - "http://x", - "--concurrency", - "1,8,32,64", - ]) - .expect("rocm bench load should parse"); - match cli.command { - Some(Command::Bench { - command: BenchCommand::Load { concurrency, .. }, - }) => { - assert_eq!(concurrency, vec![1u32, 8, 32, 64]); + fn serve_engine_selection_prefers_vllm_for_supported_gpus() { + let summary = rocm_core::HostGpuSummary { + therock_family: Some("gfx90a".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + + let selection = select_serve_engine(None, None, None, Some(&summary)); + + // vLLM is unsupported on native Windows, so the GPU-family preference is gated + // off there and selection falls back to the platform default. + let expected = if cfg!(windows) { + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "platform default", } - other => panic!("expected Bench/Load, got {other:?}"), - } + } else { + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "detected ROCm GPU family prefers vLLM", + } + }; + assert_eq!(selection, expected); } #[test] - fn setup_reset_cli_output_is_plain_and_persists_first_time_prompt() -> Result<()> { - let (_root, paths) = test_paths("setup-reset-cli"); - let mut config = RocmCliConfig { - onboarding_dismissed: true, - setup: rocm_core::SetupConfig { - completed: true, - therock_venv: Some(paths.data_dir.join("envs").join("default")), - cli_install_dir: None, - }, - ..Default::default() + fn serve_engine_selection_keeps_recipe_engine_when_gpu_preference_is_incompatible() { + // qwen-smoke is a tiny GGUF model that only Lemonade can serve and has no vLLM + // recipe. Even on a vLLM-preferred GPU it must stay on Lemonade rather than being + // forced onto vLLM (which cannot load the GGUF and fails to locate the model). + let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); + let summary = rocm_core::HostGpuSummary { + therock_family: Some("gfx90a".to_owned()), + ..rocm_core::HostGpuSummary::default() }; - config.provider_config_mut("openai").enabled = true; - config.save(&paths)?; - - let rendered = reset_setup_prompt_state(&paths, &mut config)?; - assert!(rendered.contains("Setup will show again")); - assert!(rendered.contains("ROCm installs were not deleted")); - assert!(rendered.contains("API keys")); - assert!(!rendered.contains("request plan")); - assert!(!rendered.contains("planner:")); - assert!(!rendered.contains("tool_schema")); + let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); - let saved = RocmCliConfig::load(&paths)?; - assert!(!saved.onboarding_dismissed); - assert!(!saved.setup.completed); - assert!(saved.setup.therock_venv.is_some()); - assert!(saved.provider_enabled("openai")); - Ok(()) + assert_eq!( + selection, + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + } + ); } #[test] - fn setup_status_reports_completed_active_runtime() -> Result<()> { - let (root, paths) = test_paths("setup-status-completed-runtime"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-status", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - let config = RocmCliConfig { - default_runtime_id: Some(manifest.runtime_id.clone()), - active_runtime_key: Some(manifest.runtime_key.clone()), - setup: rocm_core::SetupConfig { - completed: true, - therock_venv: Some(manifest.install_root.clone()), - cli_install_dir: None, - }, - ..Default::default() + fn serve_qwen_uses_vllm_with_hf_repo_on_vllm_preferred_gpu() { + // The qwen alias serves the GGUF via Lemonade by default, but on a vLLM-preferred + // GPU it must serve the non-GGUF Hugging Face repo through vLLM. + let recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); + let summary = rocm_core::HostGpuSummary { + therock_family: Some("gfx94X-dcgpu".to_owned()), + ..rocm_core::HostGpuSummary::default() }; - let rendered = render_setup_status_text(&paths, &config)?; - - assert!(rendered.contains("status: completed"), "{rendered}"); - assert!( - rendered.contains(&format!( - "install folder: {}", - manifest.install_root.display() - )), - "{rendered}" + let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); + // On native Windows the vLLM preference is gated off, so the qwen recipe stays on + // its own preferred engine (Lemonade) instead of being routed to vLLM. + let expected = if cfg!(windows) { + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + } + } else { + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "detected ROCm GPU family prefers vLLM", + } + }; + assert_eq!(selection, expected); + assert_eq!( + serve_model_ref_for_engine("qwen", Some(&recipe), "vllm"), + "Qwen/Qwen3-4B-Instruct-2507" ); - assert!( - rendered.contains("active_runtime_key: release-pip-gfx120x-all-status"), - "{rendered}" + // Lemonade keeps the GGUF canonical id. + assert_eq!( + serve_model_ref_for_engine("qwen", Some(&recipe), "lemonade"), + "Qwen3-4B-Instruct-2507-GGUF" ); - assert!( - rendered.contains("active_runtime_id: therock-release:gfx120X-all"), - "{rendered}" + } + + #[test] + fn sdk_install_auto_engine_selection_prefers_vllm_for_supported_families() { + // vLLM is unsupported on native Windows, so the SDK family preference is gated + // off there and resolves to None. + let expected = if cfg!(windows) { None } else { Some("vllm") }; + assert_eq!(preferred_engine_for_sdk_family("gfx90a"), expected); + assert_eq!(preferred_engine_for_sdk_family("gfx94X-dcgpu"), expected); + assert_eq!(preferred_engine_for_sdk_family("gfx120X-all"), None); + } + + #[test] + fn explicit_engine_override_keeps_alias_when_shared_recipe_is_for_another_engine() { + // `qwen-smoke` is a Lemonade-only GGUF recipe (no vLLM engine recipe). + let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); + + // Served under the engine it targets, the alias resolves to the canonical id. + assert_eq!( + serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "lemonade"), + "Qwen3-0.6B-GGUF" ); - assert!( - rendered.contains("active_runtime_status: ready"), - "{rendered}" + // Under an engine the recipe does not support, the raw alias flows through unchanged. + assert_eq!( + serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "vllm"), + "qwen-smoke" ); - assert!(rendered.contains("rocm help"), "{rendered}"); - - let _ = fs::remove_dir_all(root); - Ok(()) } #[test] - fn setup_status_reports_first_time_when_not_completed() -> Result<()> { - let (_root, paths) = test_paths("setup-status-first-time"); - let config = RocmCliConfig::default(); + fn serve_engine_selection_respects_explicit_and_configured_engines() { + let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); - let rendered = render_setup_status_text(&paths, &config)?; + let explicit = select_serve_engine(Some("vllm"), Some("lemonade"), Some(&recipe), None); + let configured = select_serve_engine(None, Some("lemonade"), Some(&recipe), None); - assert!(rendered.contains("status: first-time setup will show")); - assert!(rendered.contains("active_runtime_status: ")); - Ok(()) + assert_eq!( + explicit, + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "explicit --engine", + } + ); + assert_eq!( + configured, + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "configured default_engine", + } + ); } #[test] - fn serve_bind_validation_requires_public_ack() { - validate_bind_host("127.0.0.1", false).unwrap(); - validate_bind_host("localhost", false).unwrap(); - validate_bind_host("::1", false).unwrap(); - let error = validate_bind_host("0.0.0.0", false).unwrap_err(); - assert!( - error.to_string().contains("--allow-public-bind"), - "{error:#}" + fn protocol_engine_recipe_hint_maps_selected_engine_metadata() { + let mut recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); + recipe.engine_recipes = vec![ + rocm_core::ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + required_flags: vec!["--enable-auto-tool-choice".to_owned()], + parser_settings: BTreeMap::from([( + "reasoning_parser".to_owned(), + "qwen3".to_owned(), + )]), + preferred_endpoint: Some(rocm_core::ModelRecipeEndpointRecord { + endpoint_mode: "openai".to_owned(), + settings: BTreeMap::from([("streaming".to_owned(), "true".to_owned())]), + }), + unsupported_combinations: vec![ + rocm_core::ModelRecipeUnsupportedCombinationRecord { + combination: "native Windows GPU serving".to_owned(), + reason: "vLLM ROCm serving is Linux/WSL only".to_owned(), + }, + ], + notes: vec!["adapter hint".to_owned()], + model_id_override: None, + }, + rocm_core::ModelRecipeEngineRecord { + engine: "lemonade".to_owned(), + required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], + parser_settings: BTreeMap::new(), + preferred_endpoint: None, + unsupported_combinations: Vec::new(), + notes: Vec::new(), + model_id_override: None, + }, + ]; + + let hint = protocol_engine_recipe_hint(&recipe, "vllm").expect("vllm hint"); + + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!(hint.engine, "vllm"); + assert_eq!( + hint.required_flags, + vec!["--enable-auto-tool-choice".to_owned()] + ); + assert_eq!( + hint.parser_settings + .get("reasoning_parser") + .map(String::as_str), + Some("qwen3") ); - validate_bind_host("0.0.0.0", true).unwrap(); + assert_eq!( + hint.preferred_endpoint + .as_ref() + .map(|endpoint| endpoint.endpoint_mode.as_str()), + Some("openai") + ); + assert_eq!( + hint.preferred_endpoint + .as_ref() + .and_then(|endpoint| endpoint.settings.get("streaming")) + .map(String::as_str), + Some("true") + ); + assert_eq!(hint.unsupported_combinations.len(), 1); + assert_eq!(hint.notes, vec!["adapter hint".to_owned()]); + let serve_lines = render_serve_engine_recipe_lines(&hint); + assert!(serve_lines.contains( + "engine_recipe_policy: selected-engine required_flags are applied at launch" + )); + assert!(serve_lines.contains("engine_recipe_required_flags: --enable-auto-tool-choice")); + assert!(protocol_engine_recipe_hint(&recipe, "unknown-engine").is_none()); } #[test] - fn resolve_endpoint_auth_loopback_stays_credential_free() { - // Loopback binds never require auth, even if a key is supplied. - for host in ["127.0.0.1", "localhost", "::1"] { - assert_eq!(resolve_endpoint_auth(host, None).unwrap(), None); - assert_eq!(resolve_endpoint_auth(host, Some("ignored")).unwrap(), None); - } + fn tool_call_override_synthesizes_hint_for_vllm_without_recipe() { + // Arbitrary HF repo with no catalog recipe: the explicit override is the + // only source of the parser, and a minimal hint is synthesized to carry it. + let hint = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")) + .expect("an override should synthesize a vllm tool-choice hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!( + hint.required_flags, + vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ] + ); } #[test] - fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { - let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key ")) - .unwrap() - .expect("public bind must have a key"); - assert_eq!(key, "my-key"); + fn tool_call_override_replaces_recipe_authored_parser() { + // Override wins over an authored parser: exactly one `--tool-call-parser`, + // set to the override value, with unrelated flags preserved in order. + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--reasoning-parser".to_owned(), + "qwen3".to_owned(), + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "llama3_json".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_tool_call_override("vllm", Some(existing), Some("hermes")).unwrap(); + assert_eq!( + hint.required_flags, + vec![ + "--reasoning-parser".to_owned(), + "qwen3".to_owned(), + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ] + ); + assert_eq!( + hint.required_flags + .iter() + .filter(|flag| *flag == "--tool-call-parser") + .count(), + 1 + ); } #[test] - fn resolve_endpoint_auth_public_generates_key_when_absent() { - let key = resolve_endpoint_auth("0.0.0.0", None) - .unwrap() - .expect("public bind must generate a key"); - assert_eq!(key.len(), 48); - assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); - } + fn tool_call_override_absent_preserves_recipe_flags_without_guessing() { + // No override: authored recipe metadata flows through unchanged and no + // parser is ever guessed from the model ref. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_tool_call_override("vllm", Some(authored.clone()), None).unwrap(); + assert_eq!(hint.required_flags, authored.required_flags); - #[test] - fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { - let error = resolve_endpoint_auth("0.0.0.0", Some(" ")).unwrap_err(); - assert!(error.to_string().contains("non-empty"), "{error:#}"); + // Unknown model, no recipe, no override: nothing is injected. + assert!(engine_recipe_with_tool_call_override("vllm", None, None).is_none()); + // A blank override is treated as absent. + assert!(engine_recipe_with_tool_call_override("vllm", None, Some(" ")).is_none()); } #[test] - fn resolve_endpoint_auth_public_rejects_embedded_crlf() { - // A supplied key survives `trim()` with embedded CR/LF intact and would - // otherwise be interpolated into a raw `Authorization: Bearer` header, - // injecting an extra header line. It must be rejected at input validation. - for supplied in [ - "good-key\r\nX-Injected: value", - "good-key\nmore", - "line\rreturn", - ] { - let error = resolve_endpoint_auth("0.0.0.0", Some(supplied)).unwrap_err(); - assert!(error.to_string().contains("control character"), "{error:#}"); - } + fn tool_call_override_leaves_non_vllm_engines_untouched() { + // The override is vLLM-specific: other engines are never rewritten. + assert!(engine_recipe_with_tool_call_override("lemonade", None, Some("hermes")).is_none()); + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--some-flag".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_tool_call_override( + "lemonade", + Some(existing.clone()), + Some("hermes"), + ) + .unwrap(); + assert_eq!(hint.required_flags, existing.required_flags); } #[test] - fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { - let (root, paths) = test_paths("drop-orphaned-key-stored"); - let service_id = "svc-orphaned"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); + fn gpu_memory_utilization_absent_without_explicit_flag() { + // rocm-cli ships no default: with nothing supplied the recipe is left + // alone, so vLLM applies its own default rather than one rocm-cli owns. + assert_eq!(parse_gpu_memory_utilization(None).unwrap(), None); + assert!(engine_recipe_with_gpu_memory_utilization_override("vllm", None, None).is_none()); + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec!["--enable-auto-tool-choice".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", Some(authored), None) + .unwrap(); + assert!( + !hint + .required_flags + .iter() + .any(|flag| flag == "--gpu-memory-utilization"), + "no default may be injected: {:?}", + hint.required_flags + ); } #[test] - fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { - // A loopback attempt never stores a key (`freshly_stored == None`), so the - // helper must not panic or error, and no file must appear. - let (root, paths) = test_paths("drop-orphaned-key-loopback"); - let service_id = "svc-loopback"; - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); + fn gpu_memory_utilization_override_reaches_required_flags() { + let value = parse_gpu_memory_utilization(Some("0.35")).unwrap(); + let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", None, value) + .expect("an explicit value should synthesize a vllm hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!( + hint.required_flags, + vec!["--gpu-memory-utilization".to_owned(), "0.35".to_owned()] + ); } #[test] - fn public_bind_fails_closed_for_windows_lemonade_only() { - // Windows + Lemonade + public bind: refuse (cannot enforce the key). - let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); - assert!(error.to_string().contains("lemonade"), "{error:#}"); - // Every other combination is allowed: - ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows - ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows - ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key + fn gpu_memory_utilization_override_replaces_authored_value_and_keeps_others() { + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--gpu-memory-utilization".to_owned(), + "0.8".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override( + "vllm", + Some(existing), + parse_gpu_memory_utilization(Some("1.0")).unwrap(), + ) + .unwrap(); + assert_eq!( + hint.required_flags, + vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + "--gpu-memory-utilization".to_owned(), + "1".to_owned(), + ] + ); } #[test] - fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { - // A stop deletes the key file, so a later restart of a public service - // would otherwise respawn it with no auth at all. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false).unwrap_err(); - let message = error.to_string(); - assert!(message.contains("0.0.0.0"), "{error:#}"); - assert!(message.contains("without authentication"), "{error:#}"); - // Actionable: name the command that mints a fresh key. - assert!(message.contains("--allow-public-bind"), "{error:#}"); - - // A public service that still has its key restarts normally. - ensure_public_service_has_endpoint_key("0.0.0.0", true).unwrap(); + fn gpu_memory_utilization_override_leaves_non_vllm_engines_untouched() { + // The override is vLLM-specific: other engines are never rewritten, with + // or without a recipe of their own. + assert!( + engine_recipe_with_gpu_memory_utilization_override("lemonade", None, Some(0.5)) + .is_none() + ); + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--some-flag".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override( + "lemonade", + Some(existing.clone()), + Some(0.5), + ) + .unwrap(); + assert_eq!(hint.required_flags, existing.required_flags); } #[test] - fn respawn_allows_loopback_services_without_an_endpoint_key() { - // Loopback stays credential-free, so every accepted spelling must pass - // the guard with no key present. - for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + fn gpu_memory_utilization_rejects_out_of_range_and_unparsable_values() { + // An explicit CLI value is never silently ignored (unlike the env-var + // overrides elsewhere): each bad value must produce an actionable error. + for bad in ["0", "0.0", "1.5", "-0.2", "abc", "", "NaN", "inf"] { + let Err(error) = parse_gpu_memory_utilization(Some(bad)) else { + panic!("`{bad}` must be rejected, not silently ignored"); + }; + let message = error.to_string(); + assert!( + message.contains("--gpu-memory-utilization"), + "error for `{bad}` should name the flag: {message}" + ); } + assert_eq!( + parse_gpu_memory_utilization(Some(" 0.5 ")).unwrap(), + Some(0.5) + ); + assert_eq!(parse_gpu_memory_utilization(Some("1")).unwrap(), Some(1.0)); } #[test] - fn restart_refuses_a_public_service_without_a_key_before_stopping_it() { - // The guard runs before the stop, so a refused restart must leave the - // record exactly as it was rather than taking down a running service. - let (root, paths) = test_paths("restart-public-no-key"); - let service_id = "svc-public-nokey"; - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "model-ref", - "canonical/model", - "0.0.0.0", - 11435, - "managed", - std::process::id(), + fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { + // The interactive summary is what a default `rocm serve` prints, so a flag + // the selected engine cannot honor has to be reported through this path — + // not only on the plan path that an interactive run never takes. + let note = "--gpu-memory-utilization applies only to vLLM; ignored for engine 'lemonade'"; + let notes = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], None, + Some(note), None, - Some("gpu_required".to_owned()), ); - record.status = "running".to_owned(); - record.write().unwrap(); - - let error = restart_internal_managed_service(&paths, service_id).unwrap_err(); assert!( - error.to_string().contains("without authentication"), - "{error:#}" + notes.iter().any(|entry| entry == note), + "the ignored-flag note must reach the summary: {notes:?}" ); - // Proves the *ordering*, not just the refusal: had the guard run after - // `stop_internal_managed_service`, the stop would have written - // "stopped". It reaches that state here because the record's only pid is - // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` - // skips the caller's pid), so the stop confirms termination trivially. - let after = load_managed_service(&paths, service_id).unwrap(); - assert_ne!( - after.status, "stopped", - "a refused restart must not stop the service" + let quiet = collect_serve_notes(false, &GpuSelection::Auto, false, &[0], None, None, None); + assert!( + !quiet + .iter() + .any(|entry| entry.contains("--gpu-memory-utilization")), + "nothing to report when the flag was honored: {quiet:?}" ); - let _ = fs::remove_dir_all(root); } - /// A public record with a dead pid and a stored endpoint key, for the - /// liveness-refresh cases below. The port is one nothing listens on, so the - /// refresh's endpoint probe fails and it falls through to the pid check. - fn dead_public_service_with_key( - paths: &AppPaths, - service_id: &str, - port: u16, - ) -> ManagedServiceRecord { - paths.ensure().unwrap(); - let mut record = ManagedServiceRecord::new( - paths, - service_id, + #[test] + fn generation_defaults_inject_override_generation_config_for_vllm() { + // vLLM has no raw sampling flags: all three controls collapse into a single + // `--override-generation-config` JSON with `--max-tokens` mapped to the + // engine's `max_new_tokens` output cap. + let hint = engine_recipe_with_generation_defaults( "vllm", - "model-ref", - "canonical/model", - "0.0.0.0", - port, - "managed", - // A pid far above any plausible live process, as in - // `dead_managed_service_allows_relaunch`. - 999_999_999, - None, None, - None, - ); - record.status = "running".to_owned(); - record.write().unwrap(); - endpoint_keys::store_endpoint_api_key(paths, service_id, "secret-key").unwrap(); - record + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.25), + max_tokens: Some(128), + }, + ) + .expect("vllm defaults are supported") + .expect("supplied defaults should synthesize a vllm hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.required_flags.len(), 2); + assert_eq!(hint.required_flags[0], "--override-generation-config"); + let config: serde_json::Value = + serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); + assert_eq!(config["temperature"], 0.5); + assert_eq!(config["top_p"], 0.25); + assert_eq!(config["max_new_tokens"], 128); } #[test] - fn crashed_public_service_keeps_its_endpoint_key() { - // A crash (OOM kill, host reboot, panic) leaves the record at "running" - // with dead pids and no stop marker. Dropping the key here would make - // the fail-closed respawn guards refuse every later `rocm services - // restart` and every daemon recovery attempt — permanently, because - // nothing can re-mint the key. The refresh happens on every read - // (`rocm services list`), so this must survive it. - let (root, paths) = test_paths("liveness-crash-keeps-key"); - let service_id = "svc-crashed-public"; - let mut record = dead_public_service_with_key(&paths, service_id, 11982); - - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + fn generation_defaults_include_only_supplied_values() { + // Unset controls are omitted so the engine keeps its own defaults. + let hint = engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults { + temperature: Some(0.25), + top_p: None, + max_tokens: None, + }, + ) + .expect("vllm defaults are supported") + .expect("a single supplied default still synthesizes a hint"); + let config: serde_json::Value = + serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); + assert_eq!(config["temperature"], 0.25); + assert!(config.get("top_p").is_none()); + assert!(config.get("max_new_tokens").is_none()); + } - assert!(changed, "a dead service must be demoted to stopped"); - assert_eq!(record.status, "stopped"); + #[test] + fn generation_defaults_merge_with_recipe_authored_config() { + // CLI values win, but authored keys the CLI does not set are preserved and + // exactly one `--override-generation-config` pair remains. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--override-generation-config".to_owned(), + "{\"temperature\":0.9,\"repetition_penalty\":1.1}".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_generation_defaults( + "vllm", + Some(authored), + ServeGenerationDefaults { + temperature: Some(0.25), + top_p: Some(0.5), + max_tokens: None, + }, + ) + .expect("vllm defaults are supported") + .unwrap(); assert_eq!( - endpoint_keys::endpoint_api_key(&paths, service_id).as_deref(), - Some("secret-key"), - "a crashed public service must stay restartable" + hint.required_flags + .iter() + .filter(|flag| *flag == "--override-generation-config") + .count(), + 1 ); - let _ = fs::remove_dir_all(root); + assert_eq!(hint.required_flags[0], "--enable-auto-tool-choice"); + let config: serde_json::Value = + serde_json::from_str(hint.required_flags.last().unwrap()).unwrap(); + assert_eq!(config["temperature"], 0.25); + assert_eq!(config["top_p"], 0.5); + assert_eq!(config["repetition_penalty"], 1.1); } #[test] - fn unconfirmed_stop_clears_the_endpoint_key_once_the_processes_are_gone() { - // The other half: a stop that could not confirm termination leaves the - // key in place (the engine may still be alive and enforcing it) and - // records the intent. Once the processes are observed gone, the deferred - // cleanup runs, so no plaintext secret is stranded for a service the - // operator did ask to stop. - let (root, paths) = test_paths("liveness-pending-stop-clears-key"); - let service_id = "svc-pending-stop"; - let mut record = dead_public_service_with_key(&paths, service_id, 11983); - record.stop_requested_unix_ms = Some(1); - - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + fn generation_defaults_absent_or_non_vllm_pass_through() { + // No controls supplied: the hint flows through unchanged. + assert!( + engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults::default() + ) + .unwrap() + .is_none() + ); + assert!( + engine_recipe_with_generation_defaults( + "unknown", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.5), + max_tokens: Some(64), + }, + ) + .is_err() + ); + } - assert!(changed); - assert_eq!(record.status, "stopped"); - assert_eq!( - endpoint_keys::endpoint_api_key(&paths, service_id), + #[test] + fn generation_defaults_translate_to_lemonade_llama_server_flags() { + let hint = engine_recipe_with_generation_defaults( + "lemonade", None, - "a requested stop must still drop the key" - ); + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.25), + max_tokens: Some(128), + }, + ) + .expect("lemonade defaults are supported") + .expect("defaults synthesize a recipe"); assert_eq!( - record.stop_requested_unix_ms, None, - "the marker is consumed, so the cleanup does not run again" + hint.required_flags, + [ + "--temperature", + "0.5", + "--top-p", + "0.25", + "--n-predict", + "128" + ] ); - let _ = fs::remove_dir_all(root); } #[test] - fn pending_stop_cleanup_runs_even_once_the_record_reads_stopped() { - // Proves the cleanup sits *before* the `managed_service_is_live` gate: - // a record that reached "stopped" by another route (an engine state - // refresh, a concurrent writer) would otherwise early-return and strand - // the key of a service the operator stopped. - let (root, paths) = test_paths("liveness-pending-stop-when-stopped"); - let service_id = "svc-pending-stop-stopped"; - let mut record = dead_public_service_with_key(&paths, service_id, 11984); - record.status = "stopped".to_owned(); - record.stop_requested_unix_ms = Some(1); - - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); - - assert!( - changed, - "consuming the marker is a record change worth writing" + fn generation_defaults_preserve_unset_lemonade_recipe_flags() { + // Only --temperature is supplied via CLI; an authored --top-p already + // present in the recipe must survive untouched, mirroring the vLLM + // merge behavior instead of being deleted. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--top-p".to_owned(), "0.9".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_generation_defaults( + "lemonade", + Some(authored), + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: None, + max_tokens: None, + }, + ) + .expect("lemonade defaults are supported") + .expect("supplied defaults should synthesize a hint"); + assert_eq!( + hint.required_flags, + ["--top-p", "0.9", "--temperature", "0.5"] ); - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert_eq!(record.stop_requested_unix_ms, None); - let _ = fs::remove_dir_all(root); } #[test] - fn endpoint_client_config_shows_key_once_with_bearer_guidance() { - let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); - assert!(rendered.contains("secret-123"), "{rendered}"); - assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); - assert!(rendered.contains("shown only now"), "{rendered}"); + fn engine_recipe_enables_tool_choice_reflects_flags() { + assert!(!engine_recipe_enables_tool_choice(None)); + let without = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], + ..EngineRecipeHint::default() + }; + assert!(!engine_recipe_enables_tool_choice(Some(&without))); + let with = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")); + assert!(engine_recipe_enables_tool_choice(with.as_ref())); } #[test] - fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { - let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); + fn parse_device_policy_defaults_to_gpu_required_without_cpu_fallback() -> Result<()> { + assert_eq!(parse_device_policy(None)?, DevicePolicy::GpuRequired); + assert_eq!(parse_device_policy(Some("gpu"))?, DevicePolicy::GpuRequired); + assert_eq!( + parse_device_policy(Some("gpu_preferred"))?, + DevicePolicy::GpuRequired + ); + let cpu = parse_device_policy(Some("cpu")).unwrap_err().to_string(); + assert!(cpu.contains("CPU mode is not a fallback path")); + Ok(()) + } - let selection = select_serve_engine(None, None, Some(&recipe), None); + fn vram(index: u32, used_mb: u64, total_mb: u64) -> GpuVramUsage { + GpuVramUsage { + index, + used_mb, + total_mb, + } + } + #[test] + fn auto_selection_prefers_lowest_index_idle_gpu() { + // GPU 0 busy (only 5% free), GPU 1 idle, GPU 2 idle. + let usage = [ + vram(0, 182_000, 192_000), + vram(1, 1_000, 192_000), + vram(2, 500, 192_000), + ]; assert_eq!( - selection, - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", - } + select_auto_gpu_index(Some(3), &[], Some(&usage)), + vec![1], + "should skip the busy GPU 0 and pick the lowest idle GPU" ); + } + + #[test] + fn auto_selection_skips_managed_and_busy_gpus_then_picks_most_free() { + // GPU 0 pinned by a managed service; GPU 1 partly used; GPU 2 more free + // but none is fully idle, so pass 2 (most free) applies. + let usage = [ + vram(0, 10_000, 192_000), + vram(1, 120_000, 192_000), + vram(2, 60_000, 192_000), + ]; assert_eq!( - serve_engine_selection_line(&selection), - " engine_selection: recipe preferred engine; pass --engine to override; no automatic fallback" + select_auto_gpu_index(Some(3), &[0], Some(&usage)), + vec![2], + "with no idle GPU, pick the non-busy GPU with the most free VRAM" ); + } + + #[test] + fn auto_selection_pass_two_ranks_by_absolute_free_vram() { + // Heterogeneous VRAM with no fully-idle GPU (so pass 2 applies): + // GPU 0 is a small card with a high free *fraction* (75%) but little + // absolute free memory; GPU 1 is large with a lower fraction (~48%) + // but far more free memory. Auto-selection must prefer GPU 1. + let usage = [vram(0, 6_000, 24_000), vram(1, 100_000, 192_000)]; assert_eq!( - serve_model_ref_for_engine("qwen32b", Some(&recipe), "vllm"), - "Qwen/Qwen3-32B-FP8" + select_auto_gpu_index(Some(2), &[], Some(&usage)), + vec![1], + "pass 2 should rank by absolute free VRAM, not free percentage" ); } #[test] - fn serve_engine_selection_prefers_vllm_for_supported_gpus() { - let summary = rocm_core::HostGpuSummary { - therock_family: Some("gfx90a".to_owned()), - ..rocm_core::HostGpuSummary::default() - }; - - let selection = select_serve_engine(None, None, None, Some(&summary)); - - // vLLM is unsupported on native Windows, so the GPU-family preference is gated - // off there and selection falls back to the platform default. - let expected = if cfg!(windows) { - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "platform default", - } - } else { - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "detected ROCm GPU family prefers vLLM", - } - }; - assert_eq!(selection, expected); + fn auto_selection_falls_back_to_first_non_busy_without_vram() { + assert_eq!(select_auto_gpu_index(Some(4), &[0, 1], None), vec![2]); + // Unknown GPU count: no GPU-0 fallback — defer to the engine device probe. + assert_eq!(select_auto_gpu_index(None, &[], None), Vec::::new()); } #[test] - fn serve_engine_selection_keeps_recipe_engine_when_gpu_preference_is_incompatible() { - // qwen-smoke is a tiny GGUF model that only Lemonade can serve and has no vLLM - // recipe. Even on a vLLM-preferred GPU it must stay on Lemonade rather than being - // forced onto vLLM (which cannot load the GGUF and fails to locate the model). - let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); - let summary = rocm_core::HostGpuSummary { - therock_family: Some("gfx90a".to_owned()), - ..rocm_core::HostGpuSummary::default() - }; + fn validate_pinned_gpu_index_rejects_out_of_range() { + // Index equal to or beyond the detected count is rejected. + let error = validate_pinned_gpu_index(4, Some(4)).expect_err("index 4 is out of range"); + assert!(error.to_string().contains("out of range")); + assert!(validate_pinned_gpu_index(9, Some(2)).is_err()); + } - let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); + #[test] + fn validate_pinned_gpu_index_accepts_in_range_or_unknown_count() { + // In-range index pins exactly that ordinal. + assert_eq!(validate_pinned_gpu_index(0, Some(1)).unwrap(), vec![0]); + assert_eq!(validate_pinned_gpu_index(3, Some(4)).unwrap(), vec![3]); + // Unknown count (amd-smi unavailable) is allowed through unvalidated. + assert_eq!(validate_pinned_gpu_index(7, None).unwrap(), vec![7]); + } - assert_eq!( - selection, - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", - } - ); + #[test] + fn parse_gpu_vram_usage_reads_gpu_data_envelope() { + let value = json!({ + "gpu_data": [ + {"gpu": 0, "mem_usage": {"used_vram": {"value": 1000}, "total_vram": {"value": 192_000}}}, + {"gpu": 1, "mem_usage": {"used_vram": {"value": 50000}, "total_vram": {"value": 192_000}}} + ] + }); + let rows = parse_gpu_vram_usage(&value); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].index, 0); + assert_eq!(rows[0].used_mb, 1000); + assert_eq!(rows[1].index, 1); + assert!((rows[1].free_fraction().unwrap() - (142_000.0 / 192_000.0)).abs() < 1e-9); } #[test] - fn serve_qwen_uses_vllm_with_hf_repo_on_vllm_preferred_gpu() { - // The qwen alias serves the GGUF via Lemonade by default, but on a vLLM-preferred - // GPU it must serve the non-GGUF Hugging Face repo through vLLM. - let recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); - let summary = rocm_core::HostGpuSummary { - therock_family: Some("gfx94X-dcgpu".to_owned()), + fn gpu_low_memory_warning_flags_busy_selected_gpu() { + let usage = [vram(0, 182_000, 192_000), vram(1, 1_000, 192_000)]; + let warning = gpu_low_memory_warning(&[0], Some(&usage)).expect("warning for busy GPU 0"); + assert!(warning.contains("GPU 0")); + assert!(warning.contains("free")); + assert!(gpu_low_memory_warning(&[1], Some(&usage)).is_none()); + assert!(gpu_low_memory_warning(&[0], None).is_none()); + } + + fn host_gpu(gfx_target: &str) -> rocm_core::HostGpuSummary { + rocm_core::HostGpuSummary { + gfx_target: Some(gfx_target.to_owned()), ..rocm_core::HostGpuSummary::default() - }; + } + } - let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); - // On native Windows the vLLM preference is gated off, so the qwen recipe stays on - // its own preferred engine (Lemonade) instead of being routed to vLLM. - let expected = if cfg!(windows) { - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", - } - } else { - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "detected ROCm GPU family prefers vLLM", - } - }; - assert_eq!(selection, expected); + #[test] + fn unified_memory_apu_suppresses_the_vram_capacity_warning() { + // Strix Halo as reported by amd-smi: a 4 GiB BIOS carveout with 1.6 GiB + // free (40%, under the 90% bar) on a machine whose engine actually + // serves out of ~128 GiB of shared system RAM. The old reading — + // "only 1.6 GiB of 4.0 GiB free" — describes a pool the allocator does + // not use, so it must not reach the user. + let carveout = [vram(0, 2_458, 4_096)]; + assert!( + gpu_low_memory_warning(&[0], Some(&carveout)).is_some(), + "the underlying threshold still trips; only the serve-plan wrapper withholds it" + ); assert_eq!( - serve_model_ref_for_engine("qwen", Some(&recipe), "vllm"), - "Qwen/Qwen3-4B-Instruct-2507" + serve_gpu_low_memory_warning(&[0], Some(&carveout), Some(&host_gpu("gfx1151"))), + None ); - // Lemonade keeps the GGUF canonical id. + } + + #[test] + fn discrete_gpu_still_warns_when_genuinely_busy() { + let busy = [vram(0, 182_000, 192_000)]; + let warning = serve_gpu_low_memory_warning(&[0], Some(&busy), Some(&host_gpu("gfx1100"))) + .expect("a discrete card that is 95% full still warrants a warning"); + assert!(warning.contains("GPU 0")); + assert!(warning.contains("`--gpu `")); + + // Baseline: the same discrete card, mostly free, stays quiet. + let idle = [vram(0, 1_000, 192_000)]; assert_eq!( - serve_model_ref_for_engine("qwen", Some(&recipe), "lemonade"), - "Qwen3-4B-Instruct-2507-GGUF" + serve_gpu_low_memory_warning(&[0], Some(&idle), Some(&host_gpu("gfx1100"))), + None ); } #[test] - fn sdk_install_auto_engine_selection_prefers_vllm_for_supported_families() { - // vLLM is unsupported on native Windows, so the SDK family preference is gated - // off there and resolves to None. - let expected = if cfg!(windows) { None } else { Some("vllm") }; - assert_eq!(preferred_engine_for_sdk_family("gfx90a"), expected); - assert_eq!(preferred_engine_for_sdk_family("gfx94X-dcgpu"), expected); - assert_eq!(preferred_engine_for_sdk_family("gfx120X-all"), None); + fn apu_verdict_does_not_silence_a_second_gpu() { + // `gfx_is_apu_family` classifies a part, while host detection reports one + // target for the whole machine. On an APU+dGPU laptop that target cannot + // be attributed to the selected ordinal, so the warning must survive + // rather than be suppressed on the discrete card's behalf. + let hybrid = [vram(0, 2_458, 4_096), vram(1, 182_000, 192_000)]; + assert!( + serve_gpu_low_memory_warning(&[1], Some(&hybrid), Some(&host_gpu("gfx1151"))).is_some() + ); + assert!(vram_capacity_is_meaningful(Some("gfx1151"), 2)); + // An unknown target is never treated as unified memory. + assert!(vram_capacity_is_meaningful(None, 1)); } #[test] - fn explicit_engine_override_keeps_alias_when_shared_recipe_is_for_another_engine() { - // `qwen-smoke` is a Lemonade-only GGUF recipe (no vLLM engine recipe). - let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); + fn driver_plan_ubuntu_2404_uses_official_dkms_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=ubuntu +VERSION_ID="24.04" +VERSION_CODENAME=noble +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let commands = plan + .commands + .iter() + .map(|command| command.command.as_str()) + .collect::>(); - // Served under the engine it targets, the alias resolves to the canonical id. - assert_eq!( - serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "lemonade"), - "Qwen3-0.6B-GGUF" + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + assert!( + plan.preflight_checks + .iter() + .any(|check| check.contains("sudo -v")) ); - // Under an engine the recipe does not support, the raw alias flows through unchanged. - assert_eq!( - serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "vllm"), - "qwen-smoke" + assert!( + commands + .iter() + .any(|command| command.contains("linux-headers-$(uname -r)")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("linux-modules-extra-$(uname -r)")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("repo.radeon.com/graphics")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("amdgpu-dkms")) ); + let rendered = render_driver_install_plan(&plan, false, false); + assert!(rendered.contains("approval: required")); + assert!(rendered.contains("preflight_checks:")); + assert!(rendered.contains("root access: run as root, or ensure `sudo -v` succeeds")); + assert!(rendered.contains("execution_commands:")); + assert!(rendered.contains("Prepare: sudo apt-get update")); + assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); + assert!(rendered.contains("post_reboot_check_commands:")); + assert!(rendered.contains("dkms status amdgpu")); + assert!(rendered.contains("rerun with --yes")); } #[test] - fn serve_engine_selection_respects_explicit_and_configured_engines() { - let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); + fn driver_reconcile_without_state_gives_non_privileged_guidance() -> Result<()> { + let (root, paths) = test_paths("driver-reconcile-empty"); - let explicit = select_serve_engine(Some("vllm"), Some("lemonade"), Some(&recipe), None); - let configured = select_serve_engine(None, Some("lemonade"), Some(&recipe), None); + let rendered = reconcile_driver_install(&paths)?; + assert!(rendered.contains("driver install reconciliation")); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("privileged_commands: ")); + assert!(rendered.contains("no prior driver execution state found")); + assert!(rendered.contains("rocm install driver --dkms")); + assert!(!driver_install_state_path(&paths).exists()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn driver_reconcile_updates_state_after_reboot() -> Result<()> { + let (root, paths) = test_paths("driver-reconcile-state"); + let pre_driver = rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "not_detected".to_owned(), + detail: None, + }; + let current_driver = rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "amdgpu_available".to_owned(), + detail: Some("/dev/kfd is present".to_owned()), + }; + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: Some(2), + pre_driver, + post_driver: None, + boot_id_at_execution: Some("old-boot".to_owned()), + reboot_required: true, + reboot_observed: false, + commands: vec!["sudo apt-get install -y amdgpu-dkms".to_owned()], + reconciled_at_unix_ms: None, + reconciliation: None, + }; + let checks = vec![ + DriverPassiveCheck { + name: "/dev/kfd".to_owned(), + status: "present".to_owned(), + detail: "KFD device node".to_owned(), + }, + DriverPassiveCheck { + name: "/dev/dri/renderD*".to_owned(), + status: "missing".to_owned(), + detail: "DRM render node".to_owned(), + }, + ]; + + let rendered = reconcile_driver_install_state( + &paths, + &mut state, + current_driver, + Some("new-boot".to_owned()), + checks, + )?; + let saved = read_driver_install_state(&paths)?.expect("state should be saved"); + + assert!(rendered.contains("reboot_observed: true")); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("privileged_commands: ")); + assert!(rendered.contains("driver_status: amdgpu_available")); + assert!(rendered.contains("passive_check_summary: total=2 present=1 missing=1")); + assert!(rendered.contains("/dev/dri/renderD*: missing")); + assert!(rendered.contains("missing passive checks")); + assert!(saved.reboot_observed); + assert!(saved.reconciled_at_unix_ms.is_some()); assert_eq!( - explicit, - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "explicit --engine", - } - ); - assert_eq!( - configured, - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "configured default_engine", - } + saved + .reconciliation + .as_ref() + .map(|value| value.driver.status.as_str()), + Some("amdgpu_available") ); + let reconciliation = saved.reconciliation.as_ref().expect("reconciliation saved"); + assert_eq!(reconciliation.check_summary.total, 2); + assert_eq!(reconciliation.check_summary.present, 1); + assert_eq!(reconciliation.check_summary.missing, 1); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn protocol_engine_recipe_hint_maps_selected_engine_metadata() { - let mut recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); - recipe.engine_recipes = vec![ - rocm_core::ModelRecipeEngineRecord { - engine: "vllm".to_owned(), - required_flags: vec!["--enable-auto-tool-choice".to_owned()], - parser_settings: BTreeMap::from([( - "reasoning_parser".to_owned(), - "qwen3".to_owned(), - )]), - preferred_endpoint: Some(rocm_core::ModelRecipeEndpointRecord { - endpoint_mode: "openai".to_owned(), - settings: BTreeMap::from([("streaming".to_owned(), "true".to_owned())]), - }), - unsupported_combinations: vec![ - rocm_core::ModelRecipeUnsupportedCombinationRecord { - combination: "native Windows GPU serving".to_owned(), - reason: "vLLM ROCm serving is Linux/WSL only".to_owned(), - }, - ], - notes: vec!["adapter hint".to_owned()], - model_id_override: None, + fn driver_passive_check_summary_counts_non_present_as_missing() { + let summary = summarize_driver_passive_checks(&[ + DriverPassiveCheck { + name: "/dev/kfd".to_owned(), + status: "present".to_owned(), + detail: "KFD".to_owned(), }, - rocm_core::ModelRecipeEngineRecord { - engine: "lemonade".to_owned(), - required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], - parser_settings: BTreeMap::new(), - preferred_endpoint: None, - unsupported_combinations: Vec::new(), - notes: Vec::new(), - model_id_override: None, + DriverPassiveCheck { + name: "/dev/dri/renderD*".to_owned(), + status: "missing".to_owned(), + detail: "render".to_owned(), }, - ]; + DriverPassiveCheck { + name: "dkms".to_owned(), + status: "error".to_owned(), + detail: "dkms status failed".to_owned(), + }, + ]); + + assert_eq!(summary.total, 3); + assert_eq!(summary.present, 1); + assert_eq!(summary.missing, 2); + } + + #[test] + fn driver_plan_default_linux_preflight_has_no_execution_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=ubuntu +VERSION_ID="24.04" +VERSION_CODENAME=noble +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, false); + let rendered = render_driver_install_plan(&plan, false, false); - let hint = protocol_engine_recipe_hint(&recipe, "vllm").expect("vllm hint"); + assert!(plan.supported); + assert!(!plan.mutating); + assert!(plan.commands.is_empty()); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("execution_commands: ")); + assert!(!rendered.contains("sudo apt-get")); + assert!(rendered.contains("add --dkms")); + } - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!(hint.engine, "vllm"); - assert_eq!( - hint.required_flags, - vec!["--enable-auto-tool-choice".to_owned()] - ); + #[test] + fn resolve_shell_default_template_uses_default_when_env_unset() { + let _env = ScopedTestEnv::new(); + // A made-up variable name that nothing else sets, cleared under the lock, + // isolates the default path. assert_eq!( - hint.parser_settings - .get("reasoning_parser") - .map(String::as_str), - Some("qwen3") + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION:-7.2.4}"), + "7.2.4" ); + } + + #[test] + fn resolve_shell_default_template_prefers_env_value_when_set() { + let mut env = ScopedTestEnv::new(); + let var = "ROCM_CLI_TEST_REPO_VERSION_OVERRIDE"; + env.set(var, "9.9.9"); assert_eq!( - hint.preferred_endpoint - .as_ref() - .map(|endpoint| endpoint.endpoint_mode.as_str()), - Some("openai") + resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), + "9.9.9" ); + } + + #[test] + fn resolve_shell_default_template_treats_empty_env_as_unset() { + let mut env = ScopedTestEnv::new(); + let var = "ROCM_CLI_TEST_REPO_VERSION_EMPTY"; + env.set(var, ""); assert_eq!( - hint.preferred_endpoint - .as_ref() - .and_then(|endpoint| endpoint.settings.get("streaming")) - .map(String::as_str), - Some("true") + resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), + "7.2.4" ); - assert_eq!(hint.unsupported_combinations.len(), 1); - assert_eq!(hint.notes, vec!["adapter hint".to_owned()]); - let serve_lines = render_serve_engine_recipe_lines(&hint); - assert!(serve_lines.contains( - "engine_recipe_policy: selected-engine required_flags are applied at launch" - )); - assert!(serve_lines.contains("engine_recipe_required_flags: --enable-auto-tool-choice")); - assert!(protocol_engine_recipe_hint(&recipe, "unknown-engine").is_none()); } #[test] - fn tool_call_override_synthesizes_hint_for_vllm_without_recipe() { - // Arbitrary HF repo with no catalog recipe: the explicit override is the - // only source of the parser, and a minimal hint is synthesized to carry it. - let hint = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")) - .expect("an override should synthesize a vllm tool-choice hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + fn resolve_shell_default_template_passes_through_non_template() { + assert_eq!(resolve_shell_default_template("7.2.4"), "7.2.4"); + } + + #[test] + fn resolve_shell_default_template_leaves_bare_var_untouched() { + let _env = ScopedTestEnv::new(); + // No `:-default`, so there is nothing to resolve to; the input must pass + // through unchanged rather than being partially rewritten. assert_eq!( - hint.required_flags, - vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ] + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION}"), + "${ROCM_CLI_TEST_UNSET_REPO_VERSION}" ); } #[test] - fn tool_call_override_replaces_recipe_authored_parser() { - // Override wins over an authored parser: exactly one `--tool-call-parser`, - // set to the override value, with unrelated flags preserved in order. - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--reasoning-parser".to_owned(), - "qwen3".to_owned(), - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "llama3_json".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = - engine_recipe_with_tool_call_override("vllm", Some(existing), Some("hermes")).unwrap(); + fn resolve_shell_default_template_leaves_nested_default_untouched() { + let _env = ScopedTestEnv::new(); + // A nested default is beyond the flat matcher; returning the literal + // input keeps a `${B:-x}` fragment from leaking as a "resolved" value. assert_eq!( - hint.required_flags, - vec![ - "--reasoning-parser".to_owned(), - "qwen3".to_owned(), - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ] + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}"), + "${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}" ); - assert_eq!( - hint.required_flags - .iter() - .filter(|flag| *flag == "--tool-call-parser") - .count(), - 1 + } + + #[test] + fn driver_plan_dry_run_repo_version_line_is_resolved() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Regression for the dry-run output leaking the raw shell placeholder on + // the `repo_version:` line instead of the effective version. + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(rendered.contains("repo_version: 7.2.4")); + assert!(!rendered.contains("repo_version: ${ROCM_CLI_AMDGPU_VERSION:-7.2.4}")); + } + + #[test] + fn driver_plan_debian_12_omits_linux_modules_extra() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=debian +VERSION_ID="12" +VERSION_CODENAME=bookworm +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(plan.supported); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("linux-headers-$(uname -r)")); + assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); + assert!(rendered.contains("amdgpu-dkms")); + assert!(rendered.contains("dry run only")); + } + + #[test] + fn driver_plan_rhel_97_uses_documented_dnf_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + assert!(rendered.contains("`dnf` package manager is available")); + assert!(rendered.contains("kernel-headers-$(uname -r)")); + assert!(rendered.contains("kernel-devel-$(uname -r)")); + assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + assert!(rendered.contains("approval: required")); + } + + #[test] + fn driver_plan_oracle_linux_101_uses_el_10_uek_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=ol +VERSION_ID="10.1" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(plan.supported); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("kernel-uek-devel-$(uname -r)")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/10/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el10.noarch.rpm")); + assert!(rendered.contains("dry run only")); + } + + #[test] + fn driver_plan_rocky_97_uses_el_dnf_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=rocky +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!( + rendered + .contains("sudo dnf install -y kernel-headers kernel-devel kernel-devel-matched") ); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.7/")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); } #[test] - fn tool_call_override_absent_preserves_recipe_flags_without_guessing() { - // No override: authored recipe metadata flows through unchanged and no - // parser is ever guessed from the model ref. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = - engine_recipe_with_tool_call_override("vllm", Some(authored.clone()), None).unwrap(); - assert_eq!(hint.required_flags, authored.required_flags); + fn driver_plan_rocky_94_uses_el_dnf_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Rocky 9.x point releases must resolve like RHEL 9.x, not just 9.7. + let os_release = r#" +ID=rocky +VERSION_ID="9.4" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); - // Unknown model, no recipe, no override: nothing is injected. - assert!(engine_recipe_with_tool_call_override("vllm", None, None).is_none()); - // A blank override is treated as absent. - assert!(engine_recipe_with_tool_call_override("vllm", None, Some(" ")).is_none()); + assert!(plan.supported); + assert!(plan.mutating); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.4/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); } #[test] - fn tool_call_override_leaves_non_vllm_engines_untouched() { - // The override is vLLM-specific: other engines are never rewritten. - assert!(engine_recipe_with_tool_call_override("lemonade", None, Some("hermes")).is_none()); - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--some-flag".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_tool_call_override( - "lemonade", - Some(existing.clone()), - Some("hermes"), - ) - .unwrap(); - assert_eq!(hint.required_flags, existing.required_flags); + fn driver_plan_rocky_8_and_10_remain_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // AMD documents Rocky Linux 9 only; keep the driver matrix scoped to 9.x. + for version in ["8.10", "10.0"] { + let os_release = format!("\nID=rocky\nVERSION_ID=\"{version}\"\n"); + let plan = build_driver_install_plan(&test_examine("linux", false), &os_release, true); + assert!(!plan.supported, "rocky {version} should be unsupported"); + assert!(!plan.mutating, "rocky {version} must not mutate"); + assert!( + plan.commands.is_empty(), + "rocky {version} must emit no commands" + ); + } } #[test] - fn gpu_memory_utilization_absent_without_explicit_flag() { - // rocm-cli ships no default: with nothing supplied the recipe is left - // alone, so vLLM applies its own default rather than one rocm-cli owns. - assert_eq!(parse_gpu_memory_utilization(None).unwrap(), None); - assert!(engine_recipe_with_gpu_memory_utilization_override("vllm", None, None).is_none()); - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec!["--enable-auto-tool-choice".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", Some(authored), None) - .unwrap(); + fn driver_plan_debian_uses_intended_ubuntu_suite() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // AMD's documented Debian install deliberately serves Debian from the + // Ubuntu-suite graphics tree (Debian 12 -> jammy). Lock that in and + // ensure the plan explains the mapping is intentional. + let os_release = r#" +ID=debian +VERSION_ID="12" +VERSION_CODENAME=bookworm +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(plan.supported); + assert_eq!(plan.codename, "jammy"); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); assert!( - !hint - .required_flags - .iter() - .any(|flag| flag == "--gpu-memory-utilization"), - "no default may be injected: {:?}", - hint.required_flags + plan.reason + .contains("intentionally uses AMD's Ubuntu-suite repository") ); } #[test] - fn gpu_memory_utilization_override_reaches_required_flags() { - let value = parse_gpu_memory_utilization(Some("0.35")).unwrap(); - let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", None, value) - .expect("an explicit value should synthesize a vllm hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!( - hint.required_flags, - vec!["--gpu-memory-utilization".to_owned(), "0.35".to_owned()] - ); + fn driver_plan_sles_157_uses_documented_zypper_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=sles +VERSION_ID="15.7" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(rendered.contains("`zypper` package manager is available")); + assert!(rendered.contains("SUSEConnect")); + assert!(rendered.contains("sle-module-desktop-applications/15.7/x86_64")); + assert!(rendered.contains("sudo zypper install -y kernel-default-devel")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/sle/15.7/")); + assert!(rendered.contains("sudo zypper --no-gpg-checks install -y")); + assert!(rendered.contains("Execute: sudo zypper install -y amdgpu-dkms")); + assert!(rendered.contains("approval: required")); } #[test] - fn gpu_memory_utilization_override_replaces_authored_value_and_keeps_others() { - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--gpu-memory-utilization".to_owned(), - "0.8".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override( - "vllm", - Some(existing), - parse_gpu_memory_utilization(Some("1.0")).unwrap(), - ) - .unwrap(); - assert_eq!( - hint.required_flags, - vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - "--gpu-memory-utilization".to_owned(), - "1".to_owned(), - ] - ); + fn driver_plan_unsupported_linux_is_non_mutating() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=fedora +VERSION_ID="41" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("no driver commands will be executed")); + assert!(!rendered.contains("sudo dnf install -y amdgpu-dkms")); } #[test] - fn gpu_memory_utilization_override_leaves_non_vllm_engines_untouched() { - // The override is vLLM-specific: other engines are never rewritten, with - // or without a recipe of their own. - assert!( - engine_recipe_with_gpu_memory_utilization_override("lemonade", None, Some(0.5)) - .is_none() - ); - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--some-flag".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override( - "lemonade", - Some(existing.clone()), - Some(0.5), - ) - .unwrap(); - assert_eq!(hint.required_flags, existing.required_flags); + fn windows_install_driver_is_validate_only() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let plan = build_driver_install_plan(&test_examine("windows", false), "", true); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert_eq!(plan.policy, "windows_validate_only"); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("execution_commands: ")); + assert!(rendered.contains("post_reboot_checks:")); + assert!(rendered.contains("use `rocm examine`")); + assert!(rendered.contains("rocm examine")); + assert!(plan.commands.is_empty()); } #[test] - fn gpu_memory_utilization_rejects_out_of_range_and_unparsable_values() { - // An explicit CLI value is never silently ignored (unlike the env-var - // overrides elsewhere): each bad value must produce an actionable error. - for bad in ["0", "0.0", "1.5", "-0.2", "abc", "", "NaN", "inf"] { - let Err(error) = parse_gpu_memory_utilization(Some(bad)) else { - panic!("`{bad}` must be rejected, not silently ignored"); - }; - let message = error.to_string(); - assert!( - message.contains("--gpu-memory-utilization"), - "error for `{bad}` should name the flag: {message}" - ); - } - assert_eq!( - parse_gpu_memory_utilization(Some(" 0.5 ")).unwrap(), - Some(0.5) - ); - assert_eq!(parse_gpu_memory_utilization(Some("1")).unwrap(), Some(1.0)); + fn wsl_install_driver_uses_rocdxg_guidance_without_dkms() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let plan = build_driver_install_plan(&test_examine("linux", true), "", true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert_eq!(plan.policy, "wsl_rocdxg"); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("execution_commands: ")); + assert!(rendered.contains("scripts/wsl_setup_rocdxg.sh")); + assert!(!rendered.contains("amdgpu-dkms")); } + // EAI-7406: distro selection must honor `/etc/os-release` `ID_LIKE`, so that + // Debian/Ubuntu-family and RHEL-rebuild derivatives that share their base + // version scheme are matched to the correct apt (`ubuntu/`) or EL + // (`el/`) plan instead of falling through to the unsupported plan. + #[test] - fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { - // The interactive summary is what a default `rocm serve` prints, so a flag - // the selected engine cannot honor has to be reported through this path — - // not only on the plan path that an interactive run never takes. - let note = "--gpu-memory-utilization applies only to vLLM; ignored for engine 'lemonade'"; - let notes = collect_serve_notes( - false, - &GpuSelection::Auto, - false, - &[0], - None, - Some(note), - None, - ); - assert!( - notes.iter().any(|entry| entry == note), - "the ignored-flag note must reach the summary: {notes:?}" - ); + fn driver_plan_ubuntu_derivative_via_id_like_matches_ubuntu_plan() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Pop!_OS reports its own ID but reuses Ubuntu's version + repositories. + let os_release = r#" +ID=pop +VERSION_ID="22.04" +VERSION_CODENAME=jammy +ID_LIKE="ubuntu debian" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + // Ubuntu-family derivatives ship the Ubuntu kernel, so linux-modules-extra applies. + assert!(rendered.contains("linux-modules-extra-$(uname -r)")); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); + assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); + } + + #[test] + fn driver_plan_debian_derivative_via_id_like_matches_debian_plan() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // A Debian derivative (e.g. LMDE) that shares Debian's version scheme. + let os_release = r#" +ID=lmde +VERSION_ID="12" +ID_LIKE=debian +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); - let quiet = collect_serve_notes(false, &GpuSelection::Auto, false, &[0], None, None, None); - assert!( - !quiet - .iter() - .any(|entry| entry.contains("--gpu-memory-utilization")), - "nothing to report when the flag was honored: {quiet:?}" - ); + assert!(plan.supported); + // Debian-family maps to the Ubuntu jammy repo and omits linux-modules-extra. + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); + assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); + assert!(rendered.contains("amdgpu-dkms")); } #[test] - fn generation_defaults_inject_override_generation_config_for_vllm() { - // vLLM has no raw sampling flags: all three controls collapse into a single - // `--override-generation-config` JSON with `--max-tokens` mapped to the - // engine's `max_new_tokens` output cap. - let hint = engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.25), - max_tokens: Some(128), - }, - ) - .expect("vllm defaults are supported") - .expect("supplied defaults should synthesize a vllm hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.required_flags.len(), 2); - assert_eq!(hint.required_flags[0], "--override-generation-config"); - let config: serde_json::Value = - serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); - assert_eq!(config["temperature"], 0.5); - assert_eq!(config["top_p"], 0.25); - assert_eq!(config["max_new_tokens"], 128); + fn driver_plan_almalinux_via_id_like_uses_el_9_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // AlmaLinux is a RHEL rebuild: standard kernel, served from the el/ path. + let os_release = r#" +ID=almalinux +VERSION_ID="9.6" +ID_LIKE="rhel centos fedora" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + // EL rebuilds use the vendor-neutral el/ repo path, not rhel/. + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.6/")); + assert!(!rendered.contains("/rhel/9.6/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); + // el9 uses the version-aware standard-kernel prepare commands. + assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); } #[test] - fn generation_defaults_include_only_supplied_values() { - // Unset controls are omitted so the engine keeps its own defaults. - let hint = engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults { - temperature: Some(0.25), - top_p: None, - max_tokens: None, - }, - ) - .expect("vllm defaults are supported") - .expect("a single supplied default still synthesizes a hint"); - let config: serde_json::Value = - serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); - assert_eq!(config["temperature"], 0.25); - assert!(config.get("top_p").is_none()); - assert!(config.get("max_new_tokens").is_none()); + fn driver_plan_almalinux_8_via_id_like_uses_el_major_path() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=almalinux +VERSION_ID="8.10" +ID_LIKE="rhel centos fedora" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + // EL 8 is served from the major-version path (el/8), matching AMD docs. + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/8/")); + assert!(rendered.contains("-1.el8.noarch.rpm")); + // el8 has no kernel-devel-matched package. + assert!(!rendered.contains("kernel-devel-matched")); + assert!(rendered.contains("kernel-devel-$(uname -r)")); } #[test] - fn generation_defaults_merge_with_recipe_authored_config() { - // CLI values win, but authored keys the CLI does not set are preserved and - // exactly one `--override-generation-config` pair remains. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--override-generation-config".to_owned(), - "{\"temperature\":0.9,\"repetition_penalty\":1.1}".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_generation_defaults( - "vllm", - Some(authored), - ServeGenerationDefaults { - temperature: Some(0.25), - top_p: Some(0.5), - max_tokens: None, - }, - ) - .expect("vllm defaults are supported") - .unwrap(); - assert_eq!( - hint.required_flags - .iter() - .filter(|flag| *flag == "--override-generation-config") - .count(), - 1 - ); - assert_eq!(hint.required_flags[0], "--enable-auto-tool-choice"); - let config: serde_json::Value = - serde_json::from_str(hint.required_flags.last().unwrap()).unwrap(); - assert_eq!(config["temperature"], 0.25); - assert_eq!(config["top_p"], 0.5); - assert_eq!(config["repetition_penalty"], 1.1); + fn driver_plan_id_like_with_unsupported_version_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // A Debian-family derivative whose VERSION_ID does not align with any + // AMD-documented Debian version must not fabricate a plan. + let os_release = r#" +ID=lmde +VERSION_ID="6" +ID_LIKE=debian +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(!rendered.contains("amdgpu-dkms")); } #[test] - fn generation_defaults_absent_or_non_vllm_pass_through() { - // No controls supplied: the hint flows through unchanged. - assert!( - engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults::default() - ) - .unwrap() - .is_none() - ); - assert!( - engine_recipe_with_generation_defaults( - "unknown", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.5), - max_tokens: Some(64), - }, - ) - .is_err() - ); + fn driver_plan_exact_id_takes_precedence_over_id_like() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // An exact RHEL match must keep the rhel/ path even though ID_LIKE=fedora. + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +ID_LIKE=fedora +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); + assert!(!rendered.contains("/el/9.7/")); } #[test] - fn generation_defaults_translate_to_lemonade_llama_server_flags() { - let hint = engine_recipe_with_generation_defaults( - "lemonade", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.25), - max_tokens: Some(128), - }, - ) - .expect("lemonade defaults are supported") - .expect("defaults synthesize a recipe"); - assert_eq!( - hint.required_flags, - [ - "--temperature", - "0.5", - "--top-p", - "0.25", - "--n-predict", - "128" - ] - ); + fn driver_plan_oracle_linux_off_arm_version_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Oracle Linux reports `ID_LIKE=fedora` (not rhel) and boots UEK. An OL + // version outside the exact `ol` arm must NOT be captured by the EL + // fallback, which would emit non-UEK kernel commands that cannot install. + let os_release = r#" +ID=ol +VERSION_ID="9.6" +ID_LIKE=fedora +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(!rendered.contains("kernel-devel-matched")); + assert!(!rendered.contains("amdgpu-dkms")); } #[test] - fn generation_defaults_preserve_unset_lemonade_recipe_flags() { - // Only --temperature is supplied via CLI; an authored --top-p already - // present in the recipe must survive untouched, mirroring the vLLM - // merge behavior instead of being deleted. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--top-p".to_owned(), "0.9".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_generation_defaults( - "lemonade", - Some(authored), - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: None, - max_tokens: None, - }, - ) - .expect("lemonade defaults are supported") - .expect("supplied defaults should synthesize a hint"); - assert_eq!( - hint.required_flags, - ["--top-p", "0.9", "--temperature", "0.5"] - ); + fn driver_plan_opensuse_leap_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // openSUSE Leap shares SLES's version scheme but has no SUSEConnect/SCC + // entitlement, so it must not be matched to the SLES plan. + let os_release = r#" +ID=opensuse-leap +VERSION_ID="15.7" +ID_LIKE="suse opensuse" +"#; + let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(!rendered.contains("SUSEConnect")); + assert!(!rendered.contains("amdgpu-dkms")); } #[test] - fn engine_recipe_enables_tool_choice_reflects_flags() { - assert!(!engine_recipe_enables_tool_choice(None)); - let without = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], - ..EngineRecipeHint::default() + fn resolve_engine_selection_uses_default_runtime_after_engine_prefs() { + let mut config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() }; - assert!(!engine_recipe_enables_tool_choice(Some(&without))); - let with = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")); - assert!(engine_recipe_enables_tool_choice(with.as_ref())); - } - #[test] - fn parse_device_policy_defaults_to_gpu_required_without_cpu_fallback() -> Result<()> { - assert_eq!(parse_device_policy(None)?, DevicePolicy::GpuRequired); - assert_eq!(parse_device_policy(Some("gpu"))?, DevicePolicy::GpuRequired); + let selection = resolve_engine_selection(&config, "vllm", None, None); assert_eq!( - parse_device_policy(Some("gpu_preferred"))?, - DevicePolicy::GpuRequired + selection.runtime_id.as_deref(), + Some("therock-release:gfx120X-all") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_default_runtime_id") ); - let cpu = parse_device_policy(Some("cpu")).unwrap_err().to_string(); - assert!(cpu.contains("CPU mode is not a fallback path")); - Ok(()) - } - fn vram(index: u32, used_mb: u64, total_mb: u64) -> GpuVramUsage { - GpuVramUsage { - index, - used_mb, - total_mb, - } - } + config.active_runtime_key = Some("release-pip-gfx120x-all-7-13-0".to_owned()); + let selection = resolve_engine_selection(&config, "vllm", None, None); + assert_eq!( + selection.runtime_id.as_deref(), + Some("release-pip-gfx120x-all-7-13-0") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") + ); - #[test] - fn auto_selection_prefers_lowest_index_idle_gpu() { - // GPU 0 busy (only 5% free), GPU 1 idle, GPU 2 idle. - let usage = [ - vram(0, 182_000, 192_000), - vram(1, 1_000, 192_000), - vram(2, 500, 192_000), - ]; + config.engine_config_mut("vllm").preferred_runtime_id = + Some("therock-nightly:gfx120X-all".to_owned()); + let selection = resolve_engine_selection(&config, "vllm", None, None); assert_eq!( - select_auto_gpu_index(Some(3), &[], Some(&usage)), - vec![1], - "should skip the busy GPU 0 and pick the lowest idle GPU" + selection.runtime_id.as_deref(), + Some("release-pip-gfx120x-all-7-13-0") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") ); - } - #[test] - fn auto_selection_skips_managed_and_busy_gpus_then_picks_most_free() { - // GPU 0 pinned by a managed service; GPU 1 partly used; GPU 2 more free - // but none is fully idle, so pass 2 (most free) applies. - let usage = [ - vram(0, 10_000, 192_000), - vram(1, 120_000, 192_000), - vram(2, 60_000, 192_000), - ]; + config.active_runtime_key = None; + let selection = resolve_engine_selection(&config, "vllm", None, None); assert_eq!( - select_auto_gpu_index(Some(3), &[0], Some(&usage)), - vec![2], - "with no idle GPU, pick the non-busy GPU with the most free VRAM" + selection.runtime_id.as_deref(), + Some("therock-nightly:gfx120X-all") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_preferred_runtime_id") ); } #[test] - fn auto_selection_pass_two_ranks_by_absolute_free_vram() { - // Heterogeneous VRAM with no fully-idle GPU (so pass 2 applies): - // GPU 0 is a small card with a high free *fraction* (75%) but little - // absolute free memory; GPU 1 is large with a lower fraction (~48%) - // but far more free memory. Auto-selection must prefer GPU 1. - let usage = [vram(0, 6_000, 24_000), vram(1, 100_000, 192_000)]; + fn engine_selection_uses_single_ready_runtime_without_active_marker() -> Result<()> { + let (root, paths) = test_paths("single-ready-runtime-selection"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + assert_eq!( - select_auto_gpu_index(Some(2), &[], Some(&usage)), - vec![1], - "pass 2 should rank by absolute free VRAM, not free percentage" + selection.runtime_id.as_deref(), + Some(manifest.runtime_key.as_str()) ); + assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn auto_selection_falls_back_to_first_non_busy_without_vram() { - assert_eq!(select_auto_gpu_index(Some(4), &[0, 1], None), vec![2]); - // Unknown GPU count: no GPU-0 fallback — defer to the engine device probe. - assert_eq!(select_auto_gpu_index(None, &[], None), Vec::::new()); - } + fn engine_selection_keeps_multiple_ready_runtimes_explicit() -> Result<()> { + let (root, paths) = test_paths("multiple-ready-runtime-selection"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; - #[test] - fn validate_pinned_gpu_index_rejects_out_of_range() { - // Index equal to or beyond the detected count is rejected. - let error = validate_pinned_gpu_index(4, Some(4)).expect_err("index 4 is out of range"); - assert!(error.to_string().contains("out of range")); - assert!(validate_pinned_gpu_index(9, Some(2)).is_err()); + assert!(selection.runtime_id.is_none()); + assert!(selection.env_id.is_none()); + assert!(selection.source.is_none()); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn validate_pinned_gpu_index_accepts_in_range_or_unknown_count() { - // In-range index pins exactly that ordinal. - assert_eq!(validate_pinned_gpu_index(0, Some(1)).unwrap(), vec![0]); - assert_eq!(validate_pinned_gpu_index(3, Some(4)).unwrap(), vec![3]); - // Unknown count (amd-smi unavailable) is allowed through unvalidated. - assert_eq!(validate_pinned_gpu_index(7, None).unwrap(), vec![7]); - } + fn render_config_text_includes_default_runtime() { + let (_root, paths) = test_paths("config-default-runtime"); + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; - #[test] - fn parse_gpu_vram_usage_reads_gpu_data_envelope() { - let value = json!({ - "gpu_data": [ - {"gpu": 0, "mem_usage": {"used_vram": {"value": 1000}, "total_vram": {"value": 192_000}}}, - {"gpu": 1, "mem_usage": {"used_vram": {"value": 50000}, "total_vram": {"value": 192_000}}} - ] - }); - let rows = parse_gpu_vram_usage(&value); - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].index, 0); - assert_eq!(rows[0].used_mb, 1000); - assert_eq!(rows[1].index, 1); - assert!((rows[1].free_fraction().unwrap() - (142_000.0 / 192_000.0)).abs() < 1e-9); + let rendered = render_config_text(&paths, &config); + + assert!(rendered.contains("default_runtime_id: therock-release:gfx120X-all")); + assert!(rendered.contains("active_runtime_key: ")); } #[test] - fn gpu_low_memory_warning_flags_busy_selected_gpu() { - let usage = [vram(0, 182_000, 192_000), vram(1, 1_000, 192_000)]; - let warning = gpu_low_memory_warning(&[0], Some(&usage)).expect("warning for busy GPU 0"); - assert!(warning.contains("GPU 0")); - assert!(warning.contains("free")); - assert!(gpu_low_memory_warning(&[1], Some(&usage)).is_none()); - assert!(gpu_low_memory_warning(&[0], None).is_none()); - } + fn render_config_text_includes_telemetry_policy() { + let (_root, paths) = test_paths("config-telemetry-policy"); + let mut config = RocmCliConfig::default(); - fn host_gpu(gfx_target: &str) -> rocm_core::HostGpuSummary { - rocm_core::HostGpuSummary { - gfx_target: Some(gfx_target.to_owned()), - ..rocm_core::HostGpuSummary::default() - } + let local = render_config_text(&paths, &config); + assert!(local.contains("telemetry_mode: local")); + assert!(local.contains("telemetry_policy: local amd-smi inspection only")); + assert!(local.contains("no external reporting is implemented")); + assert!(local.contains(" providers:")); + assert!(local.contains(" local: enabled")); + assert!(local.contains(" openai: disabled")); + assert!(local.contains(" anthropic: disabled")); + + config.telemetry.mode = TELEMETRY_MODE_OFF.to_owned(); + config.provider_config_mut("openai").enabled = true; + let off = render_config_text(&paths, &config); + assert!(off.contains("telemetry_mode: off")); + assert!(off.contains("telemetry_policy: disabled")); + assert!(off.contains("no local polling")); + assert!(off.contains(" openai: enabled")); } #[test] - fn unified_memory_apu_suppresses_the_vram_capacity_warning() { - // Strix Halo as reported by amd-smi: a 4 GiB BIOS carveout with 1.6 GiB - // free (40%, under the 90% bar) on a machine whose engine actually - // serves out of ~128 GiB of shared system RAM. The old reading — - // "only 1.6 GiB of 4.0 GiB free" — describes a pool the allocator does - // not use, so it must not reach the user. - let carveout = [vram(0, 2_458, 4_096)]; - assert!( - gpu_low_memory_warning(&[0], Some(&carveout)).is_some(), - "the underlying threshold still trips; only the serve-plan wrapper withholds it" + fn engine_install_runtime_selection_requires_configured_runtime() -> Result<()> { + let (root, paths) = test_paths("engine-install-runtime-selection"); + let error = + resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "vllm", None) + .unwrap_err() + .to_string(); + assert!(error.contains("no active ROCm runtime is configured")); + assert_eq!( + resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "lemonade", None)?, + format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), + ); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + + let config = RocmCliConfig { + active_runtime_key: Some("release-pip-gfx120x-all".to_owned()), + ..RocmCliConfig::default() + }; + assert_eq!( + resolve_engine_install_runtime_id(&paths, &config, "vllm", None)?, + "release-pip-gfx120x-all" ); assert_eq!( - serve_gpu_low_memory_warning(&[0], Some(&carveout), Some(&host_gpu("gfx1151"))), - None + resolve_engine_install_runtime_id( + &paths, + &config, + "vllm", + Some("therock-release:gfx120X-all".to_owned()) + )?, + "release-pip-gfx120x-all" ); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn discrete_gpu_still_warns_when_genuinely_busy() { - let busy = [vram(0, 182_000, 192_000)]; - let warning = serve_gpu_low_memory_warning(&[0], Some(&busy), Some(&host_gpu("gfx1100"))) - .expect("a discrete card that is 95% full still warrants a warning"); - assert!(warning.contains("GPU 0")); - assert!(warning.contains("`--gpu `")); + fn runtime_selector_recovers_setup_runtime_registry_from_local_manifest() -> Result<()> { + let (root, paths) = test_paths("runtime-selector-recover-setup"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-local-manifest", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + let install_root = manifest.install_root.clone(); + let mut config = RocmCliConfig { + default_runtime_id: Some(manifest.runtime_id.clone()), + active_runtime_key: Some(manifest.runtime_key.clone()), + ..RocmCliConfig::default() + }; + config.setup.completed = true; + config.setup.therock_venv = Some(install_root.clone()); + config.save(&paths)?; - // Baseline: the same discrete card, mostly free, stays quiet. - let idle = [vram(0, 1_000, 192_000)]; + let rebased_paths = paths.with_managed_root(install_root, false); + let rebased_registry = runtime_registry_dir(&rebased_paths); + let _ = fs::remove_dir_all(&rebased_registry); + + assert!(!runtime_manifest_path(&rebased_paths, &manifest.runtime_key).is_file()); assert_eq!( - serve_gpu_low_memory_warning(&[0], Some(&idle), Some(&host_gpu("gfx1100"))), - None + resolve_runtime_selector_to_exact_key( + &rebased_paths, + &manifest.runtime_key, + "test active runtime" + )?, + manifest.runtime_key ); - } + assert!(runtime_manifest_path(&rebased_paths, &manifest.runtime_key).is_file()); - #[test] - fn apu_verdict_does_not_silence_a_second_gpu() { - // `gfx_is_apu_family` classifies a part, while host detection reports one - // target for the whole machine. On an APU+dGPU laptop that target cannot - // be attributed to the selected ordinal, so the warning must survive - // rather than be suppressed on the discrete card's behalf. - let hybrid = [vram(0, 2_458, 4_096), vram(1, 182_000, 192_000)]; - assert!( - serve_gpu_low_memory_warning(&[1], Some(&hybrid), Some(&host_gpu("gfx1151"))).is_some() - ); - assert!(vram_capacity_is_meaningful(Some("gfx1151"), 2)); - // An unknown target is never treated as unified memory. - assert!(vram_capacity_is_meaningful(None, 1)); + let rendered = render_runtimes_text(&rebased_paths, &config)?; + assert!(rendered.contains("release-pip-gfx120x-all-local-manifest")); + assert!(rendered.contains("status=ready")); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_ubuntu_2404_uses_official_dkms_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=ubuntu -VERSION_ID="24.04" -VERSION_CODENAME=noble -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let commands = plan - .commands - .iter() - .map(|command| command.command.as_str()) - .collect::>(); + fn sdk_install_finalization_activates_runtime_and_setup_root() -> Result<()> { + let (root, paths) = test_paths("sdk-install-finalization"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-finalized", + "therock-release:gfx120X-all", + "7.13.0", + 42, + )?; - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - assert!( - plan.preflight_checks - .iter() - .any(|check| check.contains("sudo -v")) - ); - assert!( - commands - .iter() - .any(|command| command.contains("linux-headers-$(uname -r)")) + let finalized = finalize_successful_sdk_install(&paths)? + .context("sdk install finalization should select the installed runtime")?; + let rebased_paths = paths.with_managed_root(manifest.install_root.clone(), false); + let config = RocmCliConfig::load(&rebased_paths)?; + + assert_eq!(finalized.runtime_key, manifest.runtime_key); + assert_eq!( + config.default_runtime_id.as_deref(), + Some(manifest.runtime_id.as_str()) ); - assert!( - commands - .iter() - .any(|command| command.contains("linux-modules-extra-$(uname -r)")) + assert!(config.setup.completed); + assert_eq!( + config.setup.therock_venv.as_deref(), + Some(manifest.install_root.as_path()) ); - assert!( - commands - .iter() - .any(|command| command.contains("repo.radeon.com/graphics")) + assert_eq!( + config.active_runtime_key.as_deref(), + Some(manifest.runtime_key.as_str()) ); - assert!( - commands - .iter() - .any(|command| command.contains("amdgpu-dkms")) + assert_eq!( + config.default_runtime_id.as_deref(), + Some(manifest.runtime_id.as_str()) ); - let rendered = render_driver_install_plan(&plan, false, false); - assert!(rendered.contains("approval: required")); - assert!(rendered.contains("preflight_checks:")); - assert!(rendered.contains("root access: run as root, or ensure `sudo -v` succeeds")); - assert!(rendered.contains("execution_commands:")); - assert!(rendered.contains("Prepare: sudo apt-get update")); - assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); - assert!(rendered.contains("post_reboot_check_commands:")); - assert!(rendered.contains("dkms status amdgpu")); - assert!(rendered.contains("rerun with --yes")); + assert!(runtime_manifest_path(&rebased_paths, &manifest.runtime_key).is_file()); + assert!(active_runtime_marker_path(&rebased_paths).is_file()); + + let success = render_sdk_install_success(&finalized); + assert!(success.contains("ROCm SDK installed successfully.")); + assert!(success.contains("next step: run `rocm help`")); + assert!(success.contains(&manifest.install_root.display().to_string())); + assert!(!success.contains("config:")); + assert!(!success.contains("marker:")); + + let mut examine = String::new(); + append_examine_runtime_state(&mut examine, &rebased_paths, &config)?; + assert!(examine.contains("active_runtime_status: ready")); + assert!(examine.contains("setup_runtime_root:")); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_reconcile_without_state_gives_non_privileged_guidance() -> Result<()> { - let (root, paths) = test_paths("driver-reconcile-empty"); + fn env_root_for_runtime_uses_runtime_install_root() -> Result<()> { + let (root, paths) = test_paths("engine-env-root-runtime"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; - let rendered = reconcile_driver_install(&paths)?; + let engine_root = env_root_for_runtime(&paths, "vllm", &manifest.runtime_key)?; - assert!(rendered.contains("driver install reconciliation")); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("privileged_commands: ")); - assert!(rendered.contains("no prior driver execution state found")); - assert!(rendered.contains("rocm install driver --dkms")); - assert!(!driver_install_state_path(&paths).exists()); + assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); + assert_eq!( + env_root_for_runtime(&paths, "lemonade", &manifest.runtime_key)?, + None + ); let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn driver_reconcile_updates_state_after_reboot() -> Result<()> { - let (root, paths) = test_paths("driver-reconcile-state"); - let pre_driver = rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "not_detected".to_owned(), - detail: None, - }; - let current_driver = rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "amdgpu_available".to_owned(), - detail: Some("/dev/kfd is present".to_owned()), - }; - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: Some(2), - pre_driver, - post_driver: None, - boot_id_at_execution: Some("old-boot".to_owned()), - reboot_required: true, - reboot_observed: false, - commands: vec!["sudo apt-get install -y amdgpu-dkms".to_owned()], - reconciled_at_unix_ms: None, - reconciliation: None, + fn env_root_for_engine_install_uses_active_runtime_root_for_lemonade() -> Result<()> { + let (root, paths) = test_paths("lemonade-engine-env-root-runtime"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + let config = RocmCliConfig { + active_runtime_key: Some(manifest.runtime_key.clone()), + ..RocmCliConfig::default() }; - let checks = vec![ - DriverPassiveCheck { - name: "/dev/kfd".to_owned(), - status: "present".to_owned(), - detail: "KFD device node".to_owned(), - }, - DriverPassiveCheck { - name: "/dev/dri/renderD*".to_owned(), - status: "missing".to_owned(), - detail: "DRM render node".to_owned(), - }, - ]; - let rendered = reconcile_driver_install_state( + let engine_root = + env_root_for_engine_install(&paths, &config, "lemonade", "lemonade-embeddable")?; + + assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn engine_runtime_selection_rejects_ambiguous_default_runtime_id() -> Result<()> { + let (root, paths) = test_paths("engine-runtime-ambiguous-default"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + write_test_pip_runtime( &paths, - &mut state, - current_driver, - Some("new-boot".to_owned()), - checks, + "vllm-source-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 2, )?; - let saved = read_driver_install_state(&paths)?.expect("state should be saved"); + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; - assert!(rendered.contains("reboot_observed: true")); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("privileged_commands: ")); - assert!(rendered.contains("driver_status: amdgpu_available")); - assert!(rendered.contains("passive_check_summary: total=2 present=1 missing=1")); - assert!(rendered.contains("/dev/dri/renderD*: missing")); - assert!(rendered.contains("missing passive checks")); - assert!(saved.reboot_observed); - assert!(saved.reconciled_at_unix_ms.is_some()); + let error = resolve_engine_install_runtime_id(&paths, &config, "vllm", None) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple installed runtimes")); + assert!(error.contains("rocm runtimes activate ")); + + let selection = resolve_engine_selection(&config, "vllm", None, None); + let error = validate_engine_selection_runtime(&paths, selection) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple installed runtimes")); + + let selection = + resolve_engine_selection(&config, "vllm", Some("release-pip-gfx120x-all"), None); + let selection = validate_engine_selection_runtime(&paths, selection)?; assert_eq!( - saved - .reconciliation - .as_ref() - .map(|value| value.driver.status.as_str()), - Some("amdgpu_available") + selection.runtime_id.as_deref(), + Some("release-pip-gfx120x-all") ); - let reconciliation = saved.reconciliation.as_ref().expect("reconciliation saved"); - assert_eq!(reconciliation.check_summary.total, 2); - assert_eq!(reconciliation.check_summary.present, 1); - assert_eq!(reconciliation.check_summary.missing, 1); let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn driver_passive_check_summary_counts_non_present_as_missing() { - let summary = summarize_driver_passive_checks(&[ - DriverPassiveCheck { - name: "/dev/kfd".to_owned(), - status: "present".to_owned(), - detail: "KFD".to_owned(), - }, - DriverPassiveCheck { - name: "/dev/dri/renderD*".to_owned(), - status: "missing".to_owned(), - detail: "render".to_owned(), - }, - DriverPassiveCheck { - name: "dkms".to_owned(), - status: "error".to_owned(), - detail: "dkms status failed".to_owned(), - }, - ]); + fn render_runtimes_text_reports_missing_configured_active_runtime() -> Result<()> { + let (root, paths) = test_paths("runtime-active-missing"); + let config = RocmCliConfig { + active_runtime_key: Some("missing-runtime-key".to_owned()), + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; - assert_eq!(summary.total, 3); - assert_eq!(summary.present, 1); - assert_eq!(summary.missing, 2); + let rendered = render_runtimes_text(&paths, &config)?; + + assert!(rendered.contains("active_runtime_key: missing-runtime-key")); + assert!(rendered.contains( + "active_status: missing manifest for active_runtime_key=missing-runtime-key" + )); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_default_linux_preflight_has_no_execution_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=ubuntu -VERSION_ID="24.04" -VERSION_CODENAME=noble -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, false); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_lists_display_build_date_from_version_string() -> Result<()> { + let (root, paths) = test_paths("runtime-build-date-display"); + let runtime_key = "release-pip-gfx120x-all-7-14-0a20260601"; + let manifest = write_test_pip_runtime( + &paths, + runtime_key, + "therock-release:gfx120X-all", + "7.14.0a20260601", + 20, + )?; + let config = RocmCliConfig { + active_runtime_key: Some(manifest.runtime_key.clone()), + default_runtime_id: Some(manifest.runtime_id), + ..RocmCliConfig::default() + }; - assert!(plan.supported); - assert!(!plan.mutating); - assert!(plan.commands.is_empty()); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("execution_commands: ")); - assert!(!rendered.contains("sudo apt-get")); - assert!(rendered.contains("add --dkms")); + let runtimes = render_runtimes_text(&paths, &config)?; + assert!(runtimes.contains("version=7.14.0a20260601 (build 2026-06-01)")); + + let mut examine = String::new(); + append_examine_runtime_state(&mut examine, &paths, &config)?; + assert!(examine.contains("active_runtime_version: 7.14.0a20260601 (build 2026-06-01)")); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn resolve_shell_default_template_uses_default_when_env_unset() { - let _env = ScopedTestEnv::new(); - // A made-up variable name that nothing else sets, cleared under the lock, - // isolates the default path. + fn runtime_activation_records_exact_key_and_rollback() -> Result<()> { + let (root, paths) = test_paths("runtime-activation"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-12-0", + "therock-release:gfx120X-all", + "7.12.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + + let mut config = RocmCliConfig::default(); + let first = activate_runtime(&paths, &mut config, "release-pip-gfx120x-all-7-12-0")?; + assert_eq!(first.previous_runtime_key, None); assert_eq!( - resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION:-7.2.4}"), - "7.2.4" + config.active_runtime_key.as_deref(), + Some("release-pip-gfx120x-all-7-12-0") ); - } - #[test] - fn resolve_shell_default_template_prefers_env_value_when_set() { - let mut env = ScopedTestEnv::new(); - let var = "ROCM_CLI_TEST_REPO_VERSION_OVERRIDE"; - env.set(var, "9.9.9"); + let second = activate_runtime(&paths, &mut config, "release-pip-gfx120x-all-7-13-0")?; assert_eq!( - resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), - "9.9.9" + second.previous_runtime_key.as_deref(), + Some("release-pip-gfx120x-all-7-12-0") ); - } - - #[test] - fn resolve_shell_default_template_treats_empty_env_as_unset() { - let mut env = ScopedTestEnv::new(); - let var = "ROCM_CLI_TEST_REPO_VERSION_EMPTY"; - env.set(var, ""); assert_eq!( - resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), - "7.2.4" + config.default_runtime_id.as_deref(), + Some("therock-release:gfx120X-all") + ); + assert_eq!( + config.active_runtime_key.as_deref(), + Some("release-pip-gfx120x-all-7-13-0") + ); + assert_eq!( + config.previous_runtime_key.as_deref(), + Some("release-pip-gfx120x-all-7-12-0") ); - } - - #[test] - fn resolve_shell_default_template_passes_through_non_template() { - assert_eq!(resolve_shell_default_template("7.2.4"), "7.2.4"); - } - #[test] - fn resolve_shell_default_template_leaves_bare_var_untouched() { - let _env = ScopedTestEnv::new(); - // No `:-default`, so there is nothing to resolve to; the input must pass - // through unchanged rather than being partially rewritten. + let marker: ActiveRuntimeMarker = + serde_json::from_slice(&fs::read(active_runtime_marker_path(&paths))?)?; + assert_eq!(marker.runtime_key, "release-pip-gfx120x-all-7-13-0"); assert_eq!( - resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION}"), - "${ROCM_CLI_TEST_UNSET_REPO_VERSION}" + marker.previous_runtime_key.as_deref(), + Some("release-pip-gfx120x-all-7-12-0") ); - } - #[test] - fn resolve_shell_default_template_leaves_nested_default_untouched() { - let _env = ScopedTestEnv::new(); - // A nested default is beyond the flat matcher; returning the literal - // input keeps a `${B:-x}` fragment from leaking as a "resolved" value. + let rendered = render_runtimes_text(&paths, &config)?; + assert!(rendered.contains("* release-pip-gfx120x-all-7-13-0")); + assert!(rendered.contains("- release-pip-gfx120x-all-7-12-0")); + assert!(rendered.contains("status=ready")); + + let rolled_back = rollback_runtime(&paths, &mut config)?; + assert_eq!(rolled_back.runtime_key, "release-pip-gfx120x-all-7-12-0"); assert_eq!( - resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}"), - "${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}" + config.previous_runtime_key.as_deref(), + Some("release-pip-gfx120x-all-7-13-0") ); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_dry_run_repo_version_line_is_resolved() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Regression for the dry-run output leaking the raw shell placeholder on - // the `repo_version:` line instead of the effective version. - let os_release = r#" -ID=rhel -VERSION_ID="9.7" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, true); + fn runtime_activation_rejects_ambiguous_runtime_id() -> Result<()> { + let (root, paths) = test_paths("runtime-ambiguous"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-12-0", + "therock-release:gfx120X-all", + "7.12.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let mut config = RocmCliConfig::default(); - assert!(rendered.contains("repo_version: 7.2.4")); - assert!(!rendered.contains("repo_version: ${ROCM_CLI_AMDGPU_VERSION:-7.2.4}")); - } + let error = activate_runtime(&paths, &mut config, "therock-release:gfx120X-all") + .unwrap_err() + .to_string(); - #[test] - fn driver_plan_debian_12_omits_linux_modules_extra() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=debian -VERSION_ID="12" -VERSION_CODENAME=bookworm -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, true); + assert!(error.contains("matches multiple installed runtimes")); + assert!(error.contains("release-pip-gfx120x-all-7-12-0")); + assert!(error.contains("release-pip-gfx120x-all-7-13-0")); - assert!(plan.supported); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("linux-headers-$(uname -r)")); - assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains("amdgpu-dkms")); - assert!(rendered.contains("dry run only")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_rhel_97_uses_documented_dnf_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=rhel -VERSION_ID="9.7" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_activation_rejects_unusable_manifest() -> Result<()> { + let (root, paths) = test_paths("runtime-unusable"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + fs::remove_file(manifest.install_root.join("Scripts").join("python.exe")).ok(); + fs::remove_file(manifest.install_root.join("bin").join("python")).ok(); + let mut config = RocmCliConfig::default(); - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - assert!(rendered.contains("`dnf` package manager is available")); - assert!(rendered.contains("kernel-headers-$(uname -r)")); - assert!(rendered.contains("kernel-devel-$(uname -r)")); - assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); - assert!(rendered.contains("approval: required")); - } + let error = activate_runtime(&paths, &mut config, "release-pip-gfx120x-all-7-13-0") + .unwrap_err() + .to_string(); - #[test] - fn driver_plan_oracle_linux_101_uses_el_10_uek_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=ol -VERSION_ID="10.1" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, true); + assert!(error.contains("runtime Python executable is missing")); - assert!(plan.supported); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("kernel-uek-devel-$(uname -r)")); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/10/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el10.noarch.rpm")); - assert!(rendered.contains("dry run only")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_rocky_97_uses_el_dnf_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=rocky -VERSION_ID="9.7" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_import_records_read_only_manifest_without_mutating_runtime_root() -> Result<()> { + let (root, paths) = test_paths("runtime-import"); + let manifest = write_test_pip_runtime( + &paths, + "external-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let exported_manifest = root.join("external-runtime.json"); + fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; + fs::remove_file(runtime_manifest_path(&paths, &manifest.runtime_key))?; + fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; - assert!(plan.supported); + let imported = import_runtime_manifest(&paths, &exported_manifest, false)?; + + assert!(imported.read_only); + let canonical_export = exported_manifest.canonicalize()?; + assert_eq!( + imported.imported_from.as_deref(), + Some(canonical_export.as_path()) + ); assert!( - rendered - .contains("sudo dnf install -y kernel-headers kernel-devel kernel-devel-matched") + !manifest + .install_root + .join(".rocm-cli-runtime.json") + .exists() ); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.7/")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); - } - #[test] - fn driver_plan_rocky_94_uses_el_dnf_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Rocky 9.x point releases must resolve like RHEL 9.x, not just 9.7. - let os_release = r#" -ID=rocky -VERSION_ID="9.4" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + let imported_registry: therock::InstalledRuntimeManifest = serde_json::from_slice( + &fs::read(runtime_manifest_path(&paths, &manifest.runtime_key))?, + )?; + assert!(imported_registry.read_only); - assert!(plan.supported); - assert!(plan.mutating); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.4/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + let mut config = RocmCliConfig::default(); + activate_runtime(&paths, &mut config, &manifest.runtime_key)?; + assert_eq!( + config.active_runtime_key.as_deref(), + Some("external-pip-gfx120x-all-7-13-0") + ); + + let rendered = render_runtimes_text(&paths, &config)?; + assert!(rendered.contains("mode=read-only")); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_rocky_8_and_10_remain_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // AMD documents Rocky Linux 9 only; keep the driver matrix scoped to 9.x. - for version in ["8.10", "10.0"] { - let os_release = format!("\nID=rocky\nVERSION_ID=\"{version}\"\n"); - let plan = build_driver_install_plan(&test_examine("linux", false), &os_release, true); - assert!(!plan.supported, "rocky {version} should be unsupported"); - assert!(!plan.mutating, "rocky {version} must not mutate"); - assert!( - plan.commands.is_empty(), - "rocky {version} must emit no commands" - ); - } + fn runtime_import_refuses_to_overwrite_without_replace() -> Result<()> { + let (root, paths) = test_paths("runtime-import-replace"); + let manifest = write_test_pip_runtime( + &paths, + "external-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let exported_manifest = root.join("external-runtime.json"); + fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; + + let error = import_runtime_manifest(&paths, &exported_manifest, false) + .unwrap_err() + .to_string(); + + assert!(error.contains("already exists")); + import_runtime_manifest(&paths, &exported_manifest, true)?; + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_debian_uses_intended_ubuntu_suite() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // AMD's documented Debian install deliberately serves Debian from the - // Ubuntu-suite graphics tree (Debian 12 -> jammy). Lock that in and - // ensure the plan explains the mapping is intentional. - let os_release = r#" -ID=debian -VERSION_ID="12" -VERSION_CODENAME=bookworm -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, true); + fn runtime_adopt_records_read_only_manifest_from_probe() -> Result<()> { + let (root, paths) = test_paths("runtime-adopt"); + let external_root = root.join("external-therock-venv"); + let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python_executable = scripts_dir.join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + let sdk_root = external_root + .join("Lib") + .join("site-packages") + .join("rocm_sdk"); + let sdk_bin = sdk_root.join("bin"); + fs::create_dir_all(&scripts_dir)?; + fs::create_dir_all(&sdk_bin)?; + let amdhip = sdk_bin.join(if cfg!(windows) { + "amdhip64_7.dll" + } else { + "libamdhip64.so" + }); + let hipblas = sdk_bin.join(if cfg!(windows) { + "hipblas.dll" + } else { + "libhipblas.so" + }); + fs::write(&python_executable, "python")?; + fs::write(&amdhip, "amdhip")?; + fs::write(&hipblas, "hipblas")?; - assert!(plan.supported); - assert_eq!(plan.codename, "jammy"); - assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); - assert!( - plan.reason - .contains("intentionally uses AMD's Ubuntu-suite repository") - ); - } + let adopted = adopt_runtime_from_probe( + &paths, + AdoptRuntimeRequest { + python_executable, + install_root: external_root.clone(), + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), + replace: false, + }, + therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root.clone()), + bin_path: Some(sdk_bin.clone()), + runtime_roots: vec![sdk_root], + bin_paths: vec![sdk_bin.clone()], + library_paths: vec![sdk_bin], + resolved_libraries: vec![ + therock::RocmSdkLibraryProbe { + shortname: "amdhip64".to_owned(), + paths: vec![amdhip], + }, + therock::RocmSdkLibraryProbe { + shortname: "hipblas".to_owned(), + paths: vec![hipblas], + }, + ], + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }, + )?; - #[test] - fn driver_plan_sles_157_uses_documented_zypper_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=sles -VERSION_ID="15.7" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + assert!(adopted.read_only); + assert_eq!(adopted.channel, "release"); + assert_eq!(adopted.family, "gfx120X-all"); + assert_eq!(adopted.version, "7.13.0"); + assert_eq!( + adopted.imported_from.as_deref(), + Some(external_root.canonicalize()?.as_path()) + ); + assert!(!external_root.join(".rocm-cli-runtime.json").exists()); + assert!(runtime_manifest_path(&paths, &adopted.runtime_key).is_file()); - assert!(plan.supported); - assert!(rendered.contains("`zypper` package manager is available")); - assert!(rendered.contains("SUSEConnect")); - assert!(rendered.contains("sle-module-desktop-applications/15.7/x86_64")); - assert!(rendered.contains("sudo zypper install -y kernel-default-devel")); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/sle/15.7/")); - assert!(rendered.contains("sudo zypper --no-gpg-checks install -y")); - assert!(rendered.contains("Execute: sudo zypper install -y amdgpu-dkms")); - assert!(rendered.contains("approval: required")); - } + let mut config = RocmCliConfig::default(); + activate_runtime(&paths, &mut config, &adopted.runtime_key)?; + assert_eq!( + config.active_runtime_key.as_deref(), + Some("adopted-release-pip-gfx120x-all-7-13-0") + ); - #[test] - fn driver_plan_unsupported_linux_is_non_mutating() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=fedora -VERSION_ID="41" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + let rendered = render_runtimes_text(&paths, &config)?; + assert!(rendered.contains("* adopted-release-pip-gfx120x-all-7-13-0")); + assert!(rendered.contains("mode=read-only")); - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("no driver commands will be executed")); - assert!(!rendered.contains("sudo dnf install -y amdgpu-dkms")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn windows_install_driver_is_validate_only() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let plan = build_driver_install_plan(&test_examine("windows", false), "", true); - let rendered = render_driver_install_plan(&plan, false, true); + fn runtime_adopt_request_infers_ids_from_probe() -> Result<()> { + let (root, _) = test_paths("runtime-adopt-infer"); + let external_root = root.join("external-therock-venv"); + let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python_executable = scripts_dir.join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + fs::create_dir_all(&scripts_dir)?; + fs::write(&python_executable, "python")?; - assert!(!plan.supported); - assert!(!plan.mutating); - assert_eq!(plan.policy, "windows_validate_only"); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("execution_commands: ")); - assert!(rendered.contains("post_reboot_checks:")); - assert!(rendered.contains("use `rocm examine`")); - assert!(rendered.contains("rocm examine")); - assert!(plan.commands.is_empty()); - } + let request = infer_adopt_runtime_request( + python_executable.clone(), + Some(external_root.clone()), + None, + None, + None, + false, + &therock::RocmSdkPythonProbe { + rocm_sdk_version: Some("7.13.0a20260423".to_owned()), + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }, + )?; - #[test] - fn wsl_install_driver_uses_rocdxg_guidance_without_dkms() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let plan = build_driver_install_plan(&test_examine("linux", true), "", true); - let rendered = render_driver_install_plan(&plan, false, false); + assert_eq!(request.python_executable, python_executable); + assert_eq!(request.install_root, external_root); + assert_eq!(request.runtime_id, "therock-release:gfx120X-all"); + assert_eq!( + request.runtime_key, + "adopted-release-pip-gfx120x-all-7-13-0a20260423" + ); + assert!(!request.replace); - assert!(!plan.supported); - assert_eq!(plan.policy, "wsl_rocdxg"); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("execution_commands: ")); - assert!(rendered.contains("scripts/wsl_setup_rocdxg.sh")); - assert!(!rendered.contains("amdgpu-dkms")); + let _ = fs::remove_dir_all(root); + Ok(()) } - // EAI-7406: distro selection must honor `/etc/os-release` `ID_LIKE`, so that - // Debian/Ubuntu-family and RHEL-rebuild derivatives that share their base - // version scheme are matched to the correct apt (`ubuntu/`) or EL - // (`el/`) plan instead of falling through to the unsupported plan. - #[test] - fn driver_plan_ubuntu_derivative_via_id_like_matches_ubuntu_plan() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Pop!_OS reports its own ID but reuses Ubuntu's version + repositories. - let os_release = r#" -ID=pop -VERSION_ID="22.04" -VERSION_CODENAME=jammy -ID_LIKE="ubuntu debian" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_adopt_request_honors_nightly_channel() -> Result<()> { + let (root, _) = test_paths("runtime-adopt-infer-nightly"); + let external_root = root.join("external-therock-venv"); + let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python_executable = scripts_dir.join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + fs::create_dir_all(&scripts_dir)?; + fs::write(&python_executable, "python")?; - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - // Ubuntu-family derivatives ship the Ubuntu kernel, so linux-modules-extra applies. - assert!(rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); - assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); - } + let request = infer_adopt_runtime_request( + python_executable, + Some(external_root), + None, + None, + Some("nightly".to_owned()), + true, + &therock::RocmSdkPythonProbe { + rocm_sdk_version: Some("7.14.0a20260531".to_owned()), + default_target_family: Some("gfx1151".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }, + )?; - #[test] - fn driver_plan_debian_derivative_via_id_like_matches_debian_plan() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // A Debian derivative (e.g. LMDE) that shares Debian's version scheme. - let os_release = r#" -ID=lmde -VERSION_ID="12" -ID_LIKE=debian -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + assert_eq!(request.runtime_id, "therock-nightly:gfx1151"); + assert_eq!( + request.runtime_key, + "adopted-nightly-pip-gfx1151-7-14-0a20260531" + ); + assert!(request.replace); - assert!(plan.supported); - // Debian-family maps to the Ubuntu jammy repo and omits linux-modules-extra. - assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); - assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains("amdgpu-dkms")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_almalinux_via_id_like_uses_el_9_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // AlmaLinux is a RHEL rebuild: standard kernel, served from the el/ path. - let os_release = r#" -ID=almalinux -VERSION_ID="9.6" -ID_LIKE="rhel centos fedora" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_uninstall_removes_managed_root_and_clears_active_state() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-managed"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + activate_runtime(&paths, &mut config, &manifest.runtime_key)?; + config.setup.completed = true; + config.setup.therock_venv = Some(manifest.install_root.clone()); + config.save(&paths)?; - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - // EL rebuilds use the vendor-neutral el/ repo path, not rhel/. - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.6/")); - assert!(!rendered.contains("/rhel/9.6/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); - // el9 uses the version-aware standard-kernel prepare commands. - assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); - } + let removed = uninstall_runtime(&paths, &mut config, &manifest.runtime_key)?; - #[test] - fn driver_plan_almalinux_8_via_id_like_uses_el_major_path() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=almalinux -VERSION_ID="8.10" -ID_LIKE="rhel centos fedora" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + assert_eq!(removed.runtime_key, manifest.runtime_key); + assert!(removed.was_active); + assert_eq!( + removed.removed_install_root.as_deref(), + Some(manifest.install_root.as_path()) + ); + assert!(!manifest.install_root.exists()); + assert!(!runtime_manifest_path(&paths, &manifest.runtime_key).exists()); + assert!(!active_runtime_marker_path(&paths).exists()); + assert_eq!(config.active_runtime_key, None); + assert_eq!(config.default_runtime_id, None); + assert_eq!(config.setup.therock_venv, None); + assert!(!config.setup.completed); - assert!(plan.supported); - // EL 8 is served from the major-version path (el/8), matching AMD docs. - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/8/")); - assert!(rendered.contains("-1.el8.noarch.rpm")); - // el8 has no kernel-devel-matched package. - assert!(!rendered.contains("kernel-devel-matched")); - assert!(rendered.contains("kernel-devel-$(uname -r)")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_id_like_with_unsupported_version_stays_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // A Debian-family derivative whose VERSION_ID does not align with any - // AMD-documented Debian version must not fabricate a plan. - let os_release = r#" -ID=lmde -VERSION_ID="6" -ID_LIKE=debian -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_uninstall_unregisters_read_only_runtime_without_deleting_external_root() -> Result<()> + { + let (root, paths) = test_paths("runtime-uninstall-read-only"); + let manifest = write_test_pip_runtime( + &paths, + "external-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let exported_manifest = root.join("external-runtime.json"); + fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; + fs::remove_file(runtime_manifest_path(&paths, &manifest.runtime_key))?; + fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; + import_runtime_manifest(&paths, &exported_manifest, false)?; + let mut config = RocmCliConfig::default(); - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(!rendered.contains("amdgpu-dkms")); - } + let removed = uninstall_runtime(&paths, &mut config, &manifest.runtime_key)?; - #[test] - fn driver_plan_exact_id_takes_precedence_over_id_like() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // An exact RHEL match must keep the rhel/ path even though ID_LIKE=fedora. - let os_release = r#" -ID=rhel -VERSION_ID="9.7" -ID_LIKE=fedora -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + assert_eq!(removed.runtime_key, manifest.runtime_key); + assert!(removed.read_only); + assert_eq!(removed.removed_install_root, None); + assert!(manifest.install_root.exists()); + assert!(!runtime_manifest_path(&paths, &manifest.runtime_key).exists()); - assert!(plan.supported); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); - assert!(!rendered.contains("/el/9.7/")); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn driver_plan_oracle_linux_off_arm_version_stays_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Oracle Linux reports `ID_LIKE=fedora` (not rhel) and boots UEK. An OL - // version outside the exact `ol` arm must NOT be captured by the EL - // fallback, which would emit non-UEK kernel commands that cannot install. - let os_release = r#" -ID=ol -VERSION_ID="9.6" -ID_LIKE=fedora -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + fn runtime_uninstall_removes_rocm_cli_managed_custom_prefix_root() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-prefix"); + let mut manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let prefix_root = root.join("user-chosen-prefix"); + fs::rename(&manifest.install_root, &prefix_root)?; + manifest.install_root = prefix_root.clone(); + let python = prefix_root + .join(if cfg!(windows) { "Scripts" } else { "bin" }) + .join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + manifest.python_executable = Some(python.display().to_string()); + fs::write( + runtime_manifest_path(&paths, &manifest.runtime_key), + serde_json::to_vec_pretty(&manifest)?, + )?; + fs::write( + prefix_root.join(".rocm-cli-runtime.json"), + serde_json::to_vec_pretty(&manifest)?, + )?; + let mut config = RocmCliConfig::default(); - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(!rendered.contains("kernel-devel-matched")); - assert!(!rendered.contains("amdgpu-dkms")); - } + let removed = uninstall_runtime(&paths, &mut config, &manifest.runtime_key)?; - #[test] - fn driver_plan_opensuse_leap_stays_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // openSUSE Leap shares SLES's version scheme but has no SUSEConnect/SCC - // entitlement, so it must not be matched to the SLES plan. - let os_release = r#" -ID=opensuse-leap -VERSION_ID="15.7" -ID_LIKE="suse opensuse" -"#; - let plan = build_driver_install_plan(&test_examine("linux", false), os_release, true); - let rendered = render_driver_install_plan(&plan, false, false); + assert_eq!( + removed.removed_install_root.as_deref(), + Some(prefix_root.as_path()) + ); + assert!(!prefix_root.exists()); + assert!(!runtime_manifest_path(&paths, &manifest.runtime_key).exists()); - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(!rendered.contains("SUSEConnect")); - assert!(!rendered.contains("amdgpu-dkms")); + let _ = fs::remove_dir_all(root); + Ok(()) } + #[cfg(unix)] #[test] - fn resolve_engine_selection_uses_default_runtime_after_engine_prefs() { - let mut config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..RocmCliConfig::default() - }; + fn runtime_adopt_preserves_venv_python_symlink_path() -> Result<()> { + use std::os::unix::fs::symlink; - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("therock-release:gfx120X-all") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_default_runtime_id") - ); + let (root, paths) = test_paths("runtime-adopt-python-symlink"); + let external_root = root.join("external-therock-venv"); + let scripts_dir = external_root.join("bin"); + let system_python = root.join("python3.12"); + let python_executable = scripts_dir.join("python"); + let sdk_root = external_root.join("rocm_sdk"); + let sdk_bin = sdk_root.join("bin"); + fs::create_dir_all(&scripts_dir)?; + fs::create_dir_all(&sdk_bin)?; + let amdhip = sdk_bin.join("libamdhip64.so"); + let hipblas = sdk_bin.join("libhipblas.so"); + fs::write(&system_python, "python")?; + fs::write(&amdhip, "amdhip")?; + fs::write(&hipblas, "hipblas")?; + symlink(&system_python, &python_executable)?; - config.active_runtime_key = Some("release-pip-gfx120x-all-7-13-0".to_owned()); - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("release-pip-gfx120x-all-7-13-0") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_active_runtime_key") - ); + let adopted = adopt_runtime_from_probe( + &paths, + AdoptRuntimeRequest { + python_executable: python_executable.clone(), + install_root: external_root, + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: "adopted-symlink-python".to_owned(), + replace: false, + }, + therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root.clone()), + bin_path: Some(sdk_bin.clone()), + runtime_roots: vec![sdk_root], + bin_paths: vec![sdk_bin.clone()], + library_paths: vec![sdk_bin], + resolved_libraries: vec![ + therock::RocmSdkLibraryProbe { + shortname: "amdhip64".to_owned(), + paths: vec![amdhip], + }, + therock::RocmSdkLibraryProbe { + shortname: "hipblas".to_owned(), + paths: vec![hipblas], + }, + ], + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }, + )?; - config.engine_config_mut("vllm").preferred_runtime_id = - Some("therock-nightly:gfx120X-all".to_owned()); - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("release-pip-gfx120x-all-7-13-0") - ); assert_eq!( - selection.source.as_deref(), - Some("config_active_runtime_key") + adopted.python_executable.as_deref(), + Some(python_executable.as_path().to_string_lossy().as_ref()) ); - config.active_runtime_key = None; - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("therock-nightly:gfx120X-all") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_preferred_runtime_id") - ); + let _ = fs::remove_dir_all(root); + Ok(()) } - #[test] - fn engine_selection_uses_single_ready_runtime_without_active_marker() -> Result<()> { - let (root, paths) = test_paths("single-ready-runtime-selection"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-14-0", - "therock-release:gfx120X-all", - "7.14.0", - 20, - )?; - let selection = validate_engine_selection_runtime( - &paths, - resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), - )?; + #[test] + fn runtime_adopt_rejects_runtime_id_without_family() -> Result<()> { + let (root, paths) = test_paths("runtime-adopt-no-family"); + let external_root = root.join("external-therock-venv"); + let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python_executable = scripts_dir.join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + let sdk_root = external_root.join("rocm_sdk"); + let sdk_bin = sdk_root.join("bin"); + fs::create_dir_all(&scripts_dir)?; + fs::create_dir_all(&sdk_bin)?; + fs::write(&python_executable, "python")?; + + let error = adopt_runtime_from_probe( + &paths, + AdoptRuntimeRequest { + python_executable, + install_root: external_root, + runtime_id: "therock-release".to_owned(), + runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), + replace: false, + }, + therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root), + bin_path: Some(sdk_bin), + ..therock::RocmSdkPythonProbe::default() + }, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("must include a TheRock family suffix")); + assert!(!runtime_registry_dir(&paths).exists()); - assert_eq!( - selection.runtime_id.as_deref(), - Some(manifest.runtime_key.as_str()) - ); - assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn engine_selection_keeps_multiple_ready_runtimes_explicit() -> Result<()> { - let (root, paths) = test_paths("multiple-ready-runtime-selection"); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 10, - )?; - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-14-0", - "therock-release:gfx120X-all", - "7.14.0", - 20, - )?; - let selection = validate_engine_selection_runtime( + fn runtime_adopt_refuses_to_overwrite_without_replace() -> Result<()> { + let (root, paths) = test_paths("runtime-adopt-replace"); + let external_root = root.join("external-therock-venv"); + let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python_executable = scripts_dir.join(if cfg!(windows) { + "python.exe" + } else { + "python" + }); + let sdk_root = external_root.join("rocm_sdk"); + let sdk_bin = sdk_root.join("bin"); + fs::create_dir_all(&scripts_dir)?; + fs::create_dir_all(&sdk_bin)?; + let amdhip = sdk_bin.join(if cfg!(windows) { + "amdhip64_7.dll" + } else { + "libamdhip64.so" + }); + let hipblas = sdk_bin.join(if cfg!(windows) { + "hipblas.dll" + } else { + "libhipblas.so" + }); + fs::write(&python_executable, "python")?; + fs::write(&amdhip, "amdhip")?; + fs::write(&hipblas, "hipblas")?; + + let probe = therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root.clone()), + bin_path: Some(sdk_bin.clone()), + runtime_roots: vec![sdk_root], + bin_paths: vec![sdk_bin.clone()], + library_paths: vec![sdk_bin], + resolved_libraries: vec![ + therock::RocmSdkLibraryProbe { + shortname: "amdhip64".to_owned(), + paths: vec![amdhip], + }, + therock::RocmSdkLibraryProbe { + shortname: "hipblas".to_owned(), + paths: vec![hipblas], + }, + ], + ..therock::RocmSdkPythonProbe::default() + }; + let request = AdoptRuntimeRequest { + python_executable: python_executable.clone(), + install_root: external_root.clone(), + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), + replace: false, + }; + + adopt_runtime_from_probe(&paths, request.clone(), probe.clone())?; + let error = adopt_runtime_from_probe(&paths, request, probe.clone()) + .unwrap_err() + .to_string(); + assert!(error.contains("already exists")); + + adopt_runtime_from_probe( &paths, - resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + AdoptRuntimeRequest { + python_executable, + install_root: external_root, + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), + replace: true, + }, + probe, )?; - assert!(selection.runtime_id.is_none()); - assert!(selection.env_id.is_none()); - assert!(selection.source.is_none()); let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn render_config_text_includes_default_runtime() { - let (_root, paths) = test_paths("config-default-runtime"); - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..RocmCliConfig::default() - }; + fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { + let (root, paths) = test_paths("engine-plugin"); + let plugin_dir = paths.primary_engine_plugin_dir(); + fs::create_dir_all(&plugin_dir)?; + let plugin_path = plugin_dir.join( + rocm_engine_protocol::platform_engine_plugin_binary_name("vllm"), + ); + fs::write(&plugin_path, "plugin")?; - let rendered = render_config_text(&paths, &config); + let discovered = find_engine_plugin_binary("vllm", engine_plugin_dirs(&paths))?; + let _ = fs::remove_dir_all(root); - assert!(rendered.contains("default_runtime_id: therock-release:gfx120X-all")); - assert!(rendered.contains("active_runtime_key: ")); + assert_eq!(discovered, Some(plugin_path)); + Ok(()) } #[test] - fn render_config_text_includes_telemetry_policy() { - let (_root, paths) = test_paths("config-telemetry-policy"); - let mut config = RocmCliConfig::default(); + fn engine_plugin_discovery_prefers_primary_plugin_dir() -> Result<()> { + let (root, paths) = test_paths("engine-plugin-precedence"); + let primary_dir = paths.primary_engine_plugin_dir(); + let compatibility_dir = paths.data_dir.join("engines"); + fs::create_dir_all(&primary_dir)?; + fs::create_dir_all(&compatibility_dir)?; + let name = rocm_engine_protocol::platform_engine_plugin_binary_name("vllm"); + let primary_path = primary_dir.join(&name); + fs::write(&primary_path, "primary")?; + fs::write(compatibility_dir.join(&name), "compatibility")?; - let local = render_config_text(&paths, &config); - assert!(local.contains("telemetry_mode: local")); - assert!(local.contains("telemetry_policy: local amd-smi inspection only")); - assert!(local.contains("no external reporting is implemented")); - assert!(local.contains(" providers:")); - assert!(local.contains(" local: enabled")); - assert!(local.contains(" openai: disabled")); - assert!(local.contains(" anthropic: disabled")); + let discovered = find_engine_plugin_binary("vllm", engine_plugin_dirs(&paths))?; + let _ = fs::remove_dir_all(root); - config.telemetry.mode = TELEMETRY_MODE_OFF.to_owned(); - config.provider_config_mut("openai").enabled = true; - let off = render_config_text(&paths, &config); - assert!(off.contains("telemetry_mode: off")); - assert!(off.contains("telemetry_policy: disabled")); - assert!(off.contains("no local polling")); - assert!(off.contains(" openai: enabled")); + assert_eq!(discovered, Some(primary_path)); + Ok(()) } #[test] - fn engine_install_runtime_selection_requires_configured_runtime() -> Result<()> { - let (root, paths) = test_paths("engine-install-runtime-selection"); - let error = - resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "vllm", None) - .unwrap_err() - .to_string(); - assert!(error.contains("no active ROCm runtime is configured")); - assert_eq!( - resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "lemonade", None)?, - format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), - ); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; + fn a_satisfied_dependency_check_is_stated_plainly() { + let rendered = render_engine_dependency_check("vllm", &EngineDependencyCheck::Satisfied); - let config = RocmCliConfig { - active_runtime_key: Some("release-pip-gfx120x-all".to_owned()), - ..RocmCliConfig::default() - }; - assert_eq!( - resolve_engine_install_runtime_id(&paths, &config, "vllm", None)?, - "release-pip-gfx120x-all" - ); - assert_eq!( - resolve_engine_install_runtime_id( - &paths, - &config, - "vllm", - Some("therock-release:gfx120X-all".to_owned()) - )?, - "release-pip-gfx120x-all" - ); - let _ = fs::remove_dir_all(root); - Ok(()) + assert_eq!(rendered, " dependency_check: satisfied\n"); } #[test] - fn runtime_selector_recovers_setup_runtime_registry_from_local_manifest() -> Result<()> { - let (root, paths) = test_paths("runtime-selector-recover-setup"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-local-manifest", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - let install_root = manifest.install_root.clone(); - let mut config = RocmCliConfig { - default_runtime_id: Some(manifest.runtime_id.clone()), - active_runtime_key: Some(manifest.runtime_key.clone()), - ..RocmCliConfig::default() - }; - config.setup.completed = true; - config.setup.therock_venv = Some(install_root.clone()); - config.save(&paths)?; + fn a_violated_dependency_check_names_the_pin_and_the_remedy() { + // The SDK torch stack replaced the build vLLM pins. The install + // succeeded, so the only signal the user gets is this block. + let rendered = render_engine_dependency_check( + "vllm", + &EngineDependencyCheck::Violated { + violations: vec![ + "The package `vllm` requires `torch==2.10.0+git8514f05`, but `2.9.1+rocm7.14.0a20260611` is installed".to_owned(), + ], + expected: Vec::new(), + }, + ); - let rebased_paths = paths.with_managed_root(install_root, false); - let rebased_registry = runtime_registry_dir(&rebased_paths); - let _ = fs::remove_dir_all(&rebased_registry); + assert!(rendered.contains(" dependency_check: violated\n")); + assert!(rendered.contains("torch==2.10.0+git8514f05")); + assert!(rendered.contains("2.9.1+rocm7.14.0a20260611")); + assert!(rendered.contains(" action: rocm engines install vllm --reinstall\n")); + } - assert!(!runtime_manifest_path(&rebased_paths, &manifest.runtime_key).is_file()); - assert_eq!( - resolve_runtime_selector_to_exact_key( - &rebased_paths, - &manifest.runtime_key, - "test active runtime" - )?, - manifest.runtime_key + #[test] + fn a_divergence_on_the_realigned_package_does_not_advise_undoing_it() { + // The reinstall remedy is correct for a real violation and catastrophic + // here: it reinstates the engine's own build and restores a runtime that + // cannot open a device. + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed".to_owned(), + ], + Some("torch"), + ); + + assert!(matches!( + outcome, + EngineDependencyCheck::ExpectedDivergence(_) + )); + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!(rendered.contains(" dependency_check: expected_divergence\n")); + assert!( + !rendered.contains("action: rocm engines install vllm --reinstall"), + "the reinstall remedy does not apply to a deliberate divergence: {rendered}" ); - assert!(runtime_manifest_path(&rebased_paths, &manifest.runtime_key).is_file()); + } - let rendered = render_runtimes_text(&rebased_paths, &config)?; - assert!(rendered.contains("release-pip-gfx120x-all-local-manifest")); - assert!(rendered.contains("status=ready")); + #[test] + fn a_divergence_on_any_other_package_is_still_a_violation() { + // The unrelated violation is real and must keep saying so, but it does not + // make the deliberate torch divergence one too — and the reinstall that + // would repair numpy is exactly what must not be advised while the + // alignment stands. + let outcome = classify_dependency_details( + vec![ + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed".to_owned(), + "The package `vllm` requires `numpy==1.26.4`, but `2.0.0` is installed".to_owned(), + ], + Some("torch"), + ); - let _ = fs::remove_dir_all(root); - Ok(()) + let EngineDependencyCheck::Violated { + violations, + expected, + } = &outcome + else { + panic!("an unrelated violation is still a violation: {outcome:?}"); + }; + assert_eq!(violations.len(), 1, "only numpy violates: {violations:?}"); + assert!(violations[0].contains("numpy")); + assert_eq!(expected.len(), 1, "torch diverged on purpose: {expected:?}"); + assert!(expected[0].contains("torch")); + + let rendered = render_engine_dependency_check("vllm", &outcome); + assert!(rendered.contains(" dependency_check: violated\n")); + assert!(rendered.contains(" violation: The package `vllm` requires `numpy")); + assert!(rendered.contains(" divergence: The package `vllm` requires `torch")); + assert!( + !rendered.contains("action: rocm engines install vllm --reinstall"), + "the reinstall would undo the alignment while repairing numpy: {rendered}" + ); } #[test] - fn sdk_install_finalization_activates_runtime_and_setup_root() -> Result<()> { - let (root, paths) = test_paths("sdk-install-finalization"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-finalized", - "therock-release:gfx120X-all", - "7.13.0", - 42, - )?; + fn a_line_whose_subject_cannot_be_parsed_stays_a_violation() { + // Conservative by construction: an unrecognised shape is not evidence that + // the divergence was intended, so it must not be quietly excused. + let outcome = classify_dependency_details( + vec!["something uv said that this parser does not recognise".to_owned()], + Some("torch"), + ); - let finalized = finalize_successful_sdk_install(&paths)? - .context("sdk install finalization should select the installed runtime")?; - let rebased_paths = paths.with_managed_root(manifest.install_root.clone(), false); - let config = RocmCliConfig::load(&rebased_paths)?; + assert!(matches!(outcome, EngineDependencyCheck::Violated { .. })); + } - assert_eq!(finalized.runtime_key, manifest.runtime_key); - assert_eq!( - config.default_runtime_id.as_deref(), - Some(manifest.runtime_id.as_str()) + /// The engine's build enumerates no devices against the installed SDK. + /// + /// Observed on MI300X: the engine pins a torch built for a different ROCm + /// version, installs it over the SDK's, and the runtime then reports zero + /// devices. The release is right; only the build is wrong. + #[test] + fn the_engines_build_is_replaced_by_the_sdk_build_of_the_same_release() { + let plan = plan_torch_alignment( + Some("rocm7.13.0"), + Some("2.11.0+gitd0c8b1f"), + Some("torch==2.11.0+gitd0c8b1f"), + "vllm", ); - assert!(config.setup.completed); + assert_eq!( - config.setup.therock_venv.as_deref(), - Some(manifest.install_root.as_path()) + plan, + TorchAlignmentPlan::Install { + wanted: "2.11.0+rocm7.13.0".to_owned(), + from: "2.11.0+gitd0c8b1f".to_owned(), + } ); - assert_eq!( - config.active_runtime_key.as_deref(), - Some(manifest.runtime_key.as_str()) + } + + /// The SDK picked a torch *release* the engine does not accept. + /// + /// The mirror-image failure: here the SDK's own choice is the wrong one, and + /// simply keeping it would break the engine. The rule takes the release the + /// engine pins while still keeping the SDK's build of it — so neither the + /// engine's build nor the SDK's release selection wins outright. + #[test] + fn the_sdks_release_is_corrected_to_the_one_the_engine_pins() { + // The installed torch already carries the SDK's build, so the build is not + // what is wrong here — the release is. Pinning it needs the two sides to + // disagree on the release and agree on the build, which is the opposite of + // the case above; identical arguments to it would only restate that one. + let plan = plan_torch_alignment( + Some("rocm7.14.0a20260611"), + Some("2.11.0+rocm7.14.0a20260611"), + Some("torch==2.10.0+git8514f05"), + "vllm", ); + assert_eq!( - config.default_runtime_id.as_deref(), - Some(manifest.runtime_id.as_str()) + plan, + TorchAlignmentPlan::Install { + wanted: "2.10.0+rocm7.14.0a20260611".to_owned(), + from: "2.11.0+rocm7.14.0a20260611".to_owned(), + }, + "the engine's release must win while the SDK's build is kept" ); - assert!(runtime_manifest_path(&rebased_paths, &manifest.runtime_key).is_file()); - assert!(active_runtime_marker_path(&rebased_paths).is_file()); - - let success = render_sdk_install_success(&finalized); - assert!(success.contains("ROCm SDK installed successfully.")); - assert!(success.contains("next step: run `rocm help`")); - assert!(success.contains(&manifest.install_root.display().to_string())); - assert!(!success.contains("config:")); - assert!(!success.contains("marker:")); + } - let mut examine = String::new(); - append_examine_runtime_state(&mut examine, &rebased_paths, &config)?; - assert!(examine.contains("active_runtime_status: ready")); - assert!(examine.contains("setup_runtime_root:")); + #[test] + fn an_install_is_failed_only_when_it_left_a_runtime_that_cannot_serve() { + let no_devices = RuntimeDeviceCheck::NoDevices { + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + hip_version: "7.2.53211".to_owned(), + }; + let kernel_failed = RuntimeDeviceCheck::KernelFailed { + torch_version: "2.11.0+rocm7.13.0".to_owned(), + error: "AcceleratorError: device kernel image is invalid".to_owned(), + }; + let usable = RuntimeDeviceCheck::Usable { + device_count: 8, + torch_version: "2.11.0+rocm7.13.0".to_owned(), + }; - let _ = fs::remove_dir_all(root); - Ok(()) + // A machine with a GPU that this runtime cannot use, either way round. + assert!(install_left_runtime_unusable( + &no_devices, + &HostGpu::Detected + )); + assert!(install_left_runtime_unusable( + &kernel_failed, + &HostGpu::Detected + )); + // A machine with a GPU and a runtime that can use it. + assert!(!install_left_runtime_unusable(&usable, &HostGpu::Detected)); + // On a host with no GPU both verdicts are the expected answer, not a + // reason to throw away a multi-gigabyte install. + assert!(!install_left_runtime_unusable( + &no_devices, + &HostGpu::Absent + )); + assert!(!install_left_runtime_unusable( + &kernel_failed, + &HostGpu::Absent + )); + // A host we could not examine is not a host without a GPU, but it is not + // evidence of one either, so it never fails the install on its own. + let unknown = HostGpu::NotVerified("lspci is not installed".to_owned()); + assert!(!install_left_runtime_unusable(&no_devices, &unknown)); + assert!(!install_left_runtime_unusable(&kernel_failed, &unknown)); } #[test] - fn env_root_for_runtime_uses_runtime_install_root() -> Result<()> { - let (root, paths) = test_paths("engine-env-root-runtime"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - - let engine_root = env_root_for_runtime(&paths, "vllm", &manifest.runtime_key)?; + fn a_kernel_launch_failure_is_not_reported_as_a_usable_device() { + // The probe enumerated a device and then failed to run anything on it. + // Reading only the count calls that healthy, which is the whole defect: + // the install passes and the first serve dies. + let outcome = classify_runtime_device_probe(therock::RuntimeDeviceProbe { + import_ok: true, + torch_version: Some("2.11.0+rocm7.14.0".to_owned()), + hip_version: Some("7.14.0".to_owned()), + device_count: Some(1), + error: None, + kernel_error: Some("AcceleratorError: device kernel image is invalid".to_owned()), + }); - assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); assert_eq!( - env_root_for_runtime(&paths, "lemonade", &manifest.runtime_key)?, - None + outcome, + RuntimeDeviceCheck::KernelFailed { + torch_version: "2.11.0+rocm7.14.0".to_owned(), + error: "AcceleratorError: device kernel image is invalid".to_owned(), + } ); - let _ = fs::remove_dir_all(root); - Ok(()) } #[test] - fn env_root_for_engine_install_uses_active_runtime_root_for_lemonade() -> Result<()> { - let (root, paths) = test_paths("lemonade-engine-env-root-runtime"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - let config = RocmCliConfig { - active_runtime_key: Some(manifest.runtime_key.clone()), - ..RocmCliConfig::default() - }; - - let engine_root = - env_root_for_engine_install(&paths, &config, "lemonade", "lemonade-embeddable")?; + fn an_enumeration_failure_stays_a_verdict_of_its_own() { + // `error` still carries the failures that happen before any kernel runs. + // Reporting one of those as a kernel failure would name the wrong remedy. + let import_failed = classify_runtime_device_probe(therock::RuntimeDeviceProbe { + import_ok: false, + torch_version: None, + hip_version: None, + device_count: None, + error: Some("ImportError: libamdhip64.so.7".to_owned()), + kernel_error: None, + }); + let count_raised = classify_runtime_device_probe(therock::RuntimeDeviceProbe { + import_ok: true, + torch_version: Some("2.11.0+rocm7.13.0".to_owned()), + hip_version: Some("7.13.0".to_owned()), + device_count: None, + error: Some("RuntimeError: HIP failed to initialize".to_owned()), + kernel_error: None, + }); + let no_devices = classify_runtime_device_probe(therock::RuntimeDeviceProbe { + import_ok: true, + torch_version: Some("2.11.0+gitd0c8b1f".to_owned()), + hip_version: Some("7.13.0".to_owned()), + device_count: Some(0), + error: None, + kernel_error: None, + }); - assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); - let _ = fs::remove_dir_all(root); - Ok(()) + assert_eq!( + import_failed, + RuntimeDeviceCheck::NotVerified("ImportError: libamdhip64.so.7".to_owned()) + ); + assert_eq!( + count_raised, + RuntimeDeviceCheck::NotVerified("RuntimeError: HIP failed to initialize".to_owned()) + ); + assert_eq!( + no_devices, + RuntimeDeviceCheck::NoDevices { + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + hip_version: "7.13.0".to_owned(), + } + ); } #[test] - fn engine_runtime_selection_rejects_ambiguous_default_runtime_id() -> Result<()> { - let (root, paths) = test_paths("engine-runtime-ambiguous-default"); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - write_test_pip_runtime( - &paths, - "vllm-source-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 2, - )?; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..RocmCliConfig::default() - }; + fn a_kernel_failure_is_reported_as_a_failure_and_says_what_broke() { + let rendered = render_runtime_device_check(&RuntimeDeviceCheck::KernelFailed { + torch_version: "2.11.0+rocm7.14.0".to_owned(), + error: "AcceleratorError: device kernel image is invalid".to_owned(), + }); - let error = resolve_engine_install_runtime_id(&paths, &config, "vllm", None) - .unwrap_err() - .to_string(); - assert!(error.contains("matches multiple installed runtimes")); - assert!(error.contains("rocm runtimes activate ")); + assert!(rendered.contains(" device_check: kernel_failed\n")); + assert!(rendered.contains("2.11.0+rocm7.14.0")); + assert!(rendered.contains("device kernel image is invalid")); + // The remedy differs from `no_devices`, so the text must not borrow its + // explanation about being built for a different ROCm version. + assert!(!rendered.contains("Failed to infer device type")); + assert!(rendered.contains("no kernel image for this GPU")); + } - let selection = resolve_engine_selection(&config, "vllm", None, None); - let error = validate_engine_selection_runtime(&paths, selection) - .unwrap_err() - .to_string(); - assert!(error.contains("matches multiple installed runtimes")); + /// The engine's own pin runs: keep it, and claim no divergence from it. + #[test] + fn a_working_engine_torch_is_kept_instead_of_realigned() { + let retention = classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::Usable { + device_count: 1, + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + }, + Some("torch==2.11.0+gitd0c8b1f"), + ); - let selection = - resolve_engine_selection(&config, "vllm", Some("release-pip-gfx120x-all"), None); - let selection = validate_engine_selection_runtime(&paths, selection)?; assert_eq!( - selection.runtime_id.as_deref(), - Some("release-pip-gfx120x-all") + retention, + TorchRetention::EngineBuild { + version: "2.11.0+gitd0c8b1f".to_owned(), + } ); - let _ = fs::remove_dir_all(root); - Ok(()) + assert_eq!(retained_diverged_package(&retention), None); + let rendered = render_torch_retention(&retention, "vllm"); + assert!( + rendered.contains(" torch_alignment: retained_engine_build (2.11.0+gitd0c8b1f)\n") + ); + assert!(rendered.contains("ran a GPU kernel with this SDK")); } + /// The SDK's build of the pinned release runs: keep it, and expect the pin to + /// stay unsatisfied so the dependency check does not report it as a violation. #[test] - fn render_runtimes_text_reports_missing_configured_active_runtime() -> Result<()> { - let (root, paths) = test_paths("runtime-active-missing"); - let config = RocmCliConfig { - active_runtime_key: Some("missing-runtime-key".to_owned()), - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..RocmCliConfig::default() - }; - - let rendered = render_runtimes_text(&paths, &config)?; - - assert!(rendered.contains("active_runtime_key: missing-runtime-key")); - assert!(rendered.contains( - "active_status: missing manifest for active_runtime_key=missing-runtime-key" - )); + fn a_working_sdk_torch_is_kept_as_an_intended_divergence() { + let retention = classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::Usable { + device_count: 8, + torch_version: "2.11.0+rocm7.13.0".to_owned(), + }, + Some("torch==2.11.0+gitd0c8b1f"), + ); - let _ = fs::remove_dir_all(root); - Ok(()) + assert_eq!( + retention, + TorchRetention::SdkBuild { + version: "2.11.0+rocm7.13.0".to_owned(), + } + ); + assert_eq!(retained_diverged_package(&retention), Some("torch")); + let rendered = render_torch_retention(&retention, "vllm"); + assert!(rendered.contains(" torch_alignment: retained_sdk_build (2.11.0+rocm7.13.0)\n")); + assert!(rendered.contains("the SDK's build of the release vllm pins ran a GPU kernel")); } + /// Both intended builds are fixed points, so a repeated install settles rather + /// than swapping torch back and forth between the two package sources. #[test] - fn runtime_lists_display_build_date_from_version_string() -> Result<()> { - let (root, paths) = test_paths("runtime-build-date-display"); - let runtime_key = "release-pip-gfx120x-all-7-14-0a20260601"; - let manifest = write_test_pip_runtime( - &paths, - runtime_key, - "therock-release:gfx120X-all", - "7.14.0a20260601", - 20, - )?; - let config = RocmCliConfig { - active_runtime_key: Some(manifest.runtime_key.clone()), - default_runtime_id: Some(manifest.runtime_id), - ..RocmCliConfig::default() - }; + fn repairing_a_runtime_converges_on_one_of_the_two_intended_builds() { + let after_engine_install = classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::Usable { + device_count: 1, + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + }, + Some("torch==2.11.0+gitd0c8b1f"), + ); + // What the realignment installs, seen on the next run over the same runtime. + let after_realignment = classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::Usable { + device_count: 1, + torch_version: "2.11.0+rocm7.13.0".to_owned(), + }, + Some("torch==2.11.0+gitd0c8b1f"), + ); - let runtimes = render_runtimes_text(&paths, &config)?; - assert!(runtimes.contains("version=7.14.0a20260601 (build 2026-06-01)")); + assert!(!matches!(after_engine_install, TorchRetention::Realign)); + assert!(!matches!(after_realignment, TorchRetention::Realign)); + } - let mut examine = String::new(); - append_examine_runtime_state(&mut examine, &paths, &config)?; - assert!(examine.contains("active_runtime_version: 7.14.0a20260601 (build 2026-06-01)")); + /// A pin that already names the SDK's build satisfies itself. Calling that an + /// intended divergence would suppress a violation that has not happened. + #[test] + fn a_pin_that_already_names_the_sdk_build_diverges_from_nothing() { + let retention = classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::Usable { + device_count: 1, + torch_version: "2.11.0+rocm7.13.0".to_owned(), + }, + Some("torch==2.11.0+rocm7.13.0"), + ); - let _ = fs::remove_dir_all(root); - Ok(()) + assert_eq!( + retention, + TorchRetention::EngineBuild { + version: "2.11.0+rocm7.13.0".to_owned(), + } + ); + assert_eq!(retained_diverged_package(&retention), None); } + /// Everything that has not been shown to work goes to the repair path — which + /// is what this tool did unconditionally before it could test a kernel. #[test] - fn runtime_activation_records_exact_key_and_rollback() -> Result<()> { - let (root, paths) = test_paths("runtime-activation"); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-12-0", - "therock-release:gfx120X-all", - "7.12.0", - 10, - )?; - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; + fn a_torch_that_has_not_been_shown_to_work_is_realigned() { + let usable = |version: &str| RuntimeDeviceCheck::Usable { + device_count: 1, + torch_version: version.to_owned(), + }; + let pin = Some("torch==2.11.0+gitd0c8b1f"); - let mut config = RocmCliConfig::default(); - let first = activate_runtime(&paths, &mut config, "release-pip-gfx120x-all-7-12-0")?; - assert_eq!(first.previous_runtime_key, None); + // A third build nobody here installed on purpose. assert_eq!( - config.active_runtime_key.as_deref(), - Some("release-pip-gfx120x-all-7-12-0") + classify_retained_torch(Some("rocm7.13.0"), &usable("2.9.0+cpu"), pin), + TorchRetention::Realign ); - - let second = activate_runtime(&paths, &mut config, "release-pip-gfx120x-all-7-13-0")?; + // No devices at all, whatever build it is. assert_eq!( - second.previous_runtime_key.as_deref(), - Some("release-pip-gfx120x-all-7-12-0") + classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::NoDevices { + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + hip_version: "7.13.0".to_owned(), + }, + pin + ), + TorchRetention::Realign ); + // A device that cannot run a kernel is never retained, not even when it + // holds exactly the build the engine pins. assert_eq!( - config.default_runtime_id.as_deref(), - Some("therock-release:gfx120X-all") + classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::KernelFailed { + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + error: "device kernel image is invalid".to_owned(), + }, + pin + ), + TorchRetention::Realign ); + // No answer is not an answer. assert_eq!( - config.active_runtime_key.as_deref(), - Some("release-pip-gfx120x-all-7-13-0") + classify_retained_torch( + Some("rocm7.13.0"), + &RuntimeDeviceCheck::NotVerified("torch did not import".to_owned()), + pin + ), + TorchRetention::Realign ); + // Nothing pins torch, so there is no release to hold either build of. assert_eq!( - config.previous_runtime_key.as_deref(), - Some("release-pip-gfx120x-all-7-12-0") + classify_retained_torch(Some("rocm7.13.0"), &usable("2.11.0+gitd0c8b1f"), None), + TorchRetention::Realign ); - - let marker: ActiveRuntimeMarker = - serde_json::from_slice(&fs::read(active_runtime_marker_path(&paths))?)?; - assert_eq!(marker.runtime_key, "release-pip-gfx120x-all-7-13-0"); + // A range is not an exact pin, so it cannot identify the engine's build. assert_eq!( - marker.previous_runtime_key.as_deref(), - Some("release-pip-gfx120x-all-7-12-0") + classify_retained_torch( + Some("rocm7.13.0"), + &usable("2.11.0+gitd0c8b1f"), + Some("torch>=2.10") + ), + TorchRetention::Realign ); - - let rendered = render_runtimes_text(&paths, &config)?; - assert!(rendered.contains("* release-pip-gfx120x-all-7-13-0")); - assert!(rendered.contains("- release-pip-gfx120x-all-7-12-0")); - assert!(rendered.contains("status=ready")); - - let rolled_back = rollback_runtime(&paths, &mut config)?; - assert_eq!(rolled_back.runtime_key, "release-pip-gfx120x-all-7-12-0"); + // The manifest does not say what the SDK's build is, so a torch that is + // not the engine's pin cannot be recognised as the other intended one. assert_eq!( - config.previous_runtime_key.as_deref(), - Some("release-pip-gfx120x-all-7-13-0") + classify_retained_torch(None, &usable("2.11.0+rocm7.13.0"), pin), + TorchRetention::Realign ); - - let _ = fs::remove_dir_all(root); - Ok(()) + // Nothing is retained, so nothing is reported in place of the repair. + assert!(render_torch_retention(&TorchRetention::Realign, "vllm").is_empty()); + assert_eq!(retained_diverged_package(&TorchRetention::Realign), None); } - #[test] - fn runtime_activation_rejects_ambiguous_runtime_id() -> Result<()> { - let (root, paths) = test_paths("runtime-ambiguous"); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-12-0", - "therock-release:gfx120X-all", - "7.12.0", - 10, - )?; - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - let mut config = RocmCliConfig::default(); - - let error = activate_runtime(&paths, &mut config, "therock-release:gfx120X-all") - .unwrap_err() - .to_string(); - - assert!(error.contains("matches multiple installed runtimes")); - assert!(error.contains("release-pip-gfx120x-all-7-12-0")); - assert!(error.contains("release-pip-gfx120x-all-7-13-0")); + fn sdk_install_finalization() -> SdkInstallFinalization { + SdkInstallFinalization { + runtime_key: "wheel-gfx942-7.13.0".to_owned(), + install_root: PathBuf::from("/tmp/does-not-need-to-exist"), + family: "gfx94X-dcgpu".to_owned(), + } + } - let _ = fs::remove_dir_all(root); - Ok(()) + /// Run the real `install sdk` completion path against a stubbed auto-install. + fn finish_sdk_install_with( + paths: &AppPaths, + outcome: Result<()>, + ) -> (Result<()>, SdkInstallFinalization) { + let finalized = sdk_install_finalization(); + let result = finish_sdk_install( + paths, + Some(&finalized), + "install_sdk", + "sdk install completed".to_owned(), + |_, _| outcome, + ); + (result, finalized) } #[test] - fn runtime_activation_rejects_unusable_manifest() -> Result<()> { - let (root, paths) = test_paths("runtime-unusable"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - fs::remove_file(manifest.install_root.join("Scripts").join("python.exe")).ok(); - fs::remove_file(manifest.install_root.join("bin").join("python")).ok(); - let mut config = RocmCliConfig::default(); - - let error = activate_runtime(&paths, &mut config, "release-pip-gfx120x-all-7-13-0") - .unwrap_err() - .to_string(); + fn install_sdk_fails_when_the_install_left_a_runtime_that_cannot_open_a_device() -> Result<()> { + let (_root, paths) = test_paths("install-sdk-unusable"); - assert!(error.contains("runtime Python executable is missing")); + // Exercises the command's own completion path, not just the predicate: if the + // catch goes back to warning and falling through, this fails. + let (result, _) = finish_sdk_install_with( + &paths, + Err(unusable_runtime_error("vllm", "wheel-gfx942-7.13.0")), + ); + let error = result.expect_err("an unusable runtime must fail `install sdk`"); + // And it must still say the SDK survived, so a transient index failure does + // not read as a ruined install that has to be started from scratch. + assert!( + error + .to_string() + .contains("The ROCm SDK itself is installed"), + "the failure must say the SDK survived:\n{error}" + ); - let _ = fs::remove_dir_all(root); + // The SDK install did complete, so its record is written before the command + // fails — otherwise the audit trail would show an install that never happened. + let actions: Vec = load_recent_audit_events(&paths, 10)? + .into_iter() + .map(|event| event.action) + .collect(); + assert!( + actions.iter().any(|action| action == "install_sdk"), + "the successful SDK install was not recorded before failing: {actions:?}" + ); Ok(()) } #[test] - fn runtime_import_records_read_only_manifest_without_mutating_runtime_root() -> Result<()> { - let (root, paths) = test_paths("runtime-import"); - let manifest = write_test_pip_runtime( - &paths, - "external-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - let exported_manifest = root.join("external-runtime.json"); - fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; - fs::remove_file(runtime_manifest_path(&paths, &manifest.runtime_key))?; - fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; - - let imported = import_runtime_manifest(&paths, &exported_manifest, false)?; + fn install_sdk_survives_an_engine_install_failure() -> Result<()> { + let (_root, paths) = test_paths("install-sdk-engine-failure"); - assert!(imported.read_only); - let canonical_export = exported_manifest.canonicalize()?; - assert_eq!( - imported.imported_from.as_deref(), - Some(canonical_export.as_path()) + // The complement, and the reason the catch exists at all: the SDK is installed + // and usable, only a separately retryable step is missing. Failing here would + // throw away a multi-gigabyte install over an unreachable engine index. + let (result, _) = finish_sdk_install_with( + &paths, + Err(anyhow::anyhow!( + "failed to reach the engine index: dns error" + )), ); assert!( - !manifest - .install_root - .join(".rocm-cli-runtime.json") - .exists() + result.is_ok(), + "an engine install failure must not discard a good SDK install: {:?}", + result.err() ); - let imported_registry: therock::InstalledRuntimeManifest = serde_json::from_slice( - &fs::read(runtime_manifest_path(&paths, &manifest.runtime_key))?, - )?; - assert!(imported_registry.read_only); - - let mut config = RocmCliConfig::default(); - activate_runtime(&paths, &mut config, &manifest.runtime_key)?; - assert_eq!( - config.active_runtime_key.as_deref(), - Some("external-pip-gfx120x-all-7-13-0") + let actions: Vec = load_recent_audit_events(&paths, 10)? + .into_iter() + .map(|event| event.action) + .collect(); + assert!(actions.iter().any(|action| action == "install_sdk")); + assert!( + actions.iter().any(|action| action == "engine_auto_install"), + "the engine failure must still be recorded: {actions:?}" ); - - let rendered = render_runtimes_text(&paths, &config)?; - assert!(rendered.contains("mode=read-only")); - - let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn runtime_import_refuses_to_overwrite_without_replace() -> Result<()> { - let (root, paths) = test_paths("runtime-import-replace"); - let manifest = write_test_pip_runtime( - &paths, - "external-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - let exported_manifest = root.join("external-runtime.json"); - fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; + fn the_unusable_runtime_failure_survives_being_wrapped_in_context() { + // The error reaches the catch through several `?` hops. Flattening it into a + // plain message is the realistic way a later change silently restores the + // exit-0 bug, since every failure would then look recoverable. + let propagated: Result<()> = Err(unusable_runtime_error("vllm", "wheel-gfx942-7.13.0")) + .context("automatic vLLM install failed"); + let propagated = propagated.expect_err("expected the unusable-runtime error"); + assert!(engine_auto_install_failure_is_fatal(&propagated)); + assert!(!engine_auto_install_failure_is_fatal(&anyhow::anyhow!( + "uv exited with status 1" + ))); + } - let error = import_runtime_manifest(&paths, &exported_manifest, false) - .unwrap_err() - .to_string(); + #[test] + fn an_install_failure_is_not_reported_as_a_missing_wheel() { + // Saying "the index publishes no such build" over a network or disk error + // sends the reader hunting for a wheel that exists. Anything the resolver + // did not actually attribute to a missing version degrades to the honest + // answer: the failure, verbatim. + assert!(!install_error_reports_version_unavailable( + "failed to launch uv: Permission denied" + )); + assert!(!install_error_reports_version_unavailable( + "error sending request: dns error: failed to lookup address" + )); + assert!(install_error_reports_version_unavailable( + "No solution found when resolving: torch==2.10.0+rocm7.14.0a20260611" + )); + } - assert!(error.contains("already exists")); - import_runtime_manifest(&paths, &exported_manifest, true)?; + #[test] + fn an_unresolvable_version_still_reads_as_unavailable() { + let rendered = render_torch_alignment( + &TorchAlignment::Unavailable { + wanted: "2.10.0+rocm7.14.0a20260611".to_owned(), + kept: "2.10.0+git8514f05".to_owned(), + }, + "vllm", + None, + ); - let _ = fs::remove_dir_all(root); - Ok(()) + assert!(rendered.contains(" torch_alignment: unavailable\n")); + assert!(rendered.contains("publishes no 2.10.0+rocm7.14.0a20260611")); } #[test] - fn runtime_adopt_records_read_only_manifest_from_probe() -> Result<()> { - let (root, paths) = test_paths("runtime-adopt"); - let external_root = root.join("external-therock-venv"); - let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); - let python_executable = scripts_dir.join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); - let sdk_root = external_root - .join("Lib") - .join("site-packages") - .join("rocm_sdk"); - let sdk_bin = sdk_root.join("bin"); - fs::create_dir_all(&scripts_dir)?; - fs::create_dir_all(&sdk_bin)?; - let amdhip = sdk_bin.join(if cfg!(windows) { - "amdhip64_7.dll" - } else { - "libamdhip64.so" - }); - let hipblas = sdk_bin.join(if cfg!(windows) { - "hipblas.dll" - } else { - "libhipblas.so" - }); - fs::write(&python_executable, "python")?; - fs::write(&amdhip, "amdhip")?; - fs::write(&hipblas, "hipblas")?; - - let adopted = adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { - python_executable, - install_root: external_root.clone(), - runtime_id: "therock-release:gfx120X-all".to_owned(), - runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), - replace: false, + fn a_failed_realignment_reports_the_error_it_actually_hit() { + let rendered = render_torch_alignment( + &TorchAlignment::InstallFailed { + wanted: "2.11.0+rocm7.13.0".to_owned(), + kept: "2.11.0+gitd0c8b1f".to_owned(), + error: "failed to launch uv: Permission denied".to_owned(), }, - therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root.clone()), - bin_path: Some(sdk_bin.clone()), - runtime_roots: vec![sdk_root], - bin_paths: vec![sdk_bin.clone()], - library_paths: vec![sdk_bin], - resolved_libraries: vec![ - therock::RocmSdkLibraryProbe { - shortname: "amdhip64".to_owned(), - paths: vec![amdhip], - }, - therock::RocmSdkLibraryProbe { - shortname: "hipblas".to_owned(), - paths: vec![hipblas], - }, - ], - resolved_target_family: Some("gfx120X-all".to_owned()), - ..therock::RocmSdkPythonProbe::default() + "vllm", + None, + ); + + assert!(rendered.contains(" torch_alignment: install_failed\n")); + assert!(rendered.contains("Permission denied")); + assert!( + !rendered.contains("publishes no"), + "an install failure must not be described as a missing wheel: {rendered}" + ); + } + + #[test] + fn a_realigned_runtime_names_both_builds_and_the_engine_that_decided_the_release() { + // The line a successful run actually prints, and the one a user reads when + // deciding whether the divergence reported just below it is expected. + let rendered = render_torch_alignment( + &TorchAlignment::Realigned { + from: "2.11.0+gitd0c8b1f".to_owned(), + to: "2.11.0+rocm7.13.0".to_owned(), }, - )?; + "vllm", + None, + ); - assert!(adopted.read_only); - assert_eq!(adopted.channel, "release"); - assert_eq!(adopted.family, "gfx120X-all"); - assert_eq!(adopted.version, "7.13.0"); - assert_eq!( - adopted.imported_from.as_deref(), - Some(external_root.canonicalize()?.as_path()) + assert!(rendered.contains(" torch_alignment: realigned\n")); + assert!(rendered.contains("2.11.0+gitd0c8b1f -> 2.11.0+rocm7.13.0")); + assert!( + rendered.contains("the release vllm pins"), + "the engine that pinned the release is named rather than left anonymous: {rendered}" ); - assert!(!external_root.join(".rocm-cli-runtime.json").exists()); - assert!(runtime_manifest_path(&paths, &adopted.runtime_key).is_file()); + } - let mut config = RocmCliConfig::default(); - activate_runtime(&paths, &mut config, &adopted.runtime_key)?; - assert_eq!( - config.active_runtime_key.as_deref(), - Some("adopted-release-pip-gfx120x-all-7-13-0") + #[test] + fn a_realignment_reports_what_the_runtime_could_do_before_it() { + // The whole point of the line: the device_check printed afterwards is the + // after state, so without this one a realignment that ends badly does not + // say whether it broke a working runtime or repaired a broken one. Here it + // repaired one, and the output says so without anyone visiting the machine. + let rendered = render_torch_alignment( + &TorchAlignment::Realigned { + from: "2.11.0+rocm7.14.0".to_owned(), + to: "2.11.0+rocm7.13.0".to_owned(), + }, + "vllm", + Some(&RuntimeDeviceCheck::KernelFailed { + torch_version: "2.11.0+rocm7.14.0".to_owned(), + error: "HIP error: hipErrorInvalidImage".to_owned(), + }), ); - let rendered = render_runtimes_text(&paths, &config)?; - assert!(rendered.contains("* adopted-release-pip-gfx120x-all-7-13-0")); - assert!(rendered.contains("mode=read-only")); - - let _ = fs::remove_dir_all(root); - Ok(()) + assert!( + rendered.contains("before this replacement: kernel_failed (torch 2.11.0+rocm7.14.0)"), + "the before verdict names itself and the torch it judged: {rendered}" + ); + assert!( + rendered.contains("device_check below is after it"), + "the reader is told which of the two blocks is which: {rendered}" + ); } #[test] - fn runtime_adopt_request_infers_ids_from_probe() -> Result<()> { - let (root, _) = test_paths("runtime-adopt-infer"); - let external_root = root.join("external-therock-venv"); - let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); - let python_executable = scripts_dir.join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); - fs::create_dir_all(&scripts_dir)?; - fs::write(&python_executable, "python")?; + fn only_a_realignment_reports_a_before_verdict() { + // Every other outcome left the runtime as it was, so the device_check + // below is already about the same torch this block names. A before/after + // pair there would invite reading a change into an outcome that made none. + let before = RuntimeDeviceCheck::Usable { + device_count: 1, + torch_version: "2.11.0+rocm7.13.0".to_owned(), + }; - let request = infer_adopt_runtime_request( - python_executable.clone(), - Some(external_root.clone()), - None, - None, - None, - false, - &therock::RocmSdkPythonProbe { - rocm_sdk_version: Some("7.13.0a20260423".to_owned()), - resolved_target_family: Some("gfx120X-all".to_owned()), - ..therock::RocmSdkPythonProbe::default() + for outcome in [ + TorchAlignment::AlreadyAligned { + version: "2.11.0+rocm7.13.0".to_owned(), }, - )?; + TorchAlignment::Disabled { + wanted: "2.11.0+rocm7.13.0".to_owned(), + kept: "2.9.0+cpu".to_owned(), + }, + TorchAlignment::Unavailable { + wanted: "2.11.0+rocm7.13.0".to_owned(), + kept: "2.11.0+gitd0c8b1f".to_owned(), + }, + ] { + let rendered = render_torch_alignment(&outcome, "vllm", Some(&before)); + assert!( + !rendered.contains("before this replacement"), + "{outcome:?} replaced nothing: {rendered}" + ); + } + } - assert_eq!(request.python_executable, python_executable); - assert_eq!(request.install_root, external_root); - assert_eq!(request.runtime_id, "therock-release:gfx120X-all"); - assert_eq!( - request.runtime_key, - "adopted-release-pip-gfx120x-all-7-13-0a20260423" + #[test] + fn an_already_aligned_runtime_reports_the_version_it_kept() { + // Every refresh after the first lands here, so this is the most frequently + // printed of the five outcomes. + let rendered = render_torch_alignment( + &TorchAlignment::AlreadyAligned { + version: "2.11.0+rocm7.13.0".to_owned(), + }, + "vllm", + None, ); - assert!(!request.replace); - let _ = fs::remove_dir_all(root); - Ok(()) + assert!(rendered.contains(" torch_alignment: already_aligned (2.11.0+rocm7.13.0)\n")); } #[test] - fn runtime_adopt_request_honors_nightly_channel() -> Result<()> { - let (root, _) = test_paths("runtime-adopt-infer-nightly"); - let external_root = root.join("external-therock-venv"); - let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); - let python_executable = scripts_dir.join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); - fs::create_dir_all(&scripts_dir)?; - fs::write(&python_executable, "python")?; - - let request = infer_adopt_runtime_request( - python_executable, - Some(external_root), - None, - None, - Some("nightly".to_owned()), - true, - &therock::RocmSdkPythonProbe { - rocm_sdk_version: Some("7.14.0a20260531".to_owned()), - default_target_family: Some("gfx1151".to_owned()), - ..therock::RocmSdkPythonProbe::default() - }, - )?; - - assert_eq!(request.runtime_id, "therock-nightly:gfx1151"); - assert_eq!( - request.runtime_key, - "adopted-nightly-pip-gfx1151-7-14-0a20260531" + fn a_managed_python_runtime_is_settled() { + assert!(settles_runtime_torch("vllm", Some(true))); + assert!( + settles_runtime_torch("vllm", None), + "an engine that does not report the field is still assumed managed, as before" ); - assert!(request.replace); + } - let _ = fs::remove_dir_all(root); - Ok(()) + #[test] + fn an_external_runtime_is_left_alone() { + assert!(!settles_runtime_torch("vllm", Some(false))); } #[test] - fn runtime_uninstall_removes_managed_root_and_clears_active_state() -> Result<()> { - let (root, paths) = test_paths("runtime-uninstall-managed"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - let mut config = RocmCliConfig::default(); - activate_runtime(&paths, &mut config, &manifest.runtime_key)?; - config.setup.completed = true; - config.setup.therock_venv = Some(manifest.install_root.clone()); - config.save(&paths)?; + fn an_engine_that_manages_its_own_runtime_is_left_alone() { + // Lemonade reports `managed_env: Some(true)` — rocm-cli did create the + // runtime — but its `python_executable` is the Lemonade binary, not an + // interpreter, and no torch ever lived there. Settling it would spawn that + // binary twice with a generated `.py` path as `argv[1]` and print torch and + // device check blocks for a runtime that has neither, on the serve path. + assert!(!settles_runtime_torch("lemonade", Some(true))); + } - let removed = uninstall_runtime(&paths, &mut config, &manifest.runtime_key)?; + #[test] + fn an_already_correct_runtime_is_left_alone() { + let plan = plan_torch_alignment( + Some("rocm7.13.0"), + Some("2.11.0+rocm7.13.0"), + Some("torch==2.11.0+gitd0c8b1f"), + "vllm", + ); - assert_eq!(removed.runtime_key, manifest.runtime_key); - assert!(removed.was_active); assert_eq!( - removed.removed_install_root.as_deref(), - Some(manifest.install_root.as_path()) + plan, + TorchAlignmentPlan::AlreadyAligned { + version: "2.11.0+rocm7.13.0".to_owned() + } ); - assert!(!manifest.install_root.exists()); - assert!(!runtime_manifest_path(&paths, &manifest.runtime_key).exists()); - assert!(!active_runtime_marker_path(&paths).exists()); - assert_eq!(config.active_runtime_key, None); - assert_eq!(config.default_runtime_id, None); - assert_eq!(config.setup.therock_venv, None); - assert!(!config.setup.completed); - - let _ = fs::remove_dir_all(root); - Ok(()) } #[test] - fn runtime_uninstall_unregisters_read_only_runtime_without_deleting_external_root() -> Result<()> - { - let (root, paths) = test_paths("runtime-uninstall-read-only"); - let manifest = write_test_pip_runtime( - &paths, - "external-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - let exported_manifest = root.join("external-runtime.json"); - fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; - fs::remove_file(runtime_manifest_path(&paths, &manifest.runtime_key))?; - fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; - import_runtime_manifest(&paths, &exported_manifest, false)?; - let mut config = RocmCliConfig::default(); + fn a_loose_torch_requirement_is_not_second_guessed() { + // Without an exact pin there is no release to carry over, so the engine's + // resolution stands rather than being overridden on a guess. + let plan = plan_torch_alignment( + Some("rocm7.13.0"), + Some("2.11.0+gitd0c8b1f"), + Some("torch>=2.10"), + "vllm", + ); - let removed = uninstall_runtime(&paths, &mut config, &manifest.runtime_key)?; + assert!( + matches!(plan, TorchAlignmentPlan::NotApplicable(reason) if reason.contains("exact")), + "a non-pinned requirement must be left alone" + ); + } - assert_eq!(removed.runtime_key, manifest.runtime_key); - assert!(removed.read_only); - assert_eq!(removed.removed_install_root, None); - assert!(manifest.install_root.exists()); - assert!(!runtime_manifest_path(&paths, &manifest.runtime_key).exists()); + #[test] + fn an_unidentified_sdk_build_is_not_acted_on() { + // Nothing to carry over, so guessing a build would be worse than doing + // nothing and letting the device check report the result. + let plan = plan_torch_alignment( + None, + Some("2.11.0+gitd0c8b1f"), + Some("torch==2.11.0+gitd0c8b1f"), + "vllm", + ); - let _ = fs::remove_dir_all(root); - Ok(()) + assert!(matches!(plan, TorchAlignmentPlan::NotApplicable(_))); } + /// The build comes from the manifest, not from whatever torch is installed. + /// + /// This is what makes a runtime already overwritten by the engine recoverable: + /// reading the environment would call the engine's build the SDK's and leave the + /// runtime broken for good. `plan_torch_alignment` takes the build as an opaque + /// argument, so the property lives here, in the lookup that produces it. #[test] - fn runtime_uninstall_removes_rocm_cli_managed_custom_prefix_root() -> Result<()> { - let (root, paths) = test_paths("runtime-uninstall-prefix"); - let mut manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 20, - )?; - let prefix_root = root.join("user-chosen-prefix"); - fs::rename(&manifest.install_root, &prefix_root)?; - manifest.install_root = prefix_root.clone(); - let python = prefix_root - .join(if cfg!(windows) { "Scripts" } else { "bin" }) - .join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); - manifest.python_executable = Some(python.display().to_string()); - fs::write( - runtime_manifest_path(&paths, &manifest.runtime_key), - serde_json::to_vec_pretty(&manifest)?, - )?; - fs::write( - prefix_root.join(".rocm-cli-runtime.json"), - serde_json::to_vec_pretty(&manifest)?, - )?; - let mut config = RocmCliConfig::default(); - - let removed = uninstall_runtime(&paths, &mut config, &manifest.runtime_key)?; + fn the_sdk_build_is_read_from_the_manifest_not_the_environment() { + let mut manifest = + test_runtime_manifest_for_update("wheel-gfx94x", "gfx94x", "gfx94x-dcgpu", "7.13.0"); + manifest.sdk_torch = Some("2.11.0+rocm7.13.0".to_owned()); assert_eq!( - removed.removed_install_root.as_deref(), - Some(prefix_root.as_path()) + sdk_torch_build_from_manifest(&manifest).as_deref(), + Some("rocm7.13.0"), + "the recorded SDK torch names the build" ); - assert!(!prefix_root.exists()); - assert!(!runtime_manifest_path(&paths, &manifest.runtime_key).exists()); - - let _ = fs::remove_dir_all(root); - Ok(()) } - #[cfg(unix)] + /// Manifests written before `sdk_torch` existed still have to yield a build. + /// + /// These are the runtimes already on real machines, so this fallback is the + /// repair path rather than a nicety. TheRock names the build `rocm`. #[test] - fn runtime_adopt_preserves_venv_python_symlink_path() -> Result<()> { - use std::os::unix::fs::symlink; + fn a_manifest_without_a_recorded_torch_derives_the_build_from_the_sdk_version() { + let manifest = + test_runtime_manifest_for_update("wheel-gfx94x", "gfx94x", "gfx94x-dcgpu", "7.13.0"); + assert!( + manifest.sdk_torch.is_none(), + "this test is about the pre-change manifest shape" + ); - let (root, paths) = test_paths("runtime-adopt-python-symlink"); - let external_root = root.join("external-therock-venv"); - let scripts_dir = external_root.join("bin"); - let system_python = root.join("python3.12"); - let python_executable = scripts_dir.join("python"); - let sdk_root = external_root.join("rocm_sdk"); - let sdk_bin = sdk_root.join("bin"); - fs::create_dir_all(&scripts_dir)?; - fs::create_dir_all(&sdk_bin)?; - let amdhip = sdk_bin.join("libamdhip64.so"); - let hipblas = sdk_bin.join("libhipblas.so"); - fs::write(&system_python, "python")?; - fs::write(&amdhip, "amdhip")?; - fs::write(&hipblas, "hipblas")?; - symlink(&system_python, &python_executable)?; + assert_eq!( + sdk_torch_build_from_manifest(&manifest).as_deref(), + Some("rocm7.13.0") + ); + } - let adopted = adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { - python_executable: python_executable.clone(), - install_root: external_root, - runtime_id: "therock-release:gfx120X-all".to_owned(), - runtime_key: "adopted-symlink-python".to_owned(), - replace: false, - }, - therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root.clone()), - bin_path: Some(sdk_bin.clone()), - runtime_roots: vec![sdk_root], - bin_paths: vec![sdk_bin.clone()], - library_paths: vec![sdk_bin], - resolved_libraries: vec![ - therock::RocmSdkLibraryProbe { - shortname: "amdhip64".to_owned(), - paths: vec![amdhip], - }, - therock::RocmSdkLibraryProbe { - shortname: "hipblas".to_owned(), - paths: vec![hipblas], - }, - ], - resolved_target_family: Some("gfx120X-all".to_owned()), - ..therock::RocmSdkPythonProbe::default() - }, - )?; + /// A dated alpha carries its date into the build, and must not be trimmed. + /// + /// Nightly SDKs version as `7.14.0a20260812`, and TheRock's torch for one is + /// `+rocm7.14.0a20260812` — the date is part of the build, not decoration on + /// the version. Shortening it to the `7.14.0` release would name a build that + /// exists for a different SDK, so the alignment would install a torch built + /// against libraries the runtime does not have. The fallback is a whole-string + /// interpolation today; this pins that, because the tempting "clean up the + /// version first" refactor is the one that breaks every nightly runtime. + #[test] + fn a_dated_alpha_sdk_keeps_its_date_in_the_derived_build() { + let manifest = test_runtime_manifest_for_update( + "wheel-gfx94x", + "gfx94x", + "gfx94x-dcgpu", + "7.14.0a20260812", + ); + assert!( + manifest.sdk_torch.is_none(), + "this test is about the pre-change manifest shape" + ); assert_eq!( - adopted.python_executable.as_deref(), - Some(python_executable.as_path().to_string_lossy().as_ref()) + sdk_torch_build_from_manifest(&manifest).as_deref(), + Some("rocm7.14.0a20260812") ); - - let _ = fs::remove_dir_all(root); - Ok(()) } + /// A manifest that names no version at all must not invent a build. #[test] - fn runtime_adopt_rejects_runtime_id_without_family() -> Result<()> { - let (root, paths) = test_paths("runtime-adopt-no-family"); - let external_root = root.join("external-therock-venv"); - let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); - let python_executable = scripts_dir.join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); - let sdk_root = external_root.join("rocm_sdk"); - let sdk_bin = sdk_root.join("bin"); - fs::create_dir_all(&scripts_dir)?; - fs::create_dir_all(&sdk_bin)?; - fs::write(&python_executable, "python")?; + fn a_manifest_with_no_version_identifies_no_build() { + let mut manifest = + test_runtime_manifest_for_update("wheel-gfx94x", "gfx94x", "gfx94x-dcgpu", " "); + manifest.sdk_torch = None; - let error = adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { - python_executable, - install_root: external_root, - runtime_id: "therock-release".to_owned(), - runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), - replace: false, - }, - therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root), - bin_path: Some(sdk_bin), - ..therock::RocmSdkPythonProbe::default() - }, - ) - .unwrap_err() - .to_string(); + assert_eq!(sdk_torch_build_from_manifest(&manifest), None); + } - assert!(error.contains("must include a TheRock family suffix")); - assert!(!runtime_registry_dir(&paths).exists()); + /// Two runtimes installed side by side, as a pre-warmed CI tree holds them. + /// + /// They differ in `runtime_key`, `version` and install root, and share one + /// `runtime_id` — that is what the field means, so this is not a corrupt + /// registry. + fn side_by_side_runtimes() -> Vec { + let mut older = test_runtime_manifest_for_update( + "release-wheel-gfx94x-dcgpu-7-13-0", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.13.0", + ); + older.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-13-0"); + let mut newer = test_runtime_manifest_for_update( + "release-wheel-gfx94x-dcgpu-7-14-0", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.14.0", + ); + newer.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0"); + vec![older, newer] + } - let _ = fs::remove_dir_all(root); - Ok(()) + /// The interpreter names its runtime where the shared `runtime_id` cannot. + /// + /// This is the cross-wiring that settled the active runtime's torch into an + /// older runtime's environment: the engine's env id drops the version, so + /// the environment belongs to 7.13.0 while the caller's selector says only + /// "release, gfx94X-dcgpu". Resolving by install root has to pick 7.13.0. + #[test] + fn the_runtime_is_resolved_by_its_interpreter_not_the_shared_runtime_id() { + let manifests = side_by_side_runtimes(); + + assert_eq!( + runtime_manifest_for_selector(&manifests, "therock-release:gfx94X-dcgpu") + .map(|manifest| manifest.runtime_key.as_str()), + None, + "the shared runtime_id names two runtimes, so a selector cannot resolve it" + ); + assert_eq!( + runtime_key_owning_python( + &manifests, + Path::new("/runtimes/release-wheel-gfx94x-dcgpu-7-13-0/bin/python3"), + ), + Some("release-wheel-gfx94x-dcgpu-7-13-0"), + "the interpreter's install root names the runtime being settled" + ); } + /// An interpreter outside every install root leaves the caller's selector alone. + /// + /// External and self-managed environments live outside the registry, and + /// inventing an owner for them would settle a runtime nobody asked about. #[test] - fn runtime_adopt_refuses_to_overwrite_without_replace() -> Result<()> { - let (root, paths) = test_paths("runtime-adopt-replace"); - let external_root = root.join("external-therock-venv"); - let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); - let python_executable = scripts_dir.join(if cfg!(windows) { - "python.exe" - } else { - "python" - }); - let sdk_root = external_root.join("rocm_sdk"); - let sdk_bin = sdk_root.join("bin"); - fs::create_dir_all(&scripts_dir)?; - fs::create_dir_all(&sdk_bin)?; - let amdhip = sdk_bin.join(if cfg!(windows) { - "amdhip64_7.dll" - } else { - "libamdhip64.so" - }); - let hipblas = sdk_bin.join(if cfg!(windows) { - "hipblas.dll" - } else { - "libhipblas.so" - }); - fs::write(&python_executable, "python")?; - fs::write(&amdhip, "amdhip")?; - fs::write(&hipblas, "hipblas")?; + fn an_interpreter_outside_every_install_root_owns_nothing() { + assert_eq!( + runtime_key_owning_python( + &side_by_side_runtimes(), + Path::new("/opt/somewhere-else/bin/python3"), + ), + None + ); + } - let probe = therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root.clone()), - bin_path: Some(sdk_bin.clone()), - runtime_roots: vec![sdk_root], - bin_paths: vec![sdk_bin.clone()], - library_paths: vec![sdk_bin], - resolved_libraries: vec![ - therock::RocmSdkLibraryProbe { - shortname: "amdhip64".to_owned(), - paths: vec![amdhip], - }, - therock::RocmSdkLibraryProbe { - shortname: "hipblas".to_owned(), - paths: vec![hipblas], - }, - ], - ..therock::RocmSdkPythonProbe::default() - }; - let request = AdoptRuntimeRequest { - python_executable: python_executable.clone(), - install_root: external_root.clone(), - runtime_id: "therock-release:gfx120X-all".to_owned(), - runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), - replace: false, + /// A prefix match alone is ambiguous once roots nest, so the longest wins. + #[test] + fn the_longest_containing_install_root_owns_the_interpreter() { + let mut outer = test_runtime_manifest_for_update( + "outer", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.13.0", + ); + outer.install_root = PathBuf::from("/runtimes"); + let mut inner = test_runtime_manifest_for_update( + "inner", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.14.0", + ); + inner.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0"); + + assert_eq!( + runtime_key_owning_python( + &[outer, inner], + Path::new("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0/bin/python3"), + ), + Some("inner") + ); + } + + /// A torch the user installed themselves is kept, and named as kept. + /// + /// The Python does not exist and an index is supplied, so an alignment that + /// still ran would reach the install and come back `InstallFailed`. Getting + /// the divergence described instead — the build that was not installed, and + /// the one still there — is what shows the rewrite was skipped rather than + /// attempted and then reported differently. + #[test] + fn disabling_alignment_keeps_the_installed_torch_and_says_what_it_declined() { + let mut env = ScopedTestEnv::new(); + env.set("ROCM_CLI_DISABLE_TORCH_ALIGNMENT", "1"); + let probe = therock::TorchAlignmentProbe { + installed_torch: Some("2.9.0+cpu".to_owned()), + engine_requires_torch: Some("torch==2.11.0+gitd0c8b1f".to_owned()), }; - adopt_runtime_from_probe(&paths, request.clone(), probe.clone())?; - let error = adopt_runtime_from_probe(&paths, request, probe.clone()) - .unwrap_err() - .to_string(); - assert!(error.contains("already exists")); + let outcome = align_runtime_torch( + &test_app_paths(), + Path::new("/nonexistent/python"), + Some("https://example.invalid/simple"), + Some("rocm7.13.0"), + "vllm", + Ok(&probe), + ); - adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { - python_executable, - install_root: external_root, - runtime_id: "therock-release:gfx120X-all".to_owned(), - runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), - replace: true, - }, - probe, - )?; + assert_eq!( + outcome, + TorchAlignment::Disabled { + wanted: "2.11.0+rocm7.13.0".to_owned(), + kept: "2.9.0+cpu".to_owned(), + } + ); + } - let _ = fs::remove_dir_all(root); - Ok(()) + /// Unset, the rewrite is attempted as usual. + /// + /// Paired with the test above so the opt-out cannot appear to work for an + /// unrelated reason. The index is withheld here on purpose: it is the first + /// thing read after the gate, so the call stops there rather than reaching a + /// real install, and the outcome still tells the two paths apart. + #[test] + fn alignment_runs_unless_the_variable_is_set() { + let mut env = ScopedTestEnv::new(); + env.clear("ROCM_CLI_DISABLE_TORCH_ALIGNMENT"); + let probe = therock::TorchAlignmentProbe { + installed_torch: Some("2.9.0+cpu".to_owned()), + engine_requires_torch: Some("torch==2.11.0+gitd0c8b1f".to_owned()), + }; + + let outcome = align_runtime_torch( + &test_app_paths(), + Path::new("/nonexistent/python"), + None, + Some("rocm7.13.0"), + "vllm", + Ok(&probe), + ); + + assert_eq!( + outcome, + TorchAlignment::NotApplicable( + "the runtime manifest records no wheel index to install from".to_owned() + ), + "the opt-out must not fire when the variable is unset" + ); } + /// Opting out of a rewrite that was never due reports the runtime as it is. + /// + /// `Disabled` says a replacement was declined. On a runtime already holding + /// the SDK's build there was none to decline, and saying otherwise would send + /// the reader looking for a torch that was spared when nothing was. #[test] - fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { - let (root, paths) = test_paths("engine-plugin"); - let plugin_dir = paths.primary_engine_plugin_dir(); - fs::create_dir_all(&plugin_dir)?; - let plugin_path = plugin_dir.join( - rocm_engine_protocol::platform_engine_plugin_binary_name("vllm"), + fn disabling_alignment_over_an_aligned_runtime_still_reports_it_aligned() { + let mut env = ScopedTestEnv::new(); + env.set("ROCM_CLI_DISABLE_TORCH_ALIGNMENT", "1"); + let probe = therock::TorchAlignmentProbe { + installed_torch: Some("2.11.0+rocm7.13.0".to_owned()), + engine_requires_torch: Some("torch==2.11.0+gitd0c8b1f".to_owned()), + }; + + let outcome = align_runtime_torch( + &test_app_paths(), + Path::new("/nonexistent/python"), + Some("https://example.invalid/simple"), + Some("rocm7.13.0"), + "vllm", + Ok(&probe), ); - fs::write(&plugin_path, "plugin")?; - let discovered = find_engine_plugin_binary("vllm", engine_plugin_dirs(&paths))?; - let _ = fs::remove_dir_all(root); + assert_eq!( + outcome, + TorchAlignment::AlreadyAligned { + version: "2.11.0+rocm7.13.0".to_owned(), + } + ); + } - assert_eq!(discovered, Some(plugin_path)); - Ok(()) + /// The kept torch is a deliberate divergence, and the remedy is not reinstall. + /// + /// The engine's pin is unsatisfied on purpose here, so reporting a violation + /// would answer the user's instruction with an error — and the remedy that + /// comes with it, `--reinstall`, is the one action that would undo the very + /// decision they made. + #[test] + fn a_disabled_alignment_diverges_on_torch_without_offering_a_reinstall() { + let outcome = TorchAlignment::Disabled { + wanted: "2.11.0+rocm7.13.0".to_owned(), + kept: "2.9.0+cpu".to_owned(), + }; + assert_eq!(deliberately_diverged_package(&outcome), Some("torch")); + + let rendered = render_torch_alignment(&outcome, "vllm", None); + assert!( + rendered.contains(" torch_alignment: disabled\n"), + "the block has to name the state, got {rendered:?}" + ); + assert!( + rendered.contains( + "ROCM_CLI_DISABLE_TORCH_ALIGNMENT is set; keeping 2.9.0+cpu rather than installing 2.11.0+rocm7.13.0" + ), + "the block has to name the variable and both builds, got {rendered:?}" + ); + + let dependencies = classify_dependency_details( + vec![ + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.9.0+cpu` is installed".to_owned(), + ], + deliberately_diverged_package(&outcome), + ); + assert!(matches!( + dependencies, + EngineDependencyCheck::ExpectedDivergence(_) + )); + let rendered = render_engine_dependency_check("vllm", &dependencies); + assert!( + !rendered.contains("--reinstall"), + "the remedy would undo the opt-out, got {rendered:?}" + ); } - #[test] - fn engine_plugin_discovery_prefers_primary_plugin_dir() -> Result<()> { - let (root, paths) = test_paths("engine-plugin-precedence"); - let primary_dir = paths.primary_engine_plugin_dir(); - let compatibility_dir = paths.data_dir.join("engines"); - fs::create_dir_all(&primary_dir)?; - fs::create_dir_all(&compatibility_dir)?; - let name = rocm_engine_protocol::platform_engine_plugin_binary_name("vllm"); - let primary_path = primary_dir.join(&name); - fs::write(&primary_path, "primary")?; - fs::write(compatibility_dir.join(&name), "compatibility")?; + /// Opting out suppresses the correction, not the diagnosis. + /// + /// The escape hatch exists because the SDK's build does not always work, so it + /// cannot also become a way to make a runtime that does not work pass. The + /// device check is taken over the torch the user kept and read exactly as it + /// would be otherwise: a host with a GPU the runtime cannot open, or can open + /// and then not run a kernel on, still fails the install. + #[test] + fn opting_out_still_fails_an_install_that_left_a_gpu_host_unable_to_serve() { + let no_devices = RuntimeDeviceCheck::NoDevices { + torch_version: "2.9.0+cpu".to_owned(), + hip_version: "7.2.53211".to_owned(), + }; + let kernel_failed = RuntimeDeviceCheck::KernelFailed { + torch_version: "2.9.0+cpu".to_owned(), + error: "AcceleratorError: device kernel image is invalid".to_owned(), + }; - let discovered = find_engine_plugin_binary("vllm", engine_plugin_dirs(&paths))?; - let _ = fs::remove_dir_all(root); + assert!(install_left_runtime_unusable( + &no_devices, + &HostGpu::Detected + )); + assert!(install_left_runtime_unusable( + &kernel_failed, + &HostGpu::Detected + )); + } - assert_eq!(discovered, Some(primary_path)); - Ok(()) + /// On a machine with no GPU the same verdict is the correct answer. + /// + /// Someone running a CPU torch deliberately is the person this opt-out is for, + /// and a runtime reporting no device there has not failed at anything. + #[test] + fn opting_out_on_a_host_with_no_gpu_leaves_the_install_successful() { + let no_devices = RuntimeDeviceCheck::NoDevices { + torch_version: "2.9.0+cpu".to_owned(), + hip_version: "7.2.53211".to_owned(), + }; + + assert!(!install_left_runtime_unusable( + &no_devices, + &HostGpu::Absent + )); } #[test] - fn a_satisfied_dependency_check_is_stated_plainly() { - let rendered = render_engine_dependency_check("vllm", &EngineDependencyCheck::Satisfied); + fn a_runtime_that_sees_no_devices_names_the_torch_that_cannot_use_the_sdk() { + // The install succeeded and the engine's requirements are satisfied, so + // every other surface reports this runtime healthy. Measured on MI300X: + // this torch loads against a ROCm 7.13 SDK and enumerates nothing, and + // the first symptom would otherwise be a serve failure. + let rendered = render_runtime_device_check(&RuntimeDeviceCheck::NoDevices { + torch_version: "2.11.0+gitd0c8b1f".to_owned(), + hip_version: "7.2.53211".to_owned(), + }); - assert_eq!(rendered, " dependency_check: satisfied\n"); + assert!(rendered.contains(" device_check: no_devices\n")); + assert!(rendered.contains("2.11.0+gitd0c8b1f")); + assert!(rendered.contains("7.2.53211")); + assert!(rendered.contains("Failed to infer device type")); + // No remedy is offered on purpose: reinstalling to satisfy the engine's + // pin is what produces this state. + assert!( + !rendered.contains("action:"), + "a remedy that recreates the fault must not be suggested: {rendered}" + ); } #[test] - fn a_violated_dependency_check_names_the_pin_and_the_remedy() { - // The SDK torch stack replaced the build vLLM pins. The install - // succeeded, so the only signal the user gets is this block. - let rendered = render_engine_dependency_check( - "vllm", - &EngineDependencyCheck::Violated(vec![ - "The package `vllm` requires `torch==2.10.0+git8514f05`, but `2.9.1+rocm7.14.0a20260611` is installed".to_owned(), - ]), + fn a_usable_runtime_states_how_many_devices_it_found() { + let rendered = render_runtime_device_check(&RuntimeDeviceCheck::Usable { + device_count: 8, + torch_version: "2.11.0+rocm7.13.0".to_owned(), + }); + + assert_eq!( + rendered, + " device_check: usable (8 device(s), torch 2.11.0+rocm7.13.0)\n" ); + } - assert!(rendered.contains(" dependency_check: violated\n")); - assert!(rendered.contains("torch==2.10.0+git8514f05")); - assert!(rendered.contains("2.9.1+rocm7.14.0a20260611")); - assert!(rendered.contains(" action: rocm engines install vllm --reinstall\n")); + #[test] + fn a_device_check_without_an_interpreter_is_not_verified_not_healthy() { + // Same rule as the dependency check: absence of an answer is never + // allowed to read as a passing one. + let outcome = runtime_device_check(None, &[]); + + assert_eq!( + outcome, + RuntimeDeviceCheck::NotVerified( + "the runtime's Python environment could not be located".to_owned() + ) + ); + assert!(render_runtime_device_check(&outcome).contains("not_verified")); } #[test] @@ -26010,7 +28371,7 @@ ID_LIKE="suse opensuse" // than fall through to `satisfied`. let (root, paths) = test_paths("engine-dependency-no-python"); - let outcome = engine_dependency_check(&paths, "vllm", None); + let outcome = engine_dependency_check(&paths, "vllm", None, None); let _ = fs::remove_dir_all(root); assert_eq!( @@ -27070,6 +29431,7 @@ ID_LIKE="suse opensuse" ], ..therock::RocmSdkPythonProbe::default() }), + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms, @@ -27108,6 +29470,7 @@ ID_LIKE="suse opensuse" python_executable: Some("python".to_owned()), pip_cache_dir: None, rocm_sdk: None, + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms: 1, diff --git a/apps/rocm/src/provider_keys.rs b/apps/rocm/src/provider_keys.rs index 356cc9eac..d05222134 100644 --- a/apps/rocm/src/provider_keys.rs +++ b/apps/rocm/src/provider_keys.rs @@ -21,17 +21,23 @@ pub(crate) struct ProviderKeyStatus { pub source: String, } -#[derive(Debug, Clone, Eq, PartialEq)] -pub(crate) struct ProviderApiKey { - pub value: String, - pub source: String, +pub(crate) struct ProviderCredential(String); + +impl ProviderCredential { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } + + pub(crate) fn into_value(self) -> String { + self.0 + } } pub(crate) trait ProviderKeyStore: Send + Sync { fn label(&self) -> &'static str; - fn get_secret(&self, provider: &str) -> Result>>; - fn set_secret(&self, provider: &str, secret: &[u8]) -> Result<()>; - fn clear_secret(&self, provider: &str) -> Result<()>; + fn get_entry(&self, provider: &str) -> Result>>; + fn store_entry(&self, provider: &str, value: &[u8]) -> Result<()>; + fn remove_entry(&self, provider: &str) -> Result<()>; } #[derive(Debug, Clone, Copy, Default)] @@ -49,9 +55,9 @@ pub(crate) fn provider_key_status(provider: &str, env_name: &str) -> ProviderKey ) } -pub(crate) fn resolve_provider_api_key(provider: &str, env_name: &str) -> Result { +pub(crate) fn provider_credential(provider: &str, env_name: &str) -> Result { let store = NativeProviderKeyStore; - resolve_provider_api_key_with_store( + provider_credential_with_store( &store, provider, env_name, @@ -61,12 +67,12 @@ pub(crate) fn resolve_provider_api_key(provider: &str, env_name: &str) -> Result ) } -pub(crate) fn set_provider_api_key(provider: &str, value: &str) -> Result { +pub(crate) fn store_provider_credential(provider: &str, value: &str) -> Result { let store = NativeProviderKeyStore; - set_provider_api_key_with_store(&store, provider, value) + store_provider_credential_with_store(&store, provider, value) } -pub(crate) fn set_provider_api_key_with_store( +pub(crate) fn store_provider_credential_with_store( store: &dyn ProviderKeyStore, provider: &str, value: &str, @@ -77,7 +83,7 @@ pub(crate) fn set_provider_api_key_with_store( } ensure_cloud_provider(provider)?; store - .set_secret(provider, trimmed.as_bytes()) + .store_entry(provider, trimmed.as_bytes()) .with_context(|| format!("failed to save {provider} API key in secure storage"))?; Ok(ProviderKeyStatus { state: ProviderKeyState::Configured, @@ -85,18 +91,18 @@ pub(crate) fn set_provider_api_key_with_store( }) } -pub(crate) fn clear_provider_api_key(provider: &str) -> Result { +pub(crate) fn remove_provider_credential(provider: &str) -> Result { let store = NativeProviderKeyStore; - clear_provider_api_key_with_store(&store, provider) + remove_provider_credential_with_store(&store, provider) } -pub(crate) fn clear_provider_api_key_with_store( +pub(crate) fn remove_provider_credential_with_store( store: &dyn ProviderKeyStore, provider: &str, ) -> Result { ensure_cloud_provider(provider)?; store - .clear_secret(provider) + .remove_entry(provider) .with_context(|| format!("failed to clear {provider} API key from secure storage"))?; Ok(ProviderKeyStatus { state: ProviderKeyState::Missing, @@ -138,8 +144,8 @@ fn provider_key_status_with_store( source: format!("env:{env_name}"), }; } - match store.get_secret(provider) { - Ok(Some(secret)) if !secret.is_empty() => ProviderKeyStatus { + match store.get_entry(provider) { + Ok(Some(value)) if !value.is_empty() => ProviderKeyStatus { state: ProviderKeyState::Configured, source: secure_source_label(store.label()), }, @@ -154,32 +160,26 @@ fn provider_key_status_with_store( } } -fn resolve_provider_api_key_with_store( +fn provider_credential_with_store( store: &dyn ProviderKeyStore, provider: &str, env_name: &str, env_value: Option, -) -> Result { +) -> Result { ensure_cloud_provider(provider)?; if let Some(value) = env_value { - return Ok(ProviderApiKey { - value, - source: format!("env:{env_name}"), - }); + return Ok(ProviderCredential(value)); } - match store.get_secret(provider) { - Ok(Some(secret)) if !secret.is_empty() => { - let value = String::from_utf8(secret) + match store.get_entry(provider) { + Ok(Some(value)) if !value.is_empty() => { + let value = String::from_utf8(value) .context("stored provider API key was not valid UTF-8")? .trim() .to_owned(); if value.is_empty() { bail!("{provider} API key in secure storage is empty"); } - Ok(ProviderApiKey { - value, - source: secure_source_label(store.label()), - }) + Ok(ProviderCredential(value)) } Ok(_) => bail!( "{provider} provider requires a saved API key; run `rocm config set-provider-key {provider}` or set {env_name} for this session" @@ -207,21 +207,21 @@ impl ProviderKeyStore for NativeProviderKeyStore { native_store_label() } - fn get_secret(&self, provider: &str) -> Result>> { + fn get_entry(&self, provider: &str) -> Result>> { with_native_entry(provider, |entry| match entry.get_secret() { - Ok(secret) => Ok(Some(secret)), + Ok(value) => Ok(Some(value)), Err(KeyringError::NoEntry) => Ok(None), Err(error) => Err(keyring_anyhow(error)), }) } - fn set_secret(&self, provider: &str, secret: &[u8]) -> Result<()> { + fn store_entry(&self, provider: &str, value: &[u8]) -> Result<()> { with_native_entry(provider, |entry| { - entry.set_secret(secret).map_err(keyring_anyhow) + entry.set_secret(value).map_err(keyring_anyhow) }) } - fn clear_secret(&self, provider: &str) -> Result<()> { + fn remove_entry(&self, provider: &str) -> Result<()> { with_native_entry(provider, |entry| match entry.delete_credential() { Ok(()) | Err(KeyringError::NoEntry) => Ok(()), Err(error) => Err(keyring_anyhow(error)), @@ -236,7 +236,7 @@ impl ProviderKeyStore for NativeProviderKeyStore { /// runtime *context* is already entered on the calling thread, that nested /// `block_on` panics with "Cannot start a runtime from within a runtime". The /// dash resolves keys off-runtime, but this is the single chokepoint for every -/// store op (get/set/clear) and for `resolve_provider_api_key` / +/// store op (get/store/remove) and for `provider_credential` / /// `provider_key_status`, so guard the whole class here: when a runtime is /// active, run the entry build *and* the action on a fresh OS thread that has no /// runtime entered. @@ -348,22 +348,22 @@ mod tests { "test keychain" } - fn get_secret(&self, provider: &str) -> Result>> { + fn get_entry(&self, provider: &str) -> Result>> { if let Some(fail) = self.fail { bail!("{fail}"); } Ok(self.secrets.lock().unwrap().get(provider).cloned()) } - fn set_secret(&self, provider: &str, secret: &[u8]) -> Result<()> { + fn store_entry(&self, provider: &str, value: &[u8]) -> Result<()> { self.secrets .lock() .unwrap() - .insert(provider.to_owned(), secret.to_vec()); + .insert(provider.to_owned(), value.to_vec()); Ok(()) } - fn clear_secret(&self, provider: &str) -> Result<()> { + fn remove_entry(&self, provider: &str) -> Result<()> { self.secrets.lock().unwrap().remove(provider); Ok(()) } @@ -392,19 +392,20 @@ mod tests { } #[test] - fn provider_key_store_round_trips_without_exposing_value_in_status() -> Result<()> { + fn provider_credential_round_trip_keeps_secret_out_of_status() -> Result<()> { let store = MemoryKeyStore::default(); - store.set_secret("openai", b"sk-secret-sentinel")?; + store_provider_credential_with_store(&store, "openai", "sk-secret-sentinel")?; let status = provider_key_status_with_store(&store, "openai", "OPENAI_API_KEY", None); - let resolved = - resolve_provider_api_key_with_store(&store, "openai", "OPENAI_API_KEY", None)?; + let credential = provider_credential_with_store(&store, "openai", "OPENAI_API_KEY", None)?; assert_eq!(status.state, ProviderKeyState::Configured); assert_eq!(status.source, "secure:test keychain"); assert!(!provider_key_status_label(&status).contains("sk-secret")); - assert_eq!(resolved.value, "sk-secret-sentinel"); - assert_eq!(resolved.source, "secure:test keychain"); + assert_eq!(credential.as_str(), "sk-secret-sentinel"); + + remove_provider_credential_with_store(&store, "openai")?; + assert!(store.get_entry("openai")?.is_none()); Ok(()) } @@ -415,9 +416,10 @@ mod tests { ..MemoryKeyStore::default() }; - let error = resolve_provider_api_key_with_store(&store, "openai", "OPENAI_API_KEY", None) - .unwrap_err() - .to_string(); + let error = match provider_credential_with_store(&store, "openai", "OPENAI_API_KEY", None) { + Ok(_) => panic!("credential resolution should fail when storage is unavailable"), + Err(error) => error.to_string(), + }; assert!(error.contains("secure API-key storage is unavailable")); assert!(error.contains("no plaintext fallback was used")); @@ -428,9 +430,10 @@ mod tests { let store = MemoryKeyStore::default(); let error = - resolve_provider_api_key_with_store(&store, "anthropic", "ANTHROPIC_API_KEY", None) - .unwrap_err() - .to_string(); + match provider_credential_with_store(&store, "anthropic", "ANTHROPIC_API_KEY", None) { + Ok(_) => panic!("credential resolution should fail when no credential is stored"), + Err(error) => error.to_string(), + }; assert!(error.contains("requires a saved API key")); assert!(error.contains("rocm config set-provider-key anthropic")); @@ -455,7 +458,7 @@ mod tests { let outcome = rt.block_on(async { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { // Read-only: never writes to a real keychain on dev machines. - NativeProviderKeyStore.get_secret("anthropic") + NativeProviderKeyStore.get_entry("anthropic") })) }); assert!( diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 665074554..af6e8609b 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -114,7 +114,7 @@ struct LocalProvider<'a> { struct RemoteProvider { provider: &'static str, - api_key_env: &'static str, + credential_env: &'static str, model_env: &'static str, endpoint_env: &'static str, default_endpoint: &'static str, @@ -261,14 +261,14 @@ fn provider_adapter<'a>( "local" => Box::new(LocalProvider { paths }), "openai" => Box::new(RemoteProvider { provider: "openai", - api_key_env: "OPENAI_API_KEY", + credential_env: "OPENAI_API_KEY", model_env: "ROCM_CLI_OPENAI_MODEL", endpoint_env: "OPENAI_BASE_URL", default_endpoint: "https://api.openai.com/v1/chat/completions", }), "anthropic" => Box::new(RemoteProvider { provider: "anthropic", - api_key_env: "ANTHROPIC_API_KEY", + credential_env: "ANTHROPIC_API_KEY", model_env: "ROCM_CLI_ANTHROPIC_MODEL", endpoint_env: "ANTHROPIC_BASE_URL", default_endpoint: "https://api.anthropic.com/v1/messages", @@ -366,7 +366,8 @@ impl ProviderAdapter for RemoteProvider { .filter(|value| !value.trim().is_empty()) .into_iter() .collect::>(); - let key_status = crate::provider_keys::provider_key_status(self.provider, self.api_key_env); + let key_status = + crate::provider_keys::provider_key_status(self.provider, self.credential_env); Ok(ProviderStatus { provider: self.provider.to_owned(), auth_status: crate::provider_keys::provider_key_status_label(&key_status), @@ -376,8 +377,8 @@ impl ProviderAdapter for RemoteProvider { } fn chat(&self, request: &ChatRequest) -> Result { - let api_key = - crate::provider_keys::resolve_provider_api_key(self.provider, self.api_key_env)?; + let credential = + crate::provider_keys::provider_credential(self.provider, self.credential_env)?; let model = resolve_remote_model(self.provider, self.model_env, request.model.as_deref())?; let endpoint = remote_endpoint(self.endpoint_env, self.default_endpoint); let (content, tool_calls) = match self.provider { @@ -386,7 +387,7 @@ impl ProviderAdapter for RemoteProvider { let json = post_json_with_headers( &endpoint, &[ - ("Authorization", format!("Bearer {}", api_key.value)), + ("Authorization", format!("Bearer {}", credential.as_str())), ("Content-Type", "application/json".to_owned()), ], &body, @@ -398,7 +399,7 @@ impl ProviderAdapter for RemoteProvider { let json = post_json_with_headers( &endpoint, &[ - ("x-api-key", api_key.value), + ("x-api-key", credential.into_value()), ("anthropic-version", "2023-06-01".to_owned()), ("Content-Type", "application/json".to_owned()), ], @@ -421,8 +422,8 @@ impl ProviderAdapter for RemoteProvider { request: &ChatRequest, on_event: &mut dyn FnMut(ProviderStreamEvent) -> Result<()>, ) -> Result { - let api_key = - crate::provider_keys::resolve_provider_api_key(self.provider, self.api_key_env)?; + let credential = + crate::provider_keys::provider_credential(self.provider, self.credential_env)?; let model = resolve_remote_model(self.provider, self.model_env, request.model.as_deref())?; let endpoint = remote_endpoint(self.endpoint_env, self.default_endpoint); match self.provider { @@ -432,7 +433,7 @@ impl ProviderAdapter for RemoteProvider { stream_json_with_headers( &endpoint, &[ - ("Authorization", format!("Bearer {}", api_key.value)), + ("Authorization", format!("Bearer {}", credential.as_str())), ("Content-Type", "application/json".to_owned()), ("Accept", "text/event-stream".to_owned()), ], @@ -447,7 +448,7 @@ impl ProviderAdapter for RemoteProvider { stream_json_with_headers( &endpoint, &[ - ("x-api-key", api_key.value), + ("x-api-key", credential.into_value()), ("anthropic-version", "2023-06-01".to_owned()), ("Content-Type", "application/json".to_owned()), ("Accept", "text/event-stream".to_owned()), diff --git a/apps/rocm/src/storage.rs b/apps/rocm/src/storage.rs index fcc0163cc..f8e910dbe 100644 --- a/apps/rocm/src/storage.rs +++ b/apps/rocm/src/storage.rs @@ -968,6 +968,7 @@ mod tests { python_executable: None, pip_cache_dir: None, rocm_sdk: None, + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms, diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index cf5fcb7e1..c520acf2f 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -266,6 +266,14 @@ pub(crate) struct InstalledRuntimeManifest { pub pip_cache_dir: Option, #[serde(default)] pub rocm_sdk: Option, + /// The torch this SDK install wrote, e.g. `2.11.0+rocm7.13.0`. + /// + /// Recorded because an engine install later overwrites torch in the same + /// environment. Reading the environment afterwards tells you what is there + /// now, not which build belongs to these libraries; only the manifest still + /// knows that, and it has to survive repeat installs to be worth anything. + #[serde(default)] + pub sdk_torch: Option, #[serde(default)] pub read_only: bool, #[serde(default)] @@ -977,6 +985,7 @@ fn install_wheel_runtime( python_executable: Some(env_python.display().to_string()), pip_cache_dir: None, rocm_sdk: Some(rocm_sdk_probe.clone()), + sdk_torch: Some(resolution.package_versions.torch.clone()), read_only: false, imported_from: None, installed_at_unix_ms: unix_time_millis(), @@ -1105,6 +1114,7 @@ fn install_tarball_runtime( python_executable: None, pip_cache_dir: None, rocm_sdk: None, + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms: unix_time_millis(), @@ -2630,6 +2640,270 @@ fn parse_rocm_sdk_probe(output: &str) -> Result { serde_json::from_str(output.trim()).context("failed to parse rocm_sdk probe output") } +/// What the runtime's torch reports about the GPUs it can actually open, and +/// whether those GPUs can actually run a kernel. +/// +/// [`validate_rocm_sdk_runtime_probe`] establishes that the SDK's libraries are +/// present and resolvable. That is not the same question as whether the torch +/// sharing the venv can enumerate a device: a torch built against a different +/// HIP version loads happily against those libraries and then reports no +/// devices at all. +/// +/// Enumeration succeeding is in turn not the same question as the device being +/// usable. A torch built against a different HIP version can enumerate the +/// GPUs and then fault on the first kernel it launches, so the two failures are +/// reported separately and must not be conflated. +#[derive(Debug, Clone, Default, serde::Deserialize)] +pub(crate) struct RuntimeDeviceProbe { + pub import_ok: bool, + pub torch_version: Option, + pub hip_version: Option, + /// `None` when torch never imported, so "unknown" stays distinct from "zero". + pub device_count: Option, + /// An import or enumeration failure. Never a kernel failure. + pub error: Option, + /// A GPU kernel failure observed *after* devices enumerated successfully. + /// `None` when no kernel was attempted (no devices, or enumeration failed). + #[serde(default)] + pub kernel_error: Option, +} + +/// Ask the runtime's own interpreter how many devices its torch can open. +/// +/// `library_paths` must be the runtime's recorded ROCm library directories (see +/// [`RocmSdkPythonProbe::library_paths`]). They are prepended to +/// `LD_LIBRARY_PATH` for the child, which is how a served process resolves them. +/// +/// This cannot be replaced with an in-process `rocm_sdk.initialize_process()` +/// call in the probe script. Measured on MI300X against a runtime that serves +/// correctly: with the library directories on `LD_LIBRARY_PATH` torch reports 8 +/// devices, and with `initialize_process()` alone it reports 0. A probe built on +/// the latter would fail healthy runtimes and send people to reinstall them, +/// which is the operation that breaks them. +pub(crate) fn probe_runtime_devices( + python_executable: &Path, + library_paths: &[PathBuf], +) -> Result { + let mut env = Vec::new(); + if !library_paths.is_empty() { + let mut entries = library_paths.to_vec(); + if let Some(existing) = std::env::var_os(LIBRARY_PATH_ENV) { + entries.extend(split_runtime_path(&existing)); + } + let joined = std::env::join_paths(entries) + .context("failed to compose the runtime library path for the device probe")?; + env.push(( + LIBRARY_PATH_ENV.to_owned(), + joined.to_string_lossy().into_owned(), + )); + } + let text = capture_python_stdout_with_env( + python_executable, + RUNTIME_DEVICE_PROBE_SCRIPT, + &env, + "launch runtime device probe", + ) + .with_context(|| { + format!( + "failed to launch runtime device probe via {}", + python_executable.display() + ) + })?; + parse_runtime_device_probe(&text) +} + +fn parse_runtime_device_probe(output: &str) -> Result { + serde_json::from_str(output.trim()).context("failed to parse runtime device probe output") +} + +/// The loader search-path variable used to expose the runtime's ROCm libraries. +#[cfg(windows)] +const LIBRARY_PATH_ENV: &str = "PATH"; +#[cfg(not(windows))] +const LIBRARY_PATH_ENV: &str = "LD_LIBRARY_PATH"; + +/// Reports what torch sees, never raising: an unusable runtime must be +/// described, not turned into a probe crash. +const RUNTIME_DEVICE_PROBE_SCRIPT: &str = r#" +import json + +out = { + "import_ok": False, + "torch_version": None, + "hip_version": None, + "device_count": None, + "error": None, + "kernel_error": None, +} + +try: + import torch + + out["import_ok"] = True + out["torch_version"] = getattr(torch, "__version__", None) + out["hip_version"] = getattr(getattr(torch, "version", None), "hip", None) + out["device_count"] = int(torch.cuda.device_count()) +except Exception as exc: + out["error"] = type(exc).__name__ + ": " + str(exc) + +# Enumeration is not execution. A runtime whose torch and HIP disagree can +# report devices and then fault on the first kernel, so the kernel is launched +# under its own guard and its failure is recorded in its own field. Only run it +# once enumeration actually produced a device: with no devices there is nothing +# to execute on, and a failed import has already been described. +if out["error"] is None and (out["device_count"] or 0) > 0: + try: + probe = torch.ones(32, device="cuda") + probe.add_(1.0) + torch.cuda.synchronize() + except Exception as exc: + out["kernel_error"] = type(exc).__name__ + ": " + str(exc) + +print(json.dumps(out)) +"#; + +/// What torch is installed, and what torch the engine's metadata demands. +#[derive(Debug, Clone, Default, serde::Deserialize)] +pub(crate) struct TorchAlignmentProbe { + /// The installed torch, e.g. `2.11.0+rocm7.13.0`. `None` if torch is absent. + pub installed_torch: Option, + /// The engine's pinned requirement, e.g. `torch==2.11.0+gitd0c8b1f`. `None` + /// when the engine is not installed yet or does not pin torch. + pub engine_requires_torch: Option, +} + +/// Read installed/required torch from distribution metadata, without importing +/// torch. Metadata questions must not depend on the runtime being usable — this +/// is called both before and after an engine install, and in the broken state +/// importing torch is exactly what fails. +pub(crate) fn probe_torch_alignment( + python_executable: &Path, + engine_distribution: &str, +) -> Result { + let env = vec![( + "ROCM_CLI_PROBE_DIST".to_owned(), + engine_distribution.to_owned(), + )]; + let text = capture_python_stdout_with_env( + python_executable, + TORCH_ALIGNMENT_PROBE_SCRIPT, + &env, + "launch torch alignment probe", + )?; + serde_json::from_str(text.trim()).context("failed to parse torch alignment probe output") +} + +const TORCH_ALIGNMENT_PROBE_SCRIPT: &str = r#" +import json +import os +import re +import importlib.metadata as md + +out = {"installed_torch": None, "engine_requires_torch": None} + +try: + out["installed_torch"] = md.version("torch") +except Exception: + pass + +try: + for raw in md.requires(os.environ.get("ROCM_CLI_PROBE_DIST", "vllm")) or []: + # `Requires-Dist` entries carry environment markers after ';'. Only the + # requirement itself matters here. + requirement = raw.split(";")[0].strip() + # The distribution name runs up to the first specifier, extra or space — + # and there is usually no space at all, as in `torch==2.11.0+gitd0c8b1f`. + matched = re.match(r"[A-Za-z0-9._-]+", requirement) + if matched is None: + continue + if matched.group(0).lower().replace("_", "-") == "torch": + out["engine_requires_torch"] = requirement + break +except Exception: + pass + +print(json.dumps(out)) +"#; + +/// Split a version into its public part and its local segment. +/// +/// `2.11.0+rocm7.13.0` -> `("2.11.0", Some("rocm7.13.0"))`. The local segment is +/// the build identifier: for TheRock wheels it names the ROCm build, and for the +/// engine's own index it is an opaque commit tag. +/// +/// Defined in `rocm-core` because the vLLM engine has to make the same split to +/// recognise a runtime the CLI deliberately realigned; two copies of this would be +/// two places for the two sides to drift apart. +pub(crate) fn split_local_version(version: &str) -> (&str, Option<&str>) { + rocm_core::uv::split_local_version(version) +} + +/// The version pinned by a `==` requirement, e.g. `torch==2.11.0+git…` -> the +/// version. Returns `None` for any looser requirement, since only an exact pin +/// tells us which release the engine was built against. +pub(crate) fn requirement_pinned_version(requirement: &str) -> Option<&str> { + let (_, version) = requirement.split_once("==")?; + let version = version.trim(); + if version.is_empty() || version.contains(',') { + return None; + } + Some(version) +} + +/// Install one exact package version from `index_url` into `python_executable`. +/// +/// Used to put the SDK's build of a package back after another installer has +/// replaced it. `--reinstall-package` is required: without it uv treats the +/// already-present distribution as satisfying the request and does nothing. +/// +/// `--index-url`, not `--extra-index-url`, so the SDK index is the only place a +/// candidate can come from. This matches how `install_therock_runtime` installs +/// from the same index, and it is load-bearing rather than cosmetic: with PyPI +/// left in the candidate set, a `+rocm` build the SDK index does not publish can +/// resolve against PyPI instead, and the caller's `Unavailable` classification — +/// the whole point of which is to say "the SDK index has no such build" — never +/// gets the resolver error it keys on. +/// +/// `--no-deps` because this is a surgical swap of one build for another build of +/// the *same release*: the environment already carries a resolved dependency tree +/// and re-resolving it here is free to move `torchvision`/`torchaudio` as a side +/// effect, which is the mixed stack this change is trying not to create. A build +/// that genuinely needs a different dependency is not silently ignored — the +/// `uv pip check` that runs immediately after reports it as a violation. +pub(crate) fn install_pinned_package( + paths: &AppPaths, + python_executable: &Path, + index_url: &str, + package: &str, + requirement: &str, +) -> Result<()> { + let uv = rocm_core::uv::ensure_uv_binary(paths) + .context("failed to acquire uv binary for the torch alignment install")?; + let mut args = rocm_core::uv::uv_pip_install_base(python_executable); + args.push("--index-url".to_owned()); + args.push(index_url.to_owned()); + args.push("--no-deps".to_owned()); + args.push("--reinstall-package".to_owned()); + args.push(package.to_owned()); + args.push(requirement.to_owned()); + let borrowed = args.iter().map(String::as_str).collect::>(); + // `run_command_with_env`, not `run_command`, for two reasons. The uv environment + // carries `UV_HTTP_TIMEOUT` (uv has no `--timeout` flag) and `UV_CACHE_DIR`; + // without the latter uv falls back to `$HOME/.cache/uv`, loses hardlinking when + // that is on another filesystem, and silently copies the whole torch stack per + // environment — and the e2e lanes' shared cache is threaded through the same + // helper. It also captures stderr on every platform, where `run_command`'s + // Windows branch reports only an exit status; the caller classifies this + // install's outcome by matching the resolver's message, so on Windows an + // unpublished build would otherwise be reported as a generic failure. + run_command_with_env( + &uv, + &borrowed, + &rocm_core::uv::uv_command_env(paths), + "install the SDK build of the engine's torch", + ) +} + pub(crate) fn validate_rocm_sdk_runtime_probe(probe: &RocmSdkPythonProbe) -> Result<()> { if !probe.import_ok { bail!( @@ -2835,6 +3109,50 @@ fn capture_command_output_with_temp_files(program: &Path, args: &[&str]) -> Resu }) } +/// Run a probe script with extra environment set and return its stdout. +/// +/// The script goes to a temp file rather than `python -c`, because the loader +/// search path this exists to set must reach the child through its environment, +/// and a file keeps the invocation identical on every platform. +fn capture_python_stdout_with_env( + python_executable: &Path, + script: &str, + env: &[(String, String)], + context_text: &str, +) -> Result { + let temp_root = if runtime_is_windows() { + windows_temp_dir("rocm-cli-python-probe")? + } else { + linux_temp_dir("rocm-cli-python-probe")? + }; + let script_path = temp_root.join("probe.py"); + fs::write(&script_path, script) + .with_context(|| format!("failed to write {}", script_path.display()))?; + + let mut command = Command::new(python_executable); + command.arg(&script_path); + for (key, value) in env { + command.env(key, value); + } + let output = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .with_context(|| format!("failed to launch {}", python_executable.display())); + let _ = fs::remove_dir_all(&temp_root); + let output = output?; + + if !output.status.success() { + bail!( + "{context_text}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + String::from_utf8(output.stdout) + .with_context(|| format!("{context_text}: failed to decode Python output")) +} + fn capture_python_stdout( python_executable: &Path, script: &str, @@ -5319,6 +5637,7 @@ echo Python 3.12.10 python_executable: Some("python".to_owned()), pip_cache_dir: None, rocm_sdk: None, + sdk_torch: None, read_only: false, imported_from: None, installed_at_unix_ms, @@ -5352,4 +5671,104 @@ echo Python 3.12.10 "invalid calendar dates should not be displayed" ); } + + #[test] + fn runtime_device_probe_without_kernel_error_field_still_parses() { + // Output produced before the kernel probe existed must not become a parse + // failure: an older runtime's probe is still a valid "no kernel attempted". + let probe = parse_runtime_device_probe( + r#"{"import_ok":true,"torch_version":"2.11.0","hip_version":"7.13", + "device_count":8,"error":null}"#, + ) + .expect("probe without kernel_error should parse"); + + assert_eq!(probe.device_count, Some(8)); + assert_eq!(probe.error, None); + assert_eq!(probe.kernel_error, None); + } + + #[test] + fn runtime_device_probe_keeps_kernel_failures_out_of_the_enumeration_error() { + // The distinction the caller acts on: devices were found, so this is not a + // "no devices" runtime, but the GPU cannot run work. + let probe = parse_runtime_device_probe( + r#"{"import_ok":true,"torch_version":"2.11.0","hip_version":"7.13", + "device_count":8,"error":null, + "kernel_error":"RuntimeError: HIP error: invalid device function"}"#, + ) + .expect("probe with kernel_error should parse"); + + assert_eq!(probe.device_count, Some(8)); + assert_eq!(probe.error, None); + assert_eq!( + probe.kernel_error.as_deref(), + Some("RuntimeError: HIP error: invalid device function") + ); + } + + #[test] + fn runtime_device_probe_reports_import_failures_only_as_enumeration_errors() { + let probe = parse_runtime_device_probe( + r#"{"import_ok":false,"torch_version":null,"hip_version":null, + "device_count":null,"error":"ImportError: no module named torch", + "kernel_error":null}"#, + ) + .expect("failed-import probe should parse"); + + assert!(!probe.import_ok); + assert_eq!(probe.device_count, None); + assert_eq!(probe.kernel_error, None); + assert_eq!( + probe.error.as_deref(), + Some("ImportError: no module named torch") + ); + } + + #[test] + fn runtime_device_probe_script_guards_the_kernel_behind_successful_enumeration() { + // The script is the contract: a kernel must never be launched when the + // import or enumeration already failed, or when there is no device to launch + // it on. Getting this wrong turns a "no devices" runtime into a crash. + assert!( + RUNTIME_DEVICE_PROBE_SCRIPT + .contains(r#"if out["error"] is None and (out["device_count"] or 0) > 0:"#), + "the kernel attempt must be gated on a clean enumeration with devices" + ); + + let guard = RUNTIME_DEVICE_PROBE_SCRIPT + .split_once(r#"if out["error"] is None"#) + .expect("script should contain the kernel guard") + .0; + assert!( + !guard.contains("device=\"cuda\"") && !guard.contains("synchronize"), + "no kernel work may run before the guard" + ); + } + + #[test] + fn runtime_device_probe_script_records_kernel_failures_in_their_own_field() { + let kernel_section = RUNTIME_DEVICE_PROBE_SCRIPT + .split_once(r#"if out["error"] is None"#) + .expect("script should contain the kernel guard") + .1; + + // Allocate, mutate, and synchronize: an unusable GPU commonly survives the + // allocation and only faults once work is actually launched and awaited. + assert!(kernel_section.contains(r#"torch.ones(32, device="cuda")"#)); + assert!(kernel_section.contains("probe.add_(1.0)")); + assert!(kernel_section.contains("torch.cuda.synchronize()")); + + assert!( + kernel_section.contains(r#"out["kernel_error"] = type(exc).__name__"#), + "a kernel failure must be recorded in kernel_error" + ); + assert!( + !kernel_section.contains(r#"out["error"] ="#), + "the kernel attempt must never overwrite the enumeration error" + ); + assert!( + !kernel_section.contains(r#"out["device_count"] ="#), + "a kernel failure must preserve the enumerated device count" + ); + } } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index e0369ecab..8471f98d0 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -71,13 +71,35 @@ pub use runtime::{ }; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, - UvCacheSource, check_dependencies, ensure_uv_binary, uv_binary_name, uv_cache_source, - uv_command_env, uv_http_timeout_secs, uv_pip_check_args, uv_pip_freeze_args, - uv_pip_install_base, uv_venv_args, violations_requiring, + UvCacheSource, ViolationSubject, check_dependencies, ensure_uv_binary, split_local_version, + uv_binary_name, uv_cache_source, uv_command_env, uv_http_timeout_secs, uv_pip_check_args, + uv_pip_freeze_args, uv_pip_install_base, uv_venv_args, violation_subject, violations_requiring, }; pub const DEFAULT_LOCAL_PORT: u16 = 11_435; pub const DEFAULT_LOCAL_HOST: &str = "127.0.0.1"; + +/// The variable that opts a machine out of rocm-cli choosing its runtime's torch. +pub const TORCH_ALIGNMENT_DISABLED_ENV: &str = "ROCM_CLI_DISABLE_TORCH_ALIGNMENT"; + +/// Whether the user has opted out of rocm-cli choosing this runtime's torch. +/// +/// Presence is the signal, so any value — including the empty string — disables the +/// alignment; that keeps `ROCM_CLI_DISABLE_TORCH_ALIGNMENT=` from reading as "off" +/// to one side and "on" to the other. +/// +/// The CLI and the vLLM engine both consult this: the engine cannot call into the +/// binary that owns the alignment, and a duplicated read is a contract that drifts. +/// If the two ever disagreed, a runtime the CLI deliberately left alone would be +/// rewritten by the engine on the very next `rocm engines install vllm` — the fight +/// the opt-out exists to end. +/// +/// This suppresses the correction, not the diagnosis. The runtime is still asked +/// what it can do, the dependency check still runs, and a runtime that opens no +/// device or cannot run a kernel on one is still reported as such. +pub fn torch_alignment_disabled() -> bool { + std::env::var_os(TORCH_ALIGNMENT_DISABLED_ENV).is_some() +} const OPTIONAL_COMMAND_TIMEOUT: Duration = Duration::from_millis(1_500); const WINDOWS_INVENTORY_QUERY_TIMEOUT: Duration = Duration::from_secs(5); const WINDOWS_VIDEO_CONTROLLER_INVENTORY_SCRIPT: &str = r#"$gpus = Get-CimInstance -ClassName Win32_VideoController -Property Name,DriverVersion,PNPDeviceID,AdapterCompatibility | Where-Object { $_.PNPDeviceID -match 'VEN_1002' -or $_.AdapterCompatibility -match 'AMD|Advanced Micro Devices' -or $_.Name -match 'AMD|Radeon|Instinct' }; foreach ($gpu in $gpus) { "GPU`t$($gpu.Name)`t$($gpu.DriverVersion)`t$($gpu.PNPDeviceID)" }"#; diff --git a/crates/rocm-core/src/uv.rs b/crates/rocm-core/src/uv.rs index ef99cf503..68bc24694 100644 --- a/crates/rocm-core/src/uv.rs +++ b/crates/rocm-core/src/uv.rs @@ -295,6 +295,78 @@ pub fn violations_requiring<'a>( .collect() } +/// What a violation line is *about*: the required package and what is there instead. +/// +/// `DependencyViolation` keeps `uv`'s line verbatim, which is right for display but +/// leaves every caller wanting to reason about a violation to re-parse it. Both the +/// CLI and the vLLM engine need the same two fields to tell a deliberate divergence +/// from a defect, so the parse lives here once. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ViolationSubject { + /// The package whose requirement is unsatisfied — `torch` in + /// ``requires `torch==2.11.0+rocm7.13.0` ``. + pub package: String, + /// The exact version the requirement pins, or `None` for any looser requirement. + /// + /// Only an exact pin says which release the requirer was built against, which is + /// the question callers reasoning about a divergence actually have. + pub required: Option, + /// The version installed instead, or `None` when `uv` reported the package as + /// not installed at all. + pub installed: Option, +} + +/// Pull the required package and the installed version out of one violation line. +/// +/// `uv` frames the line as ``The package `` requires ``, but +/// `` is installed``. Returns `None` for anything that does not carry both +/// halves of that frame, so a differently-shaped diagnostic yields no subject rather +/// than a confidently wrong one. +pub fn violation_subject(detail: &str) -> Option { + let (_, rest) = detail.split_once(UV_CHECK_REQUIREMENT_INFIX)?; + let (spec, rest) = rest.split_once('`')?; + let package = spec + .split(|character: char| { + !(character.is_alphanumeric() + || character == '-' + || character == '_' + || character == '.') + }) + .next() + .filter(|name| !name.is_empty())? + .to_owned(); + // A compound specifier (`>=1.0,<2.0`) pins nothing, and neither does a single + // inequality; only a lone `==` names the release the requirer was built against. + let required = spec + .split_once("==") + .map(|(_, version)| version.trim()) + .filter(|version| !version.is_empty() && !version.contains(',')) + .map(str::to_owned); + // `, but it's not installed` carries no version and correctly yields `None`. + let installed = rest + .split_once(", but `") + .and_then(|(_, tail)| tail.split_once('`')) + .map(|(version, _)| version.to_owned()); + Some(ViolationSubject { + package, + required, + installed, + }) +} + +/// Split a version into its public part and its local segment. +/// +/// `2.11.0+rocm7.13.0` -> `("2.11.0", Some("rocm7.13.0"))`. The local segment is the +/// build identifier: for TheRock wheels it names the ROCm build, and for an engine's +/// own index it is an opaque commit tag. Telling those two apart is what lets a +/// caller take the release from one source and the build from another. +pub fn split_local_version(version: &str) -> (&str, Option<&str>) { + match version.split_once('+') { + Some((base, local)) => (base, Some(local)), + None => (version, None), + } +} + /// Pull the unsatisfied-requirement lines out of a `uv pip check` stderr body. /// /// `uv` frames one as ``The package `` requires ``, but `` is @@ -836,4 +908,64 @@ All installed packages are compatible assert_eq!(slug("latest"), "latest"); assert_eq!(slug("weird/version space"), "weird-version-space"); } + + #[test] + fn a_violation_line_yields_the_package_and_both_versions() { + let subject = violation_subject( + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed", + ) + .expect("a well-formed violation line has a subject"); + + assert_eq!(subject.package, "torch"); + assert_eq!(subject.required.as_deref(), Some("2.11.0+gitd0c8b1f")); + assert_eq!(subject.installed.as_deref(), Some("2.11.0+rocm7.13.0")); + } + + #[test] + fn a_missing_package_has_no_installed_version() { + let subject = violation_subject( + "The package `vllm` requires `triton==3.5.0`, but it's not installed", + ) + .expect("the line still names a requirement"); + + assert_eq!(subject.package, "triton"); + assert_eq!(subject.required.as_deref(), Some("3.5.0")); + assert_eq!(subject.installed, None, "nothing is installed to name"); + } + + #[test] + fn a_loose_requirement_pins_no_release() { + // Only an exact `==` says which release the requirer was built against. A + // range or a compound specifier must not be mistaken for one. + for detail in [ + "The package `tilelang` requires `cloudpickle>=3.0`, but `2.2.1` is installed", + "The package `foo` requires `bar>=1.0,<2.0`, but `2.5` is installed", + ] { + let subject = violation_subject(detail).expect("still a violation line"); + assert_eq!( + subject.required, None, + "a looser requirement pins nothing: {detail}" + ); + } + } + + #[test] + fn a_line_without_the_requirement_frame_has_no_subject() { + // `uv` reports several other conditions under the same opening frame, and a + // confidently wrong parse of one of those is worse than no answer. + assert_eq!(violation_subject("Checked 214 packages in 12ms"), None); + assert_eq!( + violation_subject("The package `vllm` has an invalid METADATA file"), + None + ); + } + + #[test] + fn a_local_segment_is_split_from_the_release() { + assert_eq!( + split_local_version("2.11.0+rocm7.13.0"), + ("2.11.0", Some("rocm7.13.0")) + ); + assert_eq!(split_local_version("2.11.0"), ("2.11.0", None)); + } } diff --git a/docs/vllm.md b/docs/vllm.md index 04fbd71ca..aef979e05 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -18,6 +18,41 @@ the existing TheRock PyTorch stack. A prebuilt vLLM ROCm wheel can replace the TheRock torch packages or target a different ROCm soname set; that is not a valid no-fallback setup for rocm-cli GPU serving. +## Torch alignment on engine install + +Installing an engine into a managed TheRock runtime can change the torch in that +runtime. Two installers write torch into the same environment — the SDK install +writes TheRock's build, and the engine install then writes the build from its own +index — so `rocm engines install` settles which one stays and prints the result +as a `torch_alignment:` line. + +A torch that already executes a GPU kernel against the installed SDK is kept +exactly as it is, whichever installer put it there. Otherwise the runtime is +moved to the SDK's *build* of the torch *release* the engine pins: the release +comes from the engine, which was built against it, and the build comes from the +SDK, whose libraries it has to load. A `device_check:` line reports what the +result can actually do, and a realignment also reports what the runtime could do +before it. + +Set `ROCM_CLI_DISABLE_TORCH_ALIGNMENT` to keep whatever torch is installed and +skip the replacement: + +```bash +ROCM_CLI_DISABLE_TORCH_ALIGNMENT=1 rocm engines install vllm --yes +``` + +Any value works, including an empty one — the variable being set is the signal. +The install then reports `torch_alignment: disabled`, naming both the build it +would have installed and the one it kept. The device check still runs, so an +opt-out that leaves the runtime unable to serve says so rather than failing later +during serving. + +Use it when you are deliberately running a torch the alignment would replace — a +locally built wheel, a version under test, a stack pinned for a reproduction. It +is an escape hatch, not a supported configuration: the resulting combination is +not validated against the supported matrix, and a runtime that cannot execute a +kernel will fail at serving time. + Supported discovery paths: - `ROCM_CLI_VLLM_COMMAND=/path/to/vllm` diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 0ad0e4819..b7f3ac138 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -6,8 +6,8 @@ use anyhow::{Context, Result, anyhow, bail}; use clap::{Parser, Subcommand}; use rocm_core::{ AppPaths, DEFAULT_LOCAL_PORT, DependencyViolation, check_dependencies, ensure_uv_binary, - format_http_base_url, openai_models_endpoint_has_model, require_nonempty, uv_command_env, - uv_pip_install_base, violations_requiring, + format_http_base_url, openai_models_endpoint_has_model, require_nonempty, split_local_version, + uv_command_env, uv_pip_install_base, violation_subject, violations_requiring, }; use rocm_engine_protocol::{ DEFAULT_LOG_TAIL_LINES, DetectRequest, DetectResponse, DevicePolicy, @@ -141,7 +141,7 @@ struct VllmRuntime { sdk_library_paths: Vec, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Default, Deserialize)] struct TheRockRuntimeManifest { #[serde(default)] runtime_key: Option, @@ -151,11 +151,23 @@ struct TheRockRuntimeManifest { python_executable: Option, #[serde(default)] rocm_sdk: Option, + /// The SDK's own version, used only to reconstruct a build identifier for + /// manifests written before `sdk_torch` was recorded. + #[serde(default)] + version: Option, + /// The torch the SDK install wrote, e.g. `2.11.0+rocm7.13.0`. + /// + /// The CLI records it so a later engine install can be told apart from the SDK's + /// own work. Read here for the same reason in reverse: it is the only way this + /// engine can recognise a torch the CLI deliberately put back, as opposed to one + /// some other installer left behind. + #[serde(default)] + sdk_torch: Option, #[serde(default)] installed_at_unix_ms: Option, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Default, Deserialize)] struct RocmSdkRuntimeProbe { #[serde(default)] import_ok: bool, @@ -167,6 +179,8 @@ struct RocmSdkRuntimeProbe { bin_paths: Vec, #[serde(default)] library_paths: Vec, + #[serde(default)] + rocm_sdk_version: Option, } #[derive(Debug, Clone)] @@ -534,19 +548,140 @@ fn assess_runtime_repair(runtime: &VllmRuntime) -> RepairAssessment { Err(error) => return unverified_repair(&error.to_string()), }; match check_dependencies(&paths, python) { - Ok(violations) => repair_from_violations(&violations), + Ok(violations) => repair_from_violations( + &violations, + recorded_sdk_torch_build(&runtime.runtime_id).as_deref(), + torch_alignment_disabled(), + ), // An unusable `uv` or an offline host must not block an install that would // otherwise succeed; report that the check did not run and carry on as before. Err(error) => unverified_repair(&error.to_string()), } } +/// The package whose build the SDK and the engine both have an opinion about. +const TORCH_PACKAGE: &str = "torch"; + +/// Whether the user has opted out of rocm-cli choosing this runtime's torch. +/// +/// The CLI's own opt-out is the same call, not a matching one: the engine cannot +/// call into the binary that owns the alignment, and a duplicated read is a +/// contract that drifts. [`rocm_core::torch_alignment_disabled`] carries the rest. +fn torch_alignment_disabled() -> bool { + rocm_core::torch_alignment_disabled() +} + +/// Whether this violation is the torch divergence rocm-cli deliberately leaves behind. +/// +/// After an engine install, rocm-cli puts back the SDK's *build* of the torch release +/// the engine pins, because the engine's build cannot open a device against the +/// installed SDK libraries. The engine's metadata pins an exact version and cannot +/// express "same release, the SDK's build", so `uv pip check` reports the result as +/// unsatisfied forever. Treating that as a defect makes the two mechanisms fight: the +/// engine reinstalls torch to its own build, rocm-cli puts the SDK's back, and the +/// next invocation starts over — two full torch-stack flips a run, and a warning +/// claiming a repair that undid the intended state. +/// +/// With the alignment disabled the rule is simply "any torch pin": see below. +/// +/// Otherwise three conditions, all required, and they are exactly the rule rocm-cli +/// applies: take the *release* from the engine's pin and the *build* from the SDK. +/// +/// The violation must be about torch — any other unmet requirement is real. The +/// installed release must be the one the engine pins; an SDK torch of a *different* +/// release is the separate bug where the engine cannot accept what the SDK installed, +/// and a reinstall is the right answer there. And the installed build must be the one +/// the runtime's manifest records for the SDK; a torch from neither side is the +/// breakage this check exists to catch. Miss any one and the engine either fights the +/// alignment or silently accepts a runtime that cannot serve. +fn is_intended_torch_divergence( + detail: &str, + sdk_torch_build: Option<&str>, + torch_alignment_disabled: bool, +) -> bool { + let Some(subject) = violation_subject(detail) else { + return false; + }; + if !subject.package.eq_ignore_ascii_case(TORCH_PACKAGE) { + return false; + } + // Opted out, so rocm-cli does not choose this runtime's torch and no build it + // holds can be wrong *here*: the pin is unmet because the user meant it to be. + // The build and release tests below are the aligned-case rule — asking a + // hand-installed torch to match the SDK's build would fail every time, and the + // reinstall that followed would install the engine's build over exactly the torch + // the opt-out exists to keep. Only torch is spared: the check above already + // rejected every other package, so an unrelated vLLM-owned defect still repairs. + if torch_alignment_disabled { + return true; + } + let Some(sdk_torch_build) = sdk_torch_build else { + return false; + }; + let (Some(required), Some(installed)) = + (subject.required.as_deref(), subject.installed.as_deref()) + else { + return false; + }; + let (installed_release, Some(installed_build)) = split_local_version(installed) else { + return false; + }; + installed_build == sdk_torch_build && installed_release == split_local_version(required).0 +} + /// The repair decision for a set of violations found in the environment. -fn repair_from_violations(violations: &[DependencyViolation]) -> RepairAssessment { +/// +/// `sdk_torch_build` is the build identifier the runtime's manifest records for the +/// SDK's torch, or `None` when it cannot be determined — in which case nothing is +/// treated as intended and the previous behaviour stands. +/// +/// `torch_alignment_disabled` is the user's opt-out. It changes which violations count +/// as defects, never whether defects are acted on: a torch pin stops being one, and +/// everything else vLLM requires is assessed exactly as before. Returning early on the +/// opt-out instead would hide a broken torchvision behind an unrelated preference. +fn repair_from_violations( + violations: &[DependencyViolation], + sdk_torch_build: Option<&str>, + torch_alignment_disabled: bool, +) -> RepairAssessment { let owned = violations_requiring(violations, ENGINE_NAME); if owned.is_empty() { return RepairAssessment::default(); } + let (intended, defects): (Vec<&DependencyViolation>, Vec<&DependencyViolation>) = + owned.into_iter().partition(|violation| { + is_intended_torch_divergence( + &violation.detail, + sdk_torch_build, + torch_alignment_disabled, + ) + }); + + if defects.is_empty() { + // Nothing to repair. Reinstalling here would replace that torch with the + // engine's build and hand back a runtime nobody asked for. + // + // Which sentence is true depends on whose torch this is. Under the alignment it + // is the SDK's and rocm-cli put it there; under the opt-out it is the user's and + // rocm-cli never touched it. Reusing the first line for the second case would + // tell a user who hand-installed torch that the CLI had installed it for them. + let headline = if torch_alignment_disabled { + "torch alignment is disabled by ROCM_CLI_DISABLE_TORCH_ALIGNMENT; the torch this runtime holds is the user's and a reinstall would replace it" + } else { + "the runtime holds the SDK's build of the torch vLLM pins; that divergence is intended and a reinstall would undo it" + }; + let mut notes = vec![headline.to_owned()]; + notes.extend( + intended + .iter() + .map(|violation| format!("expected divergence: {}", violation.detail)), + ); + return RepairAssessment { + needed: false, + notes, + }; + } + let mut notes = vec![ "the runtime environment did not satisfy vLLM's pinned dependencies; vLLM was reinstalled to restore them".to_owned(), ]; @@ -556,13 +691,29 @@ fn repair_from_violations(violations: &[DependencyViolation]) -> RepairAssessmen // is unreadable in a terminal. Mirrors the per-finding `violation:` lines the // CLI-side renderer already emits. notes.extend( - owned + defects .iter() .map(|violation| format!("violation: {}", violation.detail)), ); - notes.push( - "if this recurs after `rocm install sdk`, the SDK torch stack is being written over vLLM's pinned torch".to_owned(), + // An intended divergence alongside a real one is still worth naming, so the reader + // is not left thinking the reinstall was about torch when it was not. + notes.extend( + intended + .iter() + .map(|violation| format!("expected divergence: {}", violation.detail)), ); + // The hint blames `rocm install sdk` for writing the SDK torch stack over vLLM's + // pins, which stops being a live theory once the manifest names the SDK's build: + // the alignment then identifies that stack and settles it, so a defect surviving + // to here is something else. It stays on under the opt-out, which spares only the + // package named `torch` — a `torchvision` or `torchaudio` defect is still the SDK + // stack written over vLLM's pins, and the opt-out has turned off the step that + // would have corrected it, so the hint is more use there rather than less. + if sdk_torch_build.is_none() { + notes.push( + "if this recurs after `rocm install sdk`, the SDK torch stack is being written over vLLM's pinned torch".to_owned(), + ); + } RepairAssessment { needed: true, notes, @@ -1178,6 +1329,70 @@ struct ManagedRuntimeCandidate { sdk_library_paths: Vec, } +/// The runtime manifests matching `runtime_id`, most recently installed first. +fn load_runtime_manifests(runtime_id: Option<&str>) -> Result> { + let paths = AppPaths::discover()?; + let registry = paths.data_dir.join("runtimes").join("registry"); + if !registry.is_dir() { + return Ok(Vec::new()); + } + let mut manifests = Vec::new(); + for entry in + fs::read_dir(®istry).with_context(|| format!("failed to read {}", registry.display()))? + { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let bytes = + fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + let Ok(manifest) = serde_json::from_slice::(&bytes) else { + continue; + }; + if !runtime_matches(&manifest, runtime_id) { + continue; + } + manifests.push((manifest.installed_at_unix_ms.unwrap_or(0), manifest)); + } + manifests.sort_by_key(|(installed_at, _)| std::cmp::Reverse(*installed_at)); + Ok(manifests + .into_iter() + .map(|(_, manifest)| manifest) + .collect()) +} + +/// The torch build the SDK installed into this runtime, as its manifest records it. +/// +/// Read from the manifest, never from the environment. By the time this runs the +/// environment may already hold some other installer's build, and taking that for +/// the SDK's would conclude the runtime is correct and leave it wrong for good. +fn recorded_sdk_torch_build(runtime_id: &str) -> Option { + let manifest = load_runtime_manifests(Some(runtime_id)) + .ok()? + .into_iter() + .next()?; + sdk_torch_build_from_manifest(&manifest) +} + +/// The SDK's torch build identifier, with the fallback for older manifests. +/// +/// Manifests written before `sdk_torch` was recorded still name the SDK version, and +/// TheRock builds that into the local segment as `rocm`. Mirrors the CLI's +/// `sdk_torch_build_for_key` so both sides agree on what "the SDK's build" means. +fn sdk_torch_build_from_manifest(manifest: &TheRockRuntimeManifest) -> Option { + if let Some(recorded) = manifest.sdk_torch.as_deref() + && let Some(build) = split_local_version(recorded).1 + { + return Some(build.to_owned()); + } + let version = manifest + .rocm_sdk + .as_ref() + .and_then(|probe| probe.rocm_sdk_version.clone()) + .or_else(|| manifest.version.clone())?; + (!version.trim().is_empty()).then(|| format!("rocm{version}")) +} + /// Registered runtimes that matched the request but were passed over because the /// interpreter they record is not there, phrased for the end of an error message. /// @@ -1231,33 +1446,8 @@ fn describe_skipped_managed_runtimes(runtime_id: Option<&str>) -> Option fn collect_managed_runtime_candidates( runtime_id: Option<&str>, ) -> Result> { - let paths = AppPaths::discover()?; - let registry = paths.data_dir.join("runtimes").join("registry"); - if !registry.is_dir() { - return Ok(Vec::new()); - } - let mut manifests = Vec::new(); - for entry in - fs::read_dir(®istry).with_context(|| format!("failed to read {}", registry.display()))? - { - let path = entry?.path(); - if path.extension().and_then(|value| value.to_str()) != Some("json") { - continue; - } - let bytes = - fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; - let Ok(manifest) = serde_json::from_slice::(&bytes) else { - continue; - }; - if !runtime_matches(&manifest, runtime_id) { - continue; - } - manifests.push((manifest.installed_at_unix_ms.unwrap_or(0), manifest)); - } - manifests.sort_by_key(|(installed_at, _)| std::cmp::Reverse(*installed_at)); - let mut candidates = Vec::new(); - for (_, manifest) in manifests { + for manifest in load_runtime_manifests(runtime_id)? { let Some(python) = manifest .python_executable .clone() @@ -2728,19 +2918,40 @@ mod tests { ); } + /// The build identifier the SDK recorded in the runtimes used by these tests. + const SDK_BUILD: &str = "rocm7.13.0"; + + /// `repair_from_violations`'s opt-out argument, named at the call sites so a bare + /// `false`/`true` does not have to be decoded against the signature. + const ALIGNED: bool = false; + const OPTED_OUT: bool = true; + #[test] fn a_consistent_environment_is_not_reinstalled() { - assert_eq!(repair_from_violations(&[]), RepairAssessment::default()); + // The other settled state: the SDK published no build of the release vLLM + // pins, so the runtime kept the engine's own build and the exact pin is + // satisfied. `uv pip check` reports nothing at all, and the recorded SDK + // build must not manufacture a finding out of that silence. + assert_eq!( + repair_from_violations(&[], Some(SDK_BUILD), ALIGNED), + RepairAssessment::default() + ); } #[test] - fn a_replaced_pinned_torch_forces_a_reinstall() { - // A second `rocm install sdk` writes the SDK's torch over the build - // vLLM pins. vLLM still imports, so resolution alone cannot see the breakage. - let assessment = repair_from_violations(&[violation( - "vllm", - "The package `vllm` requires `torch==2.10.0+git8514f05`, but `2.9.1+rocm7.14.0a20260611` is installed", - )]); + fn a_torch_of_the_wrong_release_still_forces_a_reinstall() { + // The other direction of the same problem: the SDK installed a torch + // *release* the engine does not accept. The build is the SDK's, but the + // release is not the engine's, so this is not the intended divergence and + // the reinstall that restores the engine's release must still happen. + let assessment = repair_from_violations( + &[violation( + "vllm", + "The package `vllm` requires `torch==2.10.0+git8514f05`, but `2.9.1+rocm7.14.0a20260611` is installed", + )], + Some("rocm7.14.0a20260611"), + ALIGNED, + ); assert!(assessment.needed); assert!( @@ -2754,25 +2965,106 @@ mod tests { } #[test] - fn the_whole_replaced_torch_stack_is_reported_one_finding_per_line() { - // What the failure actually looks like on hardware: the SDK moves torch, - // torchvision and torchaudio together, so all three pins are violated at - // once. Joining them into a single note produced one ~380-character line; - // each finding gets its own so a terminal can show them. - let assessment = repair_from_violations(&[ - violation( + fn the_intended_torch_divergence_alone_does_not_force_a_reinstall() { + // The steady state this change exists to stop churning. rocm-cli put the + // SDK's build of the release vLLM pins back after the engine install; the + // engine's exact pin cannot express that, so `uv pip check` reports it + // forever. Reinstalling would replace it with the build that opens no + // device, and the next invocation would do the whole thing again. + let assessment = repair_from_violations( + &[violation( "vllm", "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed", - ), - violation( + )], + Some(SDK_BUILD), + ALIGNED, + ); + + assert!( + !assessment.needed, + "the intended divergence must not trigger a reinstall: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .all(|note| !note.contains("was reinstalled")), + "no note may claim a repair that did not happen: {:?}", + assessment.notes + ); + } + + #[test] + fn a_torch_from_neither_side_still_forces_a_reinstall() { + // Same release the engine pins, but a build belonging to neither the SDK nor + // the engine — someone installed a torch by hand, or a resolver picked one + // off PyPI. Nothing about that is intended. + let assessment = repair_from_violations( + &[violation( "vllm", - "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.26.0+rocm7.13.0` is installed", - ), - violation( + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+cpu` is installed", + )], + Some(SDK_BUILD), + ALIGNED, + ); + + assert!(assessment.needed); + } + + #[test] + fn an_unidentified_sdk_build_keeps_the_previous_behaviour() { + // Without a recorded build there is no way to tell the intended divergence + // from a defect, and guessing in the permissive direction would leave a + // genuinely broken runtime alone. Fall back to repairing. + let assessment = repair_from_violations( + &[violation( "vllm", - "The package `vllm` requires `torchaudio==2.9.0+eaa9e4e`, but `2.11.0+rocm7.13.0` is installed", - ), - ]); + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed", + )], + None, + ALIGNED, + ); + + assert!(assessment.needed); + assert!( + assessment + .notes + .iter() + .any(|note| note.contains("rocm install sdk")), + "the SDK-overwrite hint belongs to exactly this un-identifiable case: {:?}", + assessment.notes + ); + } + + #[test] + fn the_whole_replaced_torch_stack_is_reported_one_finding_per_line() { + // What the failure looks like on hardware right after `rocm install sdk`: + // the SDK moves torch, torchvision and torchaudio together, so all three + // pins are violated at once. Joining them into a single note produced one + // ~380-character line; each finding gets its own so a terminal can show it. + // + // Only torch is realigned, so only torch's divergence is intended. The other + // two are genuine and still drive the reinstall — which is what restores all + // three to the engine's builds before rocm-cli puts torch back. + let assessment = repair_from_violations( + &[ + violation( + "vllm", + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed", + ), + violation( + "vllm", + "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.26.0+rocm7.13.0` is installed", + ), + violation( + "vllm", + "The package `vllm` requires `torchaudio==2.9.0+eaa9e4e`, but `2.11.0+rocm7.13.0` is installed", + ), + ], + Some(SDK_BUILD), + ALIGNED, + ); assert!(assessment.needed); let violation_notes: Vec<&String> = assessment @@ -2782,17 +3074,34 @@ mod tests { .collect(); assert_eq!( violation_notes.len(), - 3, - "every violated pin gets its own note: {:?}", + 2, + "every genuinely violated pin gets its own note: {:?}", assessment.notes ); - for package in ["torch==", "torchvision==", "torchaudio=="] { + for package in ["torchvision==", "torchaudio=="] { assert!( violation_notes.iter().any(|note| note.contains(package)), "{package} is missing from the reported notes: {:?}", assessment.notes ); } + assert!( + violation_notes + .iter() + .all(|note| !note.contains("torch==2.11.0")), + "the realigned torch is a divergence, not a violation: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .any(|note| note.starts_with("expected divergence: ") + && note.contains("torch==2.11.0+gitd0c8b1f")), + "the intended divergence is still named, so the reader is not left \ + thinking the reinstall was about torch: {:?}", + assessment.notes + ); assert!( assessment.notes.iter().all(|note| note.len() < 200), "no note should be a wall of joined findings: {:?}", @@ -2800,24 +3109,242 @@ mod tests { ); } + #[test] + fn an_sdk_built_torchvision_is_still_a_violation() { + // The SDK writes the whole torch stack, so torchvision can carry the same + // build as torch and — when the releases happen to line up — look exactly + // like the intended divergence. rocm-cli realigns torch and nothing else, so + // this is a real violation the engine must repair. The stack test above + // cannot catch a regression here: its torchvision release differs too, so + // the release check alone would still reject it. + let assessment = repair_from_violations( + &[violation( + "vllm", + "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.24.1+rocm7.13.0` is installed", + )], + Some(SDK_BUILD), + ALIGNED, + ); + + assert!( + assessment.needed, + "only torch is realigned; another package at the SDK's build is a genuine violation: {:?}", + assessment.notes + ); + } + #[test] fn unrelated_upstream_conflicts_do_not_force_a_reinstall() { // These environments routinely carry conflicts between third-party packages. // Reinstalling vLLM would not resolve them, so they must not trigger one. - let assessment = repair_from_violations(&[ - violation( - "tilelang", - "The package `tilelang` requires `cloudpickle>=3.0`, but `2.2.1` is installed", - ), - violation( - "torch", - "The package `torch` requires `sympy>=1.13`, but `1.12` is installed", - ), - ]); + let assessment = repair_from_violations( + &[ + violation( + "tilelang", + "The package `tilelang` requires `cloudpickle>=3.0`, but `2.2.1` is installed", + ), + violation( + "torch", + "The package `torch` requires `sympy>=1.13`, but `1.12` is installed", + ), + ], + Some(SDK_BUILD), + ALIGNED, + ); assert_eq!(assessment, RepairAssessment::default()); } + #[test] + fn an_opted_out_custom_torch_alone_does_not_force_a_reinstall() { + // The runtime the opt-out exists to produce: the user set + // ROCM_CLI_DISABLE_TORCH_ALIGNMENT, rocm-cli left their torch alone, and the + // engine's exact pin is therefore unmet. The build belongs to neither the SDK + // nor the engine — it is whatever the user chose — so the aligned-case rule + // would call it a defect and reinstall vLLM, which installs the engine's torch + // over the one the opt-out was set to keep. That is the CLI-side fight moved + // into the engine, and it would make the opt-out worthless on any managed + // runtime. + let assessment = repair_from_violations( + &[violation( + "vllm", + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.9.1+cu128` is installed", + )], + Some(SDK_BUILD), + OPTED_OUT, + ); + + assert!( + !assessment.needed, + "the opt-out must spare a hand-installed torch: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .all(|note| !note.contains("was reinstalled")), + "no note may claim a repair that did not happen: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .any(|note| note.contains("ROCM_CLI_DISABLE_TORCH_ALIGNMENT")), + "the reason given must be the opt-out, not a divergence rocm-cli produced: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .all(|note| !note.contains("the runtime holds the SDK's build")), + "rocm-cli did not install this torch and must not say it did: {:?}", + assessment.notes + ); + } + + #[test] + fn an_opted_out_custom_torch_still_repairs_an_unrelated_defect() { + // The opt-out is about torch, not about the environment. A vLLM-owned pin that + // has nothing to do with torch is broken the same way it was before, and + // reinstalling vLLM is still what fixes it. Returning early on the opt-out + // would hide this defect behind a preference about a different package, and the + // runtime would stay unable to serve with nothing said about why. + let assessment = repair_from_violations( + &[ + violation( + "vllm", + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.9.1+cu128` is installed", + ), + violation( + "vllm", + "The package `vllm` requires `torchvision==0.24.1+d801a34`, but `0.20.0+cu128` is installed", + ), + ], + Some(SDK_BUILD), + OPTED_OUT, + ); + + assert!( + assessment.needed, + "an unrelated vLLM pin is still a defect under the opt-out: {:?}", + assessment.notes + ); + let violation_notes: Vec<&String> = assessment + .notes + .iter() + .filter(|note| note.starts_with("violation: ")) + .collect(); + assert_eq!( + violation_notes.len(), + 1, + "only the unrelated pin is a violation: {:?}", + assessment.notes + ); + assert!( + violation_notes[0].contains("torchvision=="), + "the defect named must be the unrelated one: {:?}", + assessment.notes + ); + assert!( + assessment + .notes + .iter() + .any(|note| note.starts_with("expected divergence: ") + && note.contains("torch==2.11.0+gitd0c8b1f")), + "the spared torch is still named, so the reader is not left thinking the \ + reinstall was about torch: {:?}", + assessment.notes + ); + } + + #[test] + fn a_recorded_sdk_torch_names_the_build() { + let manifest = TheRockRuntimeManifest { + sdk_torch: Some("2.11.0+rocm7.13.0".to_owned()), + ..TheRockRuntimeManifest::default() + }; + + assert_eq!( + sdk_torch_build_from_manifest(&manifest).as_deref(), + Some("rocm7.13.0") + ); + } + + #[test] + fn a_manifest_without_sdk_torch_reconstructs_the_build_from_the_sdk_version() { + // Written before `sdk_torch` was recorded. These are the runtimes already on + // real machines, so the fallback is what repairs them rather than a nicety. + let manifest = TheRockRuntimeManifest { + rocm_sdk: Some(RocmSdkRuntimeProbe { + rocm_sdk_version: Some("7.13.0".to_owned()), + ..RocmSdkRuntimeProbe::default() + }), + ..TheRockRuntimeManifest::default() + }; + + assert_eq!( + sdk_torch_build_from_manifest(&manifest).as_deref(), + Some("rocm7.13.0") + ); + } + + #[test] + fn a_manifest_that_identifies_no_sdk_build_says_so() { + assert_eq!( + sdk_torch_build_from_manifest(&TheRockRuntimeManifest::default()), + None + ); + } + + #[test] + fn a_settled_runtime_converges_on_the_build_its_own_manifest_records() { + // The convergence proof the tests above cannot give on their own. They hand + // the classification a build literal, so a change to what + // `sdk_torch_build_from_manifest` yields — `7.13.0` where the local segment + // reads `rocm7.13.0`, say — would leave every one of them passing while the + // real pipeline churned forever: the engine would call the realigned torch a + // defect, reinstall its own build, rocm-cli would put the SDK's back, and the + // next invocation would start over. Feeding the classification the value the + // manifest actually produces is what ties the two halves together. + // + // The SDK's own torch release is deliberately not the one vLLM pins, because + // that is the case realignment exists for: the release comes from the engine, + // only the build comes from the SDK. + let settled = violation( + "vllm", + "The package `vllm` requires `torch==2.11.0+gitd0c8b1f`, but `2.11.0+rocm7.13.0` is installed", + ); + let recorded = TheRockRuntimeManifest { + sdk_torch: Some("2.9.1+rocm7.13.0".to_owned()), + ..TheRockRuntimeManifest::default() + }; + // Written before `sdk_torch` was recorded. These runtimes are already on real + // machines, so they have to settle too rather than churn forever. + let reconstructed = TheRockRuntimeManifest { + rocm_sdk: Some(RocmSdkRuntimeProbe { + rocm_sdk_version: Some("7.13.0".to_owned()), + ..RocmSdkRuntimeProbe::default() + }), + ..TheRockRuntimeManifest::default() + }; + + for manifest in [recorded, reconstructed] { + let build = sdk_torch_build_from_manifest(&manifest) + .expect("both manifest generations identify the SDK's build"); + let assessment = + repair_from_violations(std::slice::from_ref(&settled), Some(&build), ALIGNED); + + assert!( + !assessment.needed, + "the state rocm-cli settles on must survive the engine's own check: {:?}", + assessment.notes + ); + } + } + #[test] fn an_unrunnable_check_reports_itself_without_forcing_a_reinstall() { let assessment = unverified_repair("uv binary is unavailable"); diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index b66cb92e8..f05db5cfe 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -19,18 +19,65 @@ Feature: Runtime configuration When the user inspects the system Then the managed runtime folder path is not recursively nested - # The SDK and the engine share one Python environment, so a second - # `install sdk` wrote the SDK's torch stack over the build the engine pins. The - # engine still resolved, so the install reported success and every health surface - # kept saying `ready` — the first signal was a serve failure naming neither. Needs - # a real SDK install, a real engine install, and a second SDK install, so it runs - # on the nightly GPU lane. `@requires-engine:vllm` because only vLLM shares the - # runtime environment; Lemonade manages its own. + # The SDK and the engine share one Python environment and both write torch into + # it, so a second `install sdk` could leave a torch that one of the two cannot + # use. Every health surface still reported `ready` and the install still exited + # 0 — the first signal was a serve failure naming neither. The runtime now + # settles on the SDK's build of the release the engine pins, so this asserts the + # outcome that actually matters rather than the wording of a check: can the + # runtime still reach the GPU afterwards. Needs a real SDK install, a real engine + # install, and a second SDK install, so it runs on the nightly GPU lane. + # `@requires-engine:vllm` because only vLLM shares the runtime environment; + # Lemonade manages its own. + # + # The second Then is not a restatement of the first. A runtime the alignment never + # touched can still open a device, so the device check alone cannot distinguish + # "settled correctly" from "skipped entirely" — and skipping is the regression the + # gate in front of the settle step would produce. Only the alignment block + # separates them, and it is the one part of this path with no other e2e coverage. + # It reads the block's verdict rather than one string, because a torch that has + # already run a GPU kernel with this SDK is kept instead of rewritten and reports + # a `retained_*` verdict — settled, with nothing installed. @id:runtime-sdk-reinstall-keeps-engine-consistent @requires-gpu @requires-engine:vllm @nightly - Scenario: 4 - Reinstalling the SDK leaves the installed engine's requirements satisfied + Scenario: 4 - Reinstalling the SDK leaves the installed engine able to use the GPU + Given a managed runtime with an inference engine already installed + When the user installs the SDK again + Then the runtime can still use the GPU + And the torch alignment settled rather than being skipped + + # `ROCM_CLI_DISABLE_TORCH_ALIGNMENT` is the exit for the machine where the stack + # the alignment settles on — the SDK's build of the release the engine pins — + # does not work. That stack is not validated against the supported matrix, and + # the alignment runs on every path that installs an engine, so without the + # opt-out a torch the user installed deliberately is replaced again by the next + # command and the only remaining exit is to stop using the CLI. + # + # Nothing asserted it from the user's side. The unit tests reach the gate + # directly, and a gate that is honoured in isolation but bypassed by the install + # path around it looks identical to a working one from every surface a user can + # see. This is the same reinstall as scenario 4 with the opt-out set, so what + # differs between them is exactly the variable. + # + # Four claims across three Thens, because the opt-out is only coherent if all + # four hold: torch was not rewritten; the skip is reported as its own verdict + # rather than folded into the generic `not_applicable`, which would leave the + # user unable to tell whether the variable did anything; the divergence the + # opt-out deliberately leaves behind is not then sold back to that user as a + # runtime to repair by reinstalling the engine — an instruction that would undo + # what they asked for; and the checks the opt-out does not suppress still run, + # because it suppresses the correction, not the diagnosis. + # + # Same lane as scenario 4 and for the same reasons: a real SDK install and a + # real engine, on the serialized nightly GPU runners. `@requires-engine:vllm` + # because only vLLM shares the runtime environment the alignment writes into. + @id:runtime-torch-alignment-opt-out @requires-gpu @requires-engine:vllm @nightly + Scenario: 9 - Opting out of the torch alignment keeps the torch the user installed Given a managed runtime with an inference engine already installed + And the user has opted out of realigning torch When the user installs the SDK again - Then the install reports the engine's requirements as satisfied + Then the torch alignment reports the opt-out instead of rewriting torch + And the install does not offer to reinstall the engine over the kept torch + And the runtime's device health is still reported # The GPU E2E lanes no longer install the shared runtime once and keep it # forever: `xtask e2e-prewarm` asks `rocm update` whether the channel index has diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index caae41986..ea88af310 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -161,9 +161,34 @@ async fn setup_runtime_with_engine(world: &mut E2eWorld) { assert_engine_ready(world); } +/// Record the torch-alignment opt-out for this scenario's next `rocm` command. +/// +/// A behavioural precondition rather than a mechanism the feature file has to +/// name: the When step stays a plain "the user installs the SDK" and consumes +/// this on the way through. The CLI reads presence rather than value, so the +/// value is arbitrary. +#[given("the user has opted out of realigning torch")] +async fn setup_torch_alignment_opt_out(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_CLI_DISABLE_TORCH_ALIGNMENT", "1".into())); +} + #[when("the user installs the SDK")] async fn user_installs_sdk(world: &mut E2eWorld) { - let stdout = crate::run_rocm_ok(world, &["install", "sdk"]); + // Through `run_rocm_with_scenario_env` rather than `run_rocm_ok` so a Given + // can attach a behavioural fixture to this invocation — the torch-alignment + // opt-out is one — without the Gherkin naming an environment variable. The + // exit code is still asserted here, with the same diagnostic bundle + // `run_rocm_ok` prints: an install that failed leaves every Then behind it + // reading output that was never produced. + let args = ["install", "sdk"]; + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &args); + assert!( + rc == 0, + "{}", + e2e_cucumber::cli_failure_report(&args, rc, &stdout, &stderr) + ); world.cli_output = Some(stdout); } @@ -172,17 +197,199 @@ async fn user_reinstalls_sdk(world: &mut E2eWorld) { user_installs_sdk(world).await; } -#[then("the install reports the engine's requirements as satisfied")] -async fn assert_engine_requirements_satisfied(world: &mut E2eWorld) { +#[then("the runtime can still use the GPU")] +async fn assert_runtime_can_still_use_the_gpu(world: &mut E2eWorld) { let output = world.cli_output.as_deref().expect("no install output"); + // The functional signal rather than a diagnostic string: the device check asks + // the runtime's own torch how many devices it can open. A runtime left holding + // a torch that one of the two installers cannot use reports none, and that is + // the failure this scenario exists to catch. + assert!( + output.contains("device_check: usable"), + "the reinstall left a runtime that cannot open a GPU:\n{output}" + ); + // A genuine unmet requirement must still fail the scenario. Torch itself is + // expected to diverge from the engine's exact pin once it is settled on the + // SDK's build of the same release, and that is reported as a divergence. assert!( !output.contains("dependency_check: violated"), - "the reinstall left the engine's declared requirements unmet:\n{output}" + "the reinstall left a genuine requirement unmet:\n{output}" ); +} + +/// The verdict on the ` torch_alignment: ` line, without the value some +/// verdicts carry after it. +/// +/// One line carries the whole outcome, and everything that can settle this +/// question prints through it — the alignment itself, and the retention that +/// stands in for it when a torch has already run a GPU kernel with this SDK — so +/// reading the verdict is what tells the settled states apart from each other and +/// from a skip. +fn torch_alignment_verdict(output: &str) -> &str { + output + .lines() + .find_map(|line| line.trim().strip_prefix("torch_alignment: ")) + .unwrap_or_else(|| { + panic!("no torch alignment block, so the runtime was never settled:\n{output}") + }) + .split_whitespace() + .next() + .unwrap_or_default() +} + +/// Verdicts that mean the runtime was settled and its torch left in a state this +/// tool produces on purpose. +/// +/// Four, not one. Whether a reinstall rewrites torch or finds it already correct +/// depends on what the shared pre-warm tree held when the scenario started, and a +/// torch that has already executed a GPU kernel with this SDK is kept exactly as +/// it is and reported as `retained_*` — settled without anything being installed. +const SETTLED_TORCH_ALIGNMENTS: [&str; 4] = [ + "realigned", + "already_aligned", + "retained_sdk_build", + "retained_engine_build", +]; + +/// Verdicts that mean the question was left unanswered. +/// +/// `not_applicable` is the one that would otherwise go unnoticed: it is what a +/// manifest yielding no SDK build produces, which is exactly the repair path for +/// every runtime installed before `sdk_torch` was recorded. `disabled` belongs +/// here for every scenario that did not ask for the opt-out — seeing it means the +/// opt-out was applied to a user who never set it. +const UNSETTLED_TORCH_ALIGNMENTS: [&str; 4] = [ + "torch_alignment: unavailable", + "torch_alignment: install_failed", + "torch_alignment: not_applicable", + "torch_alignment: disabled", +]; + +/// The alignment ran, and settled. +/// +/// Separate from the device check above because the two can disagree: a runtime +/// whose torch was never touched at all can still open a device, so that check +/// passes whether or not the alignment fired. The `torch_alignment:` block is the +/// only evidence that `settle_engine_install` reached this engine, and the gate in +/// front of it is the part most likely to be widened or narrowed by a later change. +#[then("the torch alignment settled rather than being skipped")] +async fn assert_torch_alignment_settled(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no install output"); + let verdict = torch_alignment_verdict(output); assert!( - output.contains("dependency_check: satisfied"), - "the install did not report on the engine's requirements at all:\n{output}" + SETTLED_TORCH_ALIGNMENTS.contains(&verdict), + "torch alignment reached no settled outcome (`{verdict}`):\n{output}" ); + // Asserted negatively as well, because the check above reads only the first + // block and would pass on a second one that failed. + for unsettled in UNSETTLED_TORCH_ALIGNMENTS { + assert!( + !output.contains(unsettled), + "torch alignment reported `{unsettled}`:\n{output}" + ); + } + // Asserted on the verdict, not on the divergence lines. Requiring + // `expected_divergence` whenever `divergence:` appears cannot fail: one render + // arm emits both, the verdict first. The property worth protecting is the other + // one — that a torch this tool put here on purpose is never called a defect — + // and `violated` is the only rendering that would say so. Unconditional because + // it stays true if a future engine pin and SDK build happen to agree: then there + // is no divergence to classify, and still no violation to report. + assert!( + !output.contains("dependency_check: violated"), + "the dependency check called a settled runtime a violation:\n{output}" + ); +} + +/// The opt-out was honoured, and said so in its own words. +/// +/// `realigned` is the one verdict that proves it was ignored, and it is rejected +/// unconditionally — that is this step's falsifiable half. `disabled` cannot be +/// demanded unconditionally alongside it: the CLI only has a rewrite to skip when +/// the runtime's torch is neither of the two builds it settles on, and whether +/// this reinstall leaves such a torch depends on whether the SDK's own torch +/// release is the release the engine pins — a property of the channel index on +/// the day, not of anything the scenario controls. Every other accepted verdict +/// is one where torch was kept as it was, which is what the user asked for. +/// +/// `not_applicable` is rejected for a reason of its own: it is the generic bucket +/// for "nothing to decide", and folding the opt-out into it leaves the user who +/// set the variable unable to tell whether it took effect. The reason line is +/// read too, so the block names the variable that caused the skip rather than +/// leaving the reader to guess which of several causes applied. +#[then("the torch alignment reports the opt-out instead of rewriting torch")] +async fn assert_torch_alignment_opted_out(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no install output"); + let verdict = torch_alignment_verdict(output); + assert!( + verdict != "realigned", + "torch was realigned even though the user opted out:\n{output}" + ); + // Every settled state except the rewrite, plus the skip the opt-out produces. + let kept_torch = verdict == "disabled" || SETTLED_TORCH_ALIGNMENTS.contains(&verdict); + assert!( + kept_torch, + "the opt-out left torch in a state this tool does not produce (`{verdict}`):\n{output}" + ); + if verdict == "disabled" { + assert!( + output.contains("ROCM_CLI_DISABLE_TORCH_ALIGNMENT"), + "the skipped alignment does not name the variable that skipped it:\n{output}" + ); + } +} + +/// The kept torch is not sold to the user as a runtime to repair. +/// +/// Two surfaces print a repair for the same divergence and both have to be quiet +/// about this one: the CLI's own remedy line under a violated dependency check, +/// and the engine's built-in repair, which reports through the install's +/// `warning:` lines. A user who deliberately kept their torch and is then told to +/// reinstall the engine has been handed an instruction that undoes what they +/// asked for. +/// +/// The classification underneath is asserted as well rather than only its two +/// symptoms, because a remedy could be dropped from the renderer while the +/// divergence is still recorded as a defect — which is what every other reader of +/// that verdict, including `engines list`, would act on. +#[then("the install does not offer to reinstall the engine over the kept torch")] +async fn assert_no_reinstall_remedy(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no install output"); + assert!( + !output.contains("action: rocm engines install vllm --reinstall"), + "the CLI told the user to reinstall vLLM over the torch they kept:\n{output}" + ); + assert!( + !output.contains("vLLM was reinstalled"), + "the engine's built-in repair replaced the torch the user kept:\n{output}" + ); + assert!( + !output.contains("dependency_check: violated"), + "the torch the user kept was reported as an unmet requirement:\n{output}" + ); +} + +/// Opting out of the correction did not opt out of the diagnosis. +/// +/// The install exited 0 — the When asserts that — and on a host with a GPU that +/// is only allowed for a runtime that can use it: a runtime that opens no device, +/// or opens one it cannot run a kernel on, fails the install. So the presence of +/// the block and the absence of both bad verdicts together say the health check +/// still ran and still had teeth, without pinning a device count this scenario +/// does not own. +#[then("the runtime's device health is still reported")] +async fn assert_device_health_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_deref().expect("no install output"); + assert!( + output.contains("device_check:"), + "the opt-out suppressed the device check as well as the rewrite:\n{output}" + ); + for unusable in ["device_check: no_devices", "device_check: kernel_failed"] { + assert!( + !output.contains(unusable), + "the install reported `{unusable}` and exited 0 anyway:\n{output}" + ); + } } /// The engine inventory reports a usable engine runtime. @@ -192,8 +399,9 @@ async fn assert_engine_requirements_satisfied(world: &mut E2eWorld) { /// violated — that false green is the very thing this feature's scenario exists /// to catch — so asserting it afterwards would pass whether or not the fix /// works. Teaching that surface to notice a violated pin is tracked separately; -/// until it does, the `dependency_check: satisfied` assertion is the only -/// falsifiable signal available. +/// until it does, the device check is the falsifiable signal, because it asks +/// the runtime how many GPUs it can actually open rather than whether it looks +/// installed. fn assert_engine_ready(world: &mut E2eWorld) { let (stdout, _, _) = crate::run_rocm(world, &["engines", "list"]); assert!( diff --git a/xtask/src/e2e_prewarm.rs b/xtask/src/e2e_prewarm.rs index a8d9f3eef..ebcb130e0 100644 --- a/xtask/src/e2e_prewarm.rs +++ b/xtask/src/e2e_prewarm.rs @@ -269,6 +269,69 @@ impl RuntimeEntry { } } +/// The active-runtime selector the config still records after the runtime it +/// names has gone, if the report says so. +/// +/// This is the OTHER way the shared tree goes stale, and it is the one that kills +/// the lane outright. [`assess`] finds a runtime the registry has but the tree +/// does not; this finds a pointer the config has but the registry does not. The +/// two are independent: removing a runtime through +/// `rocm runtimes uninstall` clears the config pointers with it, so a dangling +/// pointer means something removed the tree WITHOUT the CLI — a hand cleanup on +/// the runner, or a scenario that deleted a folder it had symlinked in. +/// +/// `rocm engines install` resolves the runtime to build against as +/// `--runtime` → `config.active_runtime_key` → `config.default_runtime_id`, so a +/// dangling pointer in either field fails that resolution before anything else +/// runs: +/// +/// ```text +/// pre-warm: ensuring the vllm engine is installed +/// Error: runtime selector `release-wheel-gfx94x-dcgpu-7-14-0` from engine install +/// runtime selection is not an exact usable runtime: installed runtime not found +/// ``` +/// +/// Nine seconds in, before a single scenario, and the message names neither the +/// pre-warm nor the tree it is about. Detecting it is free because +/// `render_runtimes_text` already reports it, and reporting it is all the CLI +/// does — there is no verb that clears a pointer, so the repair has to be +/// activating something that exists. +/// +/// Only `config.json` is read this way. `engines..last_installed_runtime_id` +/// can dangle too, but nothing resolves through it — it is displayed, not +/// followed — so healing it would be motion without a failure behind it. +#[must_use] +pub fn dangling_active_runtime(runtimes_list_report: &str) -> Option<&str> { + runtimes_list_report.lines().find_map(|line| { + let rest = line + .trim() + .strip_prefix("active_status: missing manifest for ")?; + // Whichever field held the dead name — the key when one was activated, the + // id when only a default was ever set. Both are selectors the engine + // install would resolve, and both fail it the same way. + let (_, selector) = rest.split_once('=')?; + let selector = selector.trim(); + (!selector.is_empty()).then_some(selector) + }) +} + +/// Managed runtimes the tree could be pointed at instead, in report order. +/// +/// Read-only entries are excluded on purpose. `runtimes adopt` records a folder +/// the pre-warm does not own, and quietly making somebody's external ROCm install +/// the default for every GPU scenario on the runner is a larger decision than +/// repairing a pointer. If the tree holds nothing else, the caller leaves the +/// pointer dangling and the install path replaces it. +#[must_use] +pub fn activation_candidates(runtimes_list_report: &str) -> Vec { + runtimes_list_report + .lines() + .filter_map(RuntimeEntry::parse) + .filter(|entry| !entry.read_only) + .map(|entry| entry.runtime_key) + .collect() +} + /// One ` runtime format=… channel=… … status=…` line from `rocm update`. /// /// Both shapes that renderer emits are handled: the full report line, and the @@ -320,6 +383,11 @@ pub fn run(channel: &str, keep: usize, prewarm_dir: &Path) -> Result<()> { // Repairing first means `decide` reads a registry with nothing dead in it. repair_poisoned_runtimes(&rocm, prewarm_dir)?; + // And point the tree at something that exists before anything resolves the + // active runtime. Must run after the repair above, which can itself remove + // the runtime the pointer names. + repair_dangling_active_runtime(&rocm, prewarm_dir)?; + let decision = match probe(&rocm, prewarm_dir) { Ok(report) => decide(&report, channel), Err(error) => { @@ -360,10 +428,25 @@ pub fn run(channel: &str, keep: usize, prewarm_dir: &Path) -> Result<()> { } Decision::Reuse { reason } => { println!("pre-warm: reusing the shared {channel} runtime ({reason})"); - return Ok(()); } } + // Unconditional, and deliberately BEFORE the reuse early return below. The + // runtime and the serving engine are installed separately: `install sdk` + // lays down the ROCm runtime, `engines install` builds the engine venv + // against it. `decide` only ever reasons about the runtime, so a tree whose + // runtime is current but whose engine was never installed — or was left + // behind with an older runtime — resolves to `Reuse`, which used to return + // here having done nothing. The shared tree then served every GPU scenario a + // runtime with no engine, which is the one thing those lanes exist to + // exercise. Re-checking a warm tree is cheap: without `--reinstall`, + // `engines install` on a ready engine installs nothing. + ensure_default_engine(&rocm, prewarm_dir)?; + + if !runtime_changed(&decision) { + return Ok(()); + } + // An install/update that exits 0 without leaving a registry behind is the // confusing case the lanes used to call out by hand: every scenario then falls // back to installing its own runtime and the job quietly blows its time cap. @@ -394,6 +477,59 @@ pub fn run(channel: &str, keep: usize, prewarm_dir: &Path) -> Result<()> { Ok(()) } +/// Whether `decision` put a new runtime in the tree, and so whether the registry +/// check and the retention prune at the end of [`run`] have anything to do. +/// +/// Read AFTER the engine check, never inside the decision's own match arm: the +/// engine is installed separately from the runtime, so every decision — reuse +/// most of all, since it is the one a warm runner takes every time — has to +/// reach that check before this can end the pre-warm early. +const fn runtime_changed(decision: &Decision) -> bool { + !matches!(decision, Decision::Reuse { .. }) +} + +/// Install the engine the active runtime would serve on, so the shared tree has +/// one before a scenario asks it to serve. +/// +/// Which engine that is comes from the CLI rather than from a constant here: +/// `rocm engines list` marks the engine `serve` picks for the detected GPU with +/// `* ` (vLLM on Instinct, Lemonade on Strix), and the pre-warm must agree with +/// `serve` on every runner without this file learning the hardware map. +/// +/// Fatal on failure, like [`repair_poisoned_runtimes`] and unlike the freshness +/// path: reusing a stale-but-working runtime keeps a lane meaningful, whereas +/// serving with no engine fails every GPU scenario later and for reasons that +/// name none of this. +fn ensure_default_engine(rocm: &Path, prewarm_dir: &Path) -> Result<()> { + let output = rocm_command(rocm, prewarm_dir) + .args(["engines", "list"]) + .output() + .context("failed to run `rocm engines list`")?; + if !output.status.success() { + bail!("`rocm engines list` exited with {}", output.status); + } + let inventory = String::from_utf8_lossy(&output.stdout); + let engine = default_engine_from_inventory(&inventory) + .context("`rocm engines list` did not identify a default engine")?; + println!("pre-warm: ensuring the {engine} engine is installed"); + rocm_command(rocm, prewarm_dir) + .args(["engines", "install", engine, "--yes"]) + .status_ok("rocm engines install") +} + +/// The engine `rocm engines list` marks as the default for this host, if any. +/// +/// The inventory renders one line per engine as `{marker} {name:10} {note}` with +/// the marker in column 0, then indents that engine's detail lines (` adapter: +/// …`, ` runtime: …`) beneath it. Matching `* ` at the start of the line +/// unindented is therefore what separates the default engine's own line from +/// everything else the report prints. +fn default_engine_from_inventory(inventory: &str) -> Option<&str> { + inventory + .lines() + .find_map(|line| line.strip_prefix("* ")?.split_whitespace().next()) +} + /// Drop any managed runtime in the shared tree that records an install root /// outside it, so the pre-warm reinstalls instead of serving a dead one. /// @@ -466,6 +602,82 @@ fn repair_poisoned_runtimes(rocm: &Path, prewarm_dir: &Path) -> Result<()> { Ok(()) } +/// Re-point the shared tree's active runtime when it names one that is gone, so +/// the engine install resolves instead of dying nine seconds into the lane. +/// +/// See [`dangling_active_runtime`] for how the pointer goes stale and why this is +/// worth healing here. The repair is `rocm runtimes activate`: the CLI has no verb +/// that clears a pointer, so the only way out is to name something that exists. +/// Candidates are tried in report order and the first that takes it wins — which +/// may be an older runtime than the tree would ideally serve, and that is fine. +/// `decide` runs next, and the `Update` arm re-activates the newest as part of +/// updating to it. +/// +/// Doing nothing is correct when the tree holds no managed runtime at all: the +/// install path activates whatever it installs, and it runs before +/// `ensure_default_engine` — which is the only thing that would have tripped over +/// the pointer. The case that has to be handled here is the one that self-healing +/// misses: a pointer dangling while OTHER runtimes are sitting right there, where +/// `decide` says `Reuse`, nothing installs, and nothing re-activates. +fn repair_dangling_active_runtime(rocm: &Path, prewarm_dir: &Path) -> Result<()> { + if !prewarm_dir.join("data").join("runtimes").is_dir() { + return Ok(()); + } + + // Deliberately a second listing rather than one shared with the repair above: + // that repair uninstalls, which rewrites exactly the pointers read here. + let listing = match list_runtimes(rocm, prewarm_dir) { + Ok(listing) => listing, + Err(error) => { + println!( + "pre-warm: could not list runtimes ({error:#}); leaving the active runtime as it is" + ); + return Ok(()); + } + }; + + let Some(dangling) = dangling_active_runtime(&listing) else { + return Ok(()); + }; + + let candidates = activation_candidates(&listing); + if candidates.is_empty() { + println!( + "pre-warm: the active runtime {dangling} is gone and nothing is installed to take \ + its place; the install below will set it" + ); + return Ok(()); + } + + println!( + "::warning::pre-warm: the shared tree still points at the runtime {dangling}, which is \ + no longer installed — something removed it without going through \ + `rocm runtimes uninstall`. Engine installs resolve through that pointer, so re-pointing \ + it at an installed runtime. See rocm-cli#314." + ); + + for candidate in &candidates { + let activated = rocm_command(rocm, prewarm_dir) + .args(["runtimes", "activate", candidate]) + .status() + .with_context(|| format!("failed to run `rocm runtimes activate {candidate}`"))?; + if activated.success() { + println!("pre-warm: active runtime is now {candidate}"); + return Ok(()); + } + // An installed runtime can still be refused — `activate` validates the + // manifest. Try the next one rather than taking the whole lane down for + // one bad entry. + println!("pre-warm: could not activate {candidate}; trying the next runtime"); + } + + bail!( + "the shared pre-warm tree points at the runtime `{dangling}`, which is not installed, and \ + none of its {} installed runtime(s) could be activated in its place", + candidates.len() + ) +} + /// Ask the CLI what it has registered. Read-only. fn list_runtimes(rocm: &Path, prewarm_dir: &Path) -> Result { let output = rocm_command(rocm, prewarm_dir) @@ -630,6 +842,29 @@ registered ROCm runtimes next step: rocm install sdk --channel release --format wheel "; + /// A real `rocm runtimes list` on the tree that killed the lane: the config + /// still names the 7.14.0 runtime that was removed out of band, while two + /// runtimes it could be pointed at sit right there. + /// + /// Note there is no `*` marker on any entry — the renderer only marks the + /// active one by matching `active_runtime_key` against a manifest, and that is + /// exactly the match that fails here. The `active_status:` line is the only + /// thing in the document that says so. + const DANGLING_ACTIVE: &str = "\ +registered ROCm runtimes + active_runtime_id: therock-release:gfx94X-dcgpu + active_runtime_key: release-wheel-gfx94x-dcgpu-7-14-0 + previous_runtime_key: release-wheel-gfx94x-dcgpu-7-13-0 + registry: /w/e2e-prewarm/data/runtimes/registry + marker: /w/e2e-prewarm/data/runtimes/active.json + active_status: missing manifest for active_runtime_key=release-wheel-gfx94x-dcgpu-7-14-0 + installed: + adopted-external-env runtime_id=external-adopted version=7.14.0 format=wheel family=gfx94X-dcgpu mode=read-only status=usable + install_root: /opt/external-rocm + release-wheel-gfx94x-dcgpu-7-13-0 runtime_id=therock-release-gfx94x-dcgpu version=7.13.0 format=wheel family=gfx94X-dcgpu mode=managed status=usable + install_root: /w/e2e-prewarm/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-13-0 +"; + fn prewarm_runtimes_dir() -> &'static Path { Path::new("/w/e2e-prewarm/data/runtimes") } @@ -679,6 +914,65 @@ registered ROCm runtimes assert!(assess(NO_RUNTIMES, prewarm_runtimes_dir()).is_empty()); } + #[test] + fn an_active_runtime_that_is_gone_is_reported_by_name() { + assert_eq!( + dangling_active_runtime(DANGLING_ACTIVE), + Some("release-wheel-gfx94x-dcgpu-7-14-0") + ); + } + + #[test] + fn a_default_runtime_id_that_is_gone_dangles_the_same_way() { + // The other field the engine install resolves through, reported by the + // renderer under the same key when no runtime_key was ever activated. + let text = "registered ROCm runtimes\n active_runtime_id: therock-release:gfx94X-dcgpu\n \ +active_runtime_key: \n \ +active_status: missing manifest for active_runtime_id=therock-release:gfx94X-dcgpu\n"; + assert_eq!( + dangling_active_runtime(text), + Some("therock-release:gfx94X-dcgpu") + ); + } + + #[test] + fn a_tree_whose_active_runtime_is_installed_is_left_alone() { + // No active_status line at all is the healthy shape — MIXED has a dead + // runtime in it, but nothing points AT the dead one. + assert_eq!(dangling_active_runtime(MIXED), None); + assert_eq!(dangling_active_runtime(NO_RUNTIMES), None); + assert_eq!(dangling_active_runtime(""), None); + } + + #[test] + fn an_ambiguous_runtime_id_is_not_a_dangling_pointer() { + // Same `active_status:` prefix, entirely different condition: the runtimes + // are all there, one runtime_id just names several of them. Activating + // something would be a guess, and the engine install resolves it fine. + let text = "registered ROCm runtimes\n \ +active_status: ambiguous runtime_id=therock-release:gfx94X-dcgpu; activate one runtime_key: a, b\n"; + assert_eq!(dangling_active_runtime(text), None); + } + + #[test] + fn only_managed_runtimes_are_offered_as_replacements() { + // The read-only adopted entry is installed and usable, and still excluded: + // making somebody's external ROCm the default for every GPU scenario is a + // bigger decision than repairing a pointer. + assert_eq!( + activation_candidates(DANGLING_ACTIVE), + vec!["release-wheel-gfx94x-dcgpu-7-13-0".to_owned()] + ); + } + + #[test] + fn a_tree_with_nothing_installed_offers_no_replacement() { + // The caller leaves the pointer dangling here rather than failing: the + // install that follows activates whatever it installs. + assert!(activation_candidates(NO_RUNTIMES).is_empty()); + assert!(activation_candidates("").is_empty()); + } + #[test] fn a_report_that_cannot_be_read_removes_nothing() { // The conservative floor: never delete on a shape this does not recognise. @@ -731,6 +1025,73 @@ mode=managed status=ready\n install_root: /tmp/rocm-e2e-XXXX/data/runtimes/ assert_eq!(poisoned[0].format, "tarball"); } + /// A real `rocm engines list` on an Instinct host, captured verbatim: the + /// default engine's line carries the `* ` marker in column 0, and its own + /// detail lines are indented beneath it. + const ENGINES_READY: &str = "\ +Local model engines + Built-in engines are included with rocm-cli. External plugins are optional. + ROCm GPU execution is required. + Plugin folders: + 1. /w/e2e-prewarm/data/engines/plugins (primary) + lemonade default embedded Lemonade server with ROCm llama.cpp backend + adapter: built-in + runtime: not found +* vllm Linux/WSL ROCm GPU serving engine through external vLLM + adapter: built-in + runtime: /w/e2e-prewarm/data/runtimes/wheel/release-wheel-gfx94x-dcgpu-7-15-0 + protocol: 0.1.0 +"; + + #[test] + fn the_marked_engine_is_the_one_pre_warmed() { + // Which engine to install comes from the CLI's own host detection, not + // from a hardware map duplicated here. + assert_eq!(default_engine_from_inventory(ENGINES_READY), Some("vllm")); + } + + #[test] + fn an_indented_detail_line_is_not_read_as_the_default() { + // Every engine's detail lines are indented under it, and a note may well + // start with a bullet. Only the marker in column 0 names the default. + let inventory = "\ +Local model engines +* lemonade default embedded Lemonade server with ROCm llama.cpp backend + adapter: built-in + * not a marker +"; + assert_eq!(default_engine_from_inventory(inventory), Some("lemonade")); + } + + #[test] + fn an_inventory_without_a_default_engine_names_none() { + // `ensure_default_engine` turns this into an error rather than guessing an + // engine: installing the wrong one costs a multi-GiB build and still + // leaves the lane with nothing to serve on. + assert_eq!(default_engine_from_inventory(""), None); + assert_eq!( + default_engine_from_inventory("Local model engines\n lemonade embedded\n"), + None + ); + } + + #[test] + fn reusing_the_shared_runtime_still_reaches_the_engine_check() { + // The regression this guards: `Reuse` — the decision EVERY warm runner + // takes, run after run — used to end `run` with a `return` inside its own + // match arm, before anything looked at the engine. The early return is now + // this predicate, read only AFTER `ensure_default_engine`, so reuse cannot + // skip the engine. Reuse must still install and update NOTHING, which is + // what keeps it cheap enough to re-check the engine on every run. + assert!(!runtime_changed(&Decision::Reuse { + reason: "up to date".to_owned() + })); + assert!(runtime_changed(&Decision::Install)); + assert!(runtime_changed(&Decision::Update { + runtime_key: "release-wheel-gfx94x-dcgpu-7-13-0".to_owned() + })); + } + #[test] fn no_managed_runtime_installs() { assert_eq!(decide(EMPTY, "release"), Decision::Install);