feat(diagnose): report a code object manager that is not the runtime's own

crates/rocm-core/src/fix.rs

@@ -22,6 +22,13 @@ use std::time::Duration;
 const RUN_TIMEOUT: Duration = Duration::from_mins(1);
 const QUERY_TIMEOUT: Duration = Duration::from_secs(8);
 
+/// Shared verbatim by this recipe's own note and `diagnose.rs`'s
+/// `check_18_comgr_conflict` evidence, which states the same claim in its own
+/// words at the point the real paths are known. A second statement of one
+/// claim is a second thing to keep correct, and the two had already drifted
+/// apart in wording before they shared this constant.
+pub(crate) const COMGR_CONFLICT_NEITHER_OPTION_RECOMMENDED: &str = "Neither option is recommended over the other: which is right depends on which stack you mean to keep, and removing the wrong one breaks a working environment.";
+
 /// Print a failure explanation to stderr, ignoring write failures (closed
 /// stderr, full disk) so an I/O error while explaining a failure can't itself
 /// panic the process.
@@ -585,6 +592,39 @@ const RECIPES: &[FixRecipe] = &[
         applies_on: WSL_ONLY,
         runner: None,
     },
+    FixRecipe {
+        fix_id: "fix-18-comgr-conflict",
+        title: "Code object manager library does not belong to the active HIP runtime",
+        rationale: "HIP compiles device code at run time through libamd_comgr, and this machine holds more than one copy of it. The copy the loader picks belongs to a different installation than the HIP runtime that loads, so compilation fails with an error that names neither the library nor the second copy. A second copy is not itself a fault -- many correct installations hold one -- so what is reported here is specifically the mismatch.",
+        auto_applicable: false,
+        // No repair, and no recommendation between the two. Removing a stack or
+        // reordering the search path can each break a working Python
+        // environment, and which is right depends on which stack the user means
+        // to keep -- a question only they can answer. `rocm diagnose` fills in
+        // the real paths for this machine; these are the shapes.
+        commands: &[
+            "# Find every copy and which one loads:",
+            "rocm examine --json    # read comgr_paths, comgr_selected, hip_selected",
+            "# Then pick ONE of the following. They are alternatives, not steps.",
+            "# (a) Keep the system installation: remove or uninstall the wheel that",
+            "#     supplies the second copy.",
+            "# (b) Keep the wheel: order the search path so the wheel's own copy of",
+            "#     both libraries is found first, making the wheel the active runtime.",
+            "export LD_LIBRARY_PATH=\"<directory of the wheel's own copy>:$LD_LIBRARY_PATH\"",
+        ],
+        needs_sudo: false,
+        needs_reboot: false,
+        needs_relogin: false,
+        verify: "python -c \"import torch; torch.zeros(1, device='cuda')\"",
+        notes: &[
+            COMGR_CONFLICT_NEITHER_OPTION_RECOMMENDED,
+            "This describes the environment outside the CLI's managed runtimes. `rocm serve` puts a managed runtime's libraries first on purpose, so inside one the wheel copy wins by design and that is correct.",
+        ],
+        // Not `LINUX_ONLY`: which copy the loader picks has nothing to do with
+        // the amdgpu module, and the two copies collide on WSL2 just the same.
+        applies_on: LINUX_AND_WSL,
+        runner: None,
+    },
     FixRecipe {
         fix_id: "fix-19-shm-too-small",
         title: "Raise the shared memory allowance",
@@ -1766,9 +1806,9 @@ mod tests {
         let count = ids.len();
         ids.dedup();
         assert_eq!(ids.len(), count, "duplicate fix-id in RECIPES");
-        // 17 bare-metal/Windows entries (fix-17 and fix-19 among them) plus the
-        // 7 WSL ones.
-        assert_eq!(count, 24, "expected 24 catalog entries");
+        // 18 bare-metal/Windows entries (fix-17, fix-18 and fix-19 among them)
+        // plus the 7 WSL ones.
+        assert_eq!(count, 25, "expected 25 catalog entries");
     }
 
     #[test]

crates/rocm-core/src/lib.rs

@@ -4146,6 +4146,7 @@ pub(crate) fn managed_therock_sdk_probe_candidates(
             site_packages: sdk.site_packages,
             root_path,
             bin_path,
+            library_paths: sdk.library_paths,
         });
     }
     candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.installed_at_unix_ms));
@@ -4340,6 +4341,14 @@ pub(crate) struct TheRockSdkProbeCandidate {
     pub(crate) site_packages: Option<PathBuf>,
     pub(crate) root_path: PathBuf,
     bin_path: PathBuf,
+    /// The SDK's own recorded library directories -- every package root the
+    /// probe script actually imported and asked Python for (see
+    /// `ROCM_SDK_PROBE_SCRIPT`'s `add_runtime_root`), not a layout guessed from
+    /// `root_path`/`site_packages` after the fact. This is what
+    /// `probe_runtime_devices` puts on `LD_LIBRARY_PATH` for a served process;
+    /// a caller that needs to find a managed runtime's actual libraries (comgr
+    /// included) should prefer this over re-deriving the layout.
+    pub(crate) library_paths: Vec<PathBuf>,
 }
 
 pub fn detect_host_gfx_target() -> Option<String> {
@@ -10566,6 +10575,57 @@ Class Name:                Display
         Ok(())
     }
 
+    /// `managed_therock_sdk_probe_candidates` surfaces the SDK's own recorded
+    /// `library_paths` rather than dropping them.
+    ///
+    /// Those are what `examine`'s comgr/HIP search now reads to find a managed
+    /// runtime's libraries (see `known_install_roots`/`install_library_dirs` in
+    /// `examine.rs`), in place of re-deriving the layout from `root_path` and
+    /// `site_packages` after the fact -- a guess that does not hold for every
+    /// real install shape, which is what left a managed runtime's own code
+    /// object manager library unseen on a real host
+    /// (`examine-finds-the-managed-runtimes-own-compilation-library`). A
+    /// candidate whose `library_paths` came back empty would defeat that fix
+    /// silently, so this pins the field surviving the read.
+    #[test]
+    fn managed_sdk_probe_candidate_carries_recorded_library_paths() -> Result<()> {
+        let (root, paths) = temp_app_paths("managed-sdk-library-paths");
+        let registry = paths.data_dir.join("runtimes").join("registry");
+        let site_packages = root.join("site-packages");
+        let sdk_root = site_packages.join("_rocm_sdk_devel");
+        let sdk_bin = sdk_root.join("bin");
+        let comgr_dir = site_packages.join("_rocm_sdk_core").join("lib");
+        fs::create_dir_all(&sdk_bin)?;
+        fs::create_dir_all(&comgr_dir)?;
+        fs::create_dir_all(&registry)?;
+        fs::write(
+            registry.join("runtime.json"),
+            serde_json::to_vec_pretty(&serde_json::json!({
+                "runtime_id": "therock-release:gfx120X-all",
+                "family": "gfx120X-all",
+                "installed_at_unix_ms": 10,
+                "rocm_sdk": {
+                    "import_ok": true,
+                    "site_packages": site_packages,
+                    "root_path": sdk_root,
+                    "bin_path": sdk_bin,
+                    "library_paths": [comgr_dir]
+                }
+            }))?,
+        )?;
+
+        let candidates = managed_therock_sdk_probe_candidates(&registry);
+        assert_eq!(candidates.len(), 1, "expected exactly one candidate");
+        assert_eq!(
+            candidates[0].library_paths,
+            vec![comgr_dir],
+            "the recorded library_paths must survive into the candidate, or the \
+             comgr/HIP search has nowhere else reliable to find them"
+        );
+        fs::remove_dir_all(root).ok();
+        Ok(())
+    }
+
     #[test]
     fn managed_sdk_probe_skips_non_therock_manifests() -> Result<()> {
         let (root, paths) = temp_app_paths("managed-sdk-skip-non-therock");

docs/wsl.md

@@ -200,13 +200,18 @@ The WSL entries, in the order a broken stack usually reveals them:
 | `fix-wsl-5-distro-too-old` | The distro release is below the floor in the prerequisites above |
 | `fix-wsl-6-host-driver-too-old` | The distro-side plumbing is complete but the Windows host driver is missing or too old |
 
-One entry outside this list also applies here. `fix-19-shm-too-small` is not a
+Two entries outside this list also apply here. `fix-19-shm-too-small` is not a
 WSL entry, but WSL2 ships the same 64 MiB `/dev/shm` a container does, and a
 serving workload needs gigabytes of it. It reports below 1 GiB, so a WSL2 user
 can meet it on an otherwise healthy stack. Its guidance names the container and
 bare-metal cases; under WSL2 the remedy is the host one, remounting `/dev/shm`
 larger and adding the matching `/etc/fstab` line inside the distro.
 
+`fix-18-comgr-conflict` also applies here: a wheel copy and a system copy of
+the code object manager library collide on WSL2 exactly as they do on bare
+metal, through the same `LD_LIBRARY_PATH`/loader-cache search, so the finding
+is not specific to either platform.
+
 Every WSL remedy is print-only. `rocm fix <id>` shows the commands and does not
 run them: they either install packages with `sudo`, edit loader configuration, or
 belong to the Windows host, and none of that meets the bar the four

skills/rocm-doctor/SKILL.md

@@ -145,7 +145,7 @@ passes — the GPU is AMD. Linux, Windows and WSL2 all run the same workflow.
    rocm fix <fix-id> --yes      # required to apply in a non-interactive shell
    ```
 
-   Only the four auto-applicable fixes are ones the CLI runs itself. The other 20
+   Only the four auto-applicable fixes are ones the CLI runs itself. The other 21
    are **print-only** (bootloader, kernel, reinstall, Windows driver, …): `rocm
    fix <id>` just prints the plan for the user to run themselves — no prompt, and
    the CLI never performs those.

skills/rocm-doctor/reference.md

@@ -74,7 +74,7 @@ non-interactive shell without `--yes`, and confirm first. **Two exceptions:**
   pinned. Run `rocm fix fix-9-igpu-dgpu --device-index N` (not the bare id)
   once you know N.
 
-## Closed catalog (24 failure modes)
+## Closed catalog (25 failure modes)
 
 The OS column is the platform family the CLI scopes an entry to, and WSL2 is a
 family of its own — not a flavour of `linux`. An entry reaches a WSL host only
@@ -100,6 +100,7 @@ reporting confident nonsense.
 | `fix-14-adrenalin-too-old` | windows | Adrenalin / kernel-mode driver too old for the HIP SDK | `hipInfo` can't enumerate, "driver too old", HSA "no agents found" | no |
 | `fix-15-msvc-redist` | windows | MSVC runtime missing (HIP DLLs can't load) | `vcruntime140.dll` / `vcruntime140_1.dll` missing | no |
 | `fix-17-torch-dlpack` | linux | `torch-c-dlpack-ext` loads its CUDA prebuilt on a ROCm torch, aborting vLLM's engine start at import time | vLLM engine start fails on import; error names `torch_c_dlpack_ext` or tvm_ffi's `_optional_torch_c_dlpack` | no |
+| `fix-18-comgr-conflict` | linux/wsl | The code object manager library (`libamd_comgr`) that would load belongs to a different installation than the HIP runtime that would load, so device code compilation fails with an error naming neither | compilation error naming neither library; `rocm examine --json`'s `comgr_selected`/`hip_selected` resolve to two different `install_root`s | no |
 | `fix-19-shm-too-small` | linux/wsl | `/dev/shm` too small for a serving workload, which needs gigabytes where a container and WSL2 both default to 64 MB | reported under 1 GiB; a data-loader worker killed by a bus error, or a failed write to a temporary file, with nothing naming shared memory | no |
 | `fix-wsl-1-gpu-not-exposed` | wsl | `/dev/dxg` absent, so the distro cannot reach the GPU at all | no `/dev/dxg`; in a container, the device was never passed through | no |
 | `fix-wsl-2-dxcore-missing` | wsl | `/usr/lib/wsl/lib` DXCore shims missing, so the runtime cannot reach the host driver | `/usr/lib/wsl/lib/libdxcore.so` missing (or the directory absent entirely) | no |
@@ -116,7 +117,7 @@ they answer for a different platform family.
 
 Linux-only: fix-3, -4, -5, -7, -10, -11, -12, -17. Windows-only: fix-13, -14, -15.
 WSL-only: fix-wsl-1 through fix-wsl-7. Linux + Windows: fix-9.
-Linux + WSL: fix-19. Linux + Windows + WSL: fix-1, -2, -6, -8.
+Linux + WSL: fix-18, -19. Linux + Windows + WSL: fix-1, -2, -6, -8.
 
 ## Framework routing
 

tests/e2e-cucumber/features/diagnose.feature

@@ -289,3 +289,29 @@ Feature: Diagnosing failures and listing fixes
     When the user previews that fix without applying it
     Then the preview states that the fix requires sudo and a re-login
     And the preview states that the CLI can run it automatically
+  # HIP compiles device code at run time through a library a machine can hold
+  # more than one copy of. When the copy that loads belongs to a different
+  # installation than the runtime, compilation fails with an error naming
+  # neither. Both remedies — remove one stack, or reorder the search path — can
+  # break a working Python environment, and which is right depends on which
+  # stack the user means to keep. So the CLI states them and changes nothing.
+  #
+  # The conflict itself cannot be provoked here: the suite cannot install a
+  # second ROCm stack, and the detection rule is proven by unit tests that build
+  # the machine state directly. What this pins is the half that matters if the
+  # entry ever stops being advisory — that asking for it changes nothing and
+  # recommends neither option.
+  #
+  # `@requires-os:linux` because `fix-18-comgr-conflict` is registered for
+  # `["linux", "wsl"]` (comgr and LD_LIBRARY_PATH are POSIX-loader concepts, not
+  # Windows ones). Unlike diagnose-20's preview, this step applies the fix for
+  # real, so it goes through the fix's own platform gate and would be refused
+  # for the wrong reason -- "wrong OS", not "advisory" -- on a native Windows
+  # lane. `@requires-os:linux` matches WSL2 too, which is where this fix does
+  # apply.
+  @id:diagnose-fix-comgr-conflict-is-advisory-only @requires-os:linux
+  Scenario: diagnose-21 - The fix for a shadowed compilation library changes nothing and recommends nothing
+    Given a user who has chosen the fix for a shadowed compilation library
+    When the user asks the CLI to apply that fix
+    Then the CLI explains that it will not make the change itself
+    And the CLI offers both options without ranking them

tests/e2e-cucumber/features/examine.feature

@@ -247,6 +247,8 @@ Feature: GPU detection and system inspection
     When the user inspects the system in machine-readable form
     Then the inspection lists the code object manager libraries it found
     And it names which of them would load, or says it found none
+    And it lists the HIP runtime libraries the machine holds the same way
+    And it names which HIP runtime copy would load, or says it found none
 
   # The copy this CLI installs itself, which is the case the whole entry exists
   # for: the install path puts ROCm wheels into a managed environment, so a user
@@ -259,14 +261,15 @@ Feature: GPU detection and system inspection
   # described, not that it matches the one the installer actually produces. A
   # real managed runtime is the only thing that distinguishes those.
   #
-  # `@requires-os:linux` because `probe_comgr` only ever looks for
-  # `libamd_comgr` -- an ELF shared-object name, found via `LD_LIBRARY_PATH`,
+  # `@requires-os:linux` because `probe_comgr` only ever looks for `libamd_comgr`
+  # and `libamdhip64` -- ELF shared-object names, found via `LD_LIBRARY_PATH`,
   # the loader cache, or an install root's `lib/` tree. None of that exists on
-  # native Windows, which ships the equivalent library under a different name,
-  # so the assertion that a managed runtime's copy must be found does not hold
-  # there. WSL reports `os_family` "linux" (see `expectation.rs`), so this
-  # still runs on the WSL lane, where the managed runtime really does carry a
-  # `.so`.
+  # native Windows, which ships `.dll`s under other names, so the assertion
+  # that a managed runtime's library must be found does not hold there. WSL
+  # reports `os_family` "linux" (see `expectation.rs`), so this still runs on
+  # the WSL lane, where the managed runtime really does carry a `.so`. This
+  # matches `check_18_comgr_conflict`'s own `&["linux", "wsl"]` gate in
+  # diagnose.rs -- the same boundary, stated once there and once here.
   @id:examine-finds-the-managed-runtimes-own-compilation-library @requires-gpu @requires-os:linux
   Scenario: examine-19 - The inspection finds the compilation library the CLI installed itself
     Given a managed runtime is active

tests/e2e-cucumber/tests/e2e/diagnose_steps.rs

@@ -88,8 +88,7 @@ const CATALOG_FIX_IDS: &[&str] = &[
     "fix-wsl-5-distro-too-old",
     "fix-wsl-6-host-driver-too-old",
     "fix-wsl-7-wsl1",
-    // `fix-18` is the code object manager entry on its own branch, kept
-    // distinct for the same reason `fix-16` is.
+    "fix-18-comgr-conflict",
     "fix-19-shm-too-small",
 ];
 
@@ -137,6 +136,11 @@ const fn fix_id_for_the_other_os() -> &'static str {
     }
 }
 
+/// The entry for a code object manager library belonging to a different
+/// installation than the active runtime. Advisory by design: both remedies can
+/// break a working Python environment.
+const COMGR_CONFLICT_FIX_ID: &str = "fix-18-comgr-conflict";
+
 /// Contents planted in the scenario's own shell rc file. The assertion is that
 /// this survives the run byte for byte.
 const PLANTED_RC: &str = "# planted by the e2e suite; the fix must not touch this\n";
@@ -206,6 +210,11 @@ async fn user_chose_fix_needing_sudo_and_relogin(world: &mut E2eWorld) {
     world.model_name = Some(COMMAND_FAILURE_FIX_ID.to_string());
 }
 
+#[given("a user who has chosen the fix for a shadowed compilation library")]
+async fn user_chose_comgr_conflict_fix(world: &mut E2eWorld) {
+    world.model_name = Some(COMGR_CONFLICT_FIX_ID.to_string());
+}
+
 #[given("a user who names a fix the CLI does not offer")]
 async fn user_named_unknown_fix(world: &mut E2eWorld) {
     world.model_name = Some("fix-does-not-exist".to_string());
@@ -1133,3 +1142,41 @@ async fn assert_command_failure_reported_on_stderr(world: &mut E2eWorld) {
         "the command-failure explanation must not also be on stdout:\n{stdout}"
     );
 }
+
+#[then("the CLI explains that it will not make the change itself")]
+async fn assert_fix_is_advisory(world: &mut E2eWorld) {
+    let output = world.cli_output.as_ref().expect("no fix output");
+    assert_eq!(
+        world.cli_rc,
+        Some(0),
+        "printing advice is not a failure:\n{output}"
+    );
+    assert!(
+        output.contains("print-only") || output.contains("will NOT run it"),
+        "the user has to be told the CLI is not going to do this for them:\n{output}"
+    );
+}
+
+#[then("the CLI offers both options without ranking them")]
+async fn assert_both_options_unranked(world: &mut E2eWorld) {
+    let output = world.cli_output.as_ref().expect("no fix output");
+    // Both remedies have to be present. Offering one is a recommendation by
+    // omission, and the wrong one breaks a working environment.
+    //
+    // Keyed on the option markers rather than on words like "remove", which also
+    // occur in the surrounding prose -- an assertion that matched those would
+    // still pass with one of the two options deleted, which is exactly the
+    // regression it exists to catch.
+    for option in ["(a)", "(b)"] {
+        assert!(
+            output.contains(option),
+            "only one way out was offered; option `{option}` is missing, which makes \
+             the other a recommendation by omission:\n{output}"
+        );
+    }
+    assert!(
+        output.contains("Neither option is recommended"),
+        "the CLI has to say it is not choosing between them -- which is right \
+         depends on which stack the user means to keep:\n{output}"
+    );
+}