From db42fd2f50eb1b216ca6d2bb77642310e1f041f3 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Mon, 17 Aug 2026 13:01:14 +0000 Subject: [PATCH 1/4] fix(therock): resolve ROCm releases from the current multi-arch pip index repo.amd.com/rocm/whl-multi-arch is AMD's current release index and is where 7.14.0+ is published, but therock_index_urls() appended a per-family path segment to it (a scheme only the older whl/{family} index uses), so the request always 403'd and the CLI silently fell back to the stale classic index topping out at 7.13.0. Try the multi-arch index (flat, no per-family path) first, keeping the classic per-family index as the second candidate so older families and explicit version pins that only exist there keep resolving exactly as before. The multi-arch rocm sdist also needs an explicit device-* extra to pull in a GPU backend, so requests to it now add device-gfxNNNN extras for the families with an exact enumerable chip set, or device-all for the remaining prefix-bucket families. Signed-off-by: Juho Vainio --- apps/rocm/src/therock.rs | 101 ++++++++++++++++++++++++++++++++---- crates/rocm-core/src/lib.rs | 35 +++++++++++++ 2 files changed, 127 insertions(+), 9 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index cf5fcb7e1..8d8b137fc 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -6,7 +6,8 @@ use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, ManagedToolConfig, RocmCliConfig, detect_host_gpu_diagnostics, detect_host_therock_family, detect_managed_therock_family, disk_space, ensure_uv_binary, - known_therock_families, managed_tools_dir, normalize_runtime_path_for_host, + known_therock_families, known_therock_family_device_chips, managed_tools_dir, + normalize_runtime_path_for_host, normalize_runtime_path_for_storage, normalize_runtime_path_text_for_host, normalize_runtime_path_text_for_storage, normalize_therock_family, runtime_is_windows, runtime_os_name, runtime_path_for_windows_child, runtime_path_list_split, @@ -826,6 +827,7 @@ fn install_wheel_runtime( "Found TheRock package family {} version {} with a matching PyTorch stack.", resolution.family, resolution.latest_version )); + let rocm_extras = therock_rocm_extras(&resolution.family, &resolution.index_url); let runtime_key = runtime_key( channel, "wheel", @@ -880,7 +882,7 @@ fn install_wheel_runtime( let _ = writeln!( output, " package_specs: {}", - therock_pip_package_specs(&resolution.package_versions).join(" ") + therock_pip_package_specs(&resolution.package_versions, &rocm_extras).join(" ") ); let _ = writeln!( output, @@ -893,7 +895,10 @@ fn install_wheel_runtime( if matches!(channel, TheRockChannel::Nightly) { install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); } - install_args.extend(therock_pip_package_specs(&resolution.package_versions)); + install_args.extend(therock_pip_package_specs( + &resolution.package_versions, + &rocm_extras, + )); let venv_args = uv_venv_args(&python_launcher.executable, &install_root); let venv_args_display = venv_args .iter() @@ -933,7 +938,7 @@ fn install_wheel_runtime( progress_line(format!( "Installing {} from {}", - therock_pip_package_specs(&resolution.package_versions).join(" "), + therock_pip_package_specs(&resolution.package_versions, &rocm_extras).join(" "), resolution.index_url )); let mut install_args = uv_pip_install_base(&env_python); @@ -941,7 +946,10 @@ fn install_wheel_runtime( if matches!(channel, TheRockChannel::Nightly) { install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); } - install_args.extend(therock_pip_package_specs(&resolution.package_versions)); + install_args.extend(therock_pip_package_specs( + &resolution.package_versions, + &rocm_extras, + )); run_uv_progress_command( paths, &uv, @@ -1012,15 +1020,42 @@ fn install_wheel_runtime( Ok(output) } -fn therock_pip_package_specs(package_versions: &TheRockPipPackageVersions) -> Vec { +fn therock_pip_package_specs( + package_versions: &TheRockPipPackageVersions, + rocm_extras: &str, +) -> Vec { vec![ - format!("rocm[libraries,devel]=={}", package_versions.rocm), + format!("rocm[{rocm_extras}]=={}", package_versions.rocm), format!("torch=={}", package_versions.torch), format!("torchvision=={}", package_versions.torchvision), format!("torchaudio=={}", package_versions.torchaudio), ] } +fn is_multi_arch_pip_index(index_url: &str) -> bool { + index_url.trim_end_matches('/') == THEROCK_RELEASE_PIP_MULTI_ARCH_INDEX_BASE +} + +/// The `rocm[...]` extras to request for a resolved pip index. The classic +/// per-family index needs only the base `libraries,devel` extras; the flat +/// multi-arch index additionally needs an explicit `device-*` extra or no GPU +/// backend gets installed at all. +fn therock_rocm_extras(family: &str, index_url: &str) -> String { + let mut extras = "libraries,devel".to_owned(); + if !is_multi_arch_pip_index(index_url) { + return extras; + } + match known_therock_family_device_chips(family) { + Some(chips) => { + for chip in chips { + let _ = write!(extras, ",device-{chip}"); + } + } + None => extras.push_str(",device-all"), + } + extras +} + fn quote_display_arg(value: &str) -> String { if value.is_empty() || value @@ -3368,9 +3403,12 @@ fn parse_version(value: &str) -> Option { fn therock_index_urls(channel: TheRockChannel, family: &str) -> Vec { match channel { + // Multi-arch is flat (no per-family path segment) and is where AMD + // publishes current releases; try it first. The classic per-family + // index is kept as a fallback so older releases stay installable. TheRockChannel::Release => vec![ + THEROCK_RELEASE_PIP_MULTI_ARCH_INDEX_BASE.to_owned(), format!("{THEROCK_RELEASE_PIP_INDEX_BASE}/{family}"), - format!("{THEROCK_RELEASE_PIP_MULTI_ARCH_INDEX_BASE}/{family}"), ], TheRockChannel::Nightly => vec![format!("{THEROCK_NIGHTLY_PIP_INDEX_BASE}/{family}")], } @@ -4065,7 +4103,7 @@ mod tests { torchaudio: "2.10.0+rocm7.13.0a20260513".to_owned(), compatibility_key: "7.13.0a20260513".to_owned(), }; - let package_specs = therock_pip_package_specs(&package_versions); + let package_specs = therock_pip_package_specs(&package_versions, "libraries,devel"); assert_eq!( package_specs, @@ -4078,6 +4116,51 @@ mod tests { ); } + #[test] + fn therock_index_urls_prefers_multi_arch_then_classic_for_release() { + let urls = therock_index_urls(TheRockChannel::Release, "gfx110X-all"); + assert_eq!( + urls, + vec![ + "https://repo.amd.com/rocm/whl-multi-arch".to_owned(), + "https://repo.amd.com/rocm/whl/gfx110X-all".to_owned(), + ] + ); + } + + #[test] + fn therock_index_urls_nightly_unchanged() { + let urls = therock_index_urls(TheRockChannel::Nightly, "gfx110X-all"); + assert_eq!( + urls, + vec!["https://rocm.nightlies.amd.com/v2/gfx110X-all".to_owned()] + ); + } + + #[test] + fn therock_rocm_extras_classic_index_is_unchanged() { + assert_eq!( + therock_rocm_extras("gfx110X-all", "https://repo.amd.com/rocm/whl/gfx110X-all"), + "libraries,devel" + ); + } + + #[test] + fn therock_rocm_extras_multi_arch_adds_exact_device_chips() { + assert_eq!( + therock_rocm_extras("gfx110X-all", "https://repo.amd.com/rocm/whl-multi-arch"), + "libraries,devel,device-gfx1100,device-gfx1101,device-gfx1102,device-gfx1103" + ); + } + + #[test] + fn therock_rocm_extras_multi_arch_falls_back_to_device_all_for_ambiguous_bucket() { + assert_eq!( + therock_rocm_extras("gfx90X-dcgpu", "https://repo.amd.com/rocm/whl-multi-arch"), + "libraries,devel,device-all" + ); + } + /// The downloaded archive is removed once it has been unpacked; keeping it /// would double the disk cost of every installed SDK version. #[test] diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index e0369ecab..5957e0b74 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -3969,6 +3969,26 @@ pub const fn known_therock_families() -> &'static [&'static str] { ] } +/// Exact chip IDs covered by a [`known_therock_families`] bucket. +/// +/// `None` for the prefix catch-all buckets (e.g. `gfx90X-dgpu`) whose exact +/// chip membership isn't enumerable from [`normalize_therock_family`] alone. +pub fn known_therock_family_device_chips(family: &str) -> Option<&'static [&'static str]> { + match family { + "gfx900" => Some(&["gfx900"]), + "gfx906" => Some(&["gfx906"]), + "gfx908" => Some(&["gfx908"]), + "gfx90a" => Some(&["gfx90a"]), + "gfx110X-all" => Some(&["gfx1100", "gfx1101", "gfx1102", "gfx1103"]), + "gfx1150" => Some(&["gfx1150"]), + "gfx1151" => Some(&["gfx1151"]), + "gfx1152" => Some(&["gfx1152"]), + "gfx1153" => Some(&["gfx1153"]), + "gfx120X-all" => Some(&["gfx1200", "gfx1201"]), + _ => None, + } +} + fn capture_optional_command(program: &str, args: &[&str]) -> Option { capture_optional_command_with_timeout(program, args, OPTIONAL_COMMAND_TIMEOUT) } @@ -8897,6 +8917,21 @@ mod tests { assert!(!known_therock_families().is_empty()); } + #[test] + fn known_therock_family_device_chips_round_trip_to_their_family() { + for family in known_therock_families() { + if let Some(chips) = known_therock_family_device_chips(family) { + for chip in chips { + assert_eq!( + normalize_therock_family(chip).as_deref(), + Some(*family), + "chip `{chip}` must normalize back to family `{family}`" + ); + } + } + } + } + #[test] fn preferred_serve_engine_uses_vllm_for_supported_therock_families() { assert_eq!( From 8c62a8ba35bc8c8d5a5fc43a860eb917ef510988 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Mon, 17 Aug 2026 15:28:40 +0000 Subject: [PATCH 2/4] test(e2e): guard against the broken multi-arch pip index URL shape Adds a GPU-free e2e scenario that dry-runs `install sdk --family gfx110X-all` and asserts the resolved index_url never has the /whl-multi-arch/{family}/ shape that 403s and used to mask the fix by silently falling through to the stale classic index every time. Signed-off-by: Juho Vainio --- .../features/runtime_setup.feature | 12 ++++++++++++ tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index b66cb92e8..d2a44d31a 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -84,3 +84,15 @@ Feature: Runtime configuration When the user tries to adopt the existing install Then the adoption is refused And the error explains which install types can be adopted + + # Regression guard for the bug where the release-channel multi-arch pip index + # was queried with a per-family path segment (.../whl-multi-arch/{family}/), + # which 403s because that index is flat and 404/403-ed straight into the + # stale classic index every time — silently pinning every install to + # whatever version predated the migration. `--family` bypasses GPU + # auto-detection so this needs no GPU, and `--dry-run` resolves the real + # index without installing anything. + @id:runtime-install-sdk-release-index-shape + Scenario: 4 - Resolving the SDK from the release channel never uses the broken multi-arch URL shape + When the user dry-runs installing the SDK for a known family + Then the resolved package index is not the broken per-family multi-arch path diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index caae41986..7743692c2 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -202,6 +202,15 @@ fn assert_engine_ready(world: &mut E2eWorld) { ); } +#[when("the user dry-runs installing the SDK for a known family")] +async fn user_dry_runs_install_sdk_for_family(world: &mut E2eWorld) { + let stdout = crate::run_rocm_ok( + world, + &["install", "sdk", "--family", "gfx110X-all", "--dry-run"], + ); + world.cli_output = Some(stdout); +} + #[when("the user tries to adopt the existing install")] async fn user_tries_adopt(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm( @@ -365,6 +374,15 @@ async fn assert_update_reports_freshness(world: &mut E2eWorld) { } } +#[then("the resolved package index is not the broken per-family multi-arch path")] +async fn assert_index_not_broken_multi_arch(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().expect("no dry-run output"); + assert!( + !stdout.contains("whl-multi-arch/gfx110X-all"), + "dry-run resolved the broken per-family multi-arch index shape:\n{stdout}" + ); +} + #[then("the adoption is refused")] async fn assert_adoption_refused(world: &mut E2eWorld) { let rc = world.cli_rc.expect("no command was run"); From 50ecdbb4dc64db1254d13d59413ea5c6e6a8b01f Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Mon, 31 Aug 2026 14:35:12 +0000 Subject: [PATCH 3/4] fix: address PR #272 review feedback Tag the new release-index-shape dry-run scenario @nightly (same eai-7960 timing-fragility reason as the sibling scenario), and clarify the package_policy dry-run prose to mention resolved device extras instead of implying gfx-specific wheels always ship in the pip specs. Signed-off-by: Juho Vainio --- apps/rocm/src/therock.rs | 2 +- tests/e2e-cucumber/features/runtime_setup.feature | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 8d8b137fc..e2311d53e 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -886,7 +886,7 @@ fn install_wheel_runtime( ); let _ = writeln!( output, - " package_policy: find the newest TheRock ROCm SDK version that has a matching PyTorch stack in the same index, then install pinned rocm[libraries,devel], torch, torchvision, and torchaudio versions in one uv transaction" + " package_policy: find the newest TheRock ROCm SDK version that has a matching PyTorch stack in the same index, then install pinned rocm (with any resolved device extras), torch, torchvision, and torchaudio versions in one uv transaction" ); if dry_run { let env_python = venv_python_path(&install_root); diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index d2a44d31a..14a5308bd 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -88,11 +88,19 @@ Feature: Runtime configuration # Regression guard for the bug where the release-channel multi-arch pip index # was queried with a per-family path segment (.../whl-multi-arch/{family}/), # which 403s because that index is flat and 404/403-ed straight into the - # stale classic index every time — silently pinning every install to + # stale classic index every time, silently pinning every install to # whatever version predated the migration. `--family` bypasses GPU # auto-detection so this needs no GPU, and `--dry-run` resolves the real # index without installing anything. - @id:runtime-install-sdk-release-index-shape + # + # `@nightly` for the same reason as scenario 8 above: this dry-run still + # resolves the real channel index over the network (dry-run only skips the + # venv/download, not index resolution), and the no-GPU mock lane's 64-way + # concurrency from that extra network work is what pushes + # `eai-7960-gen-tps-held-after-scrape-failure` and + # `eai-7960-gen-tps-expiry-boundary` past their validity window. Runs on the + # nightly lanes instead, where scenarios are serialized. + @id:runtime-install-sdk-release-index-shape @nightly Scenario: 4 - Resolving the SDK from the release channel never uses the broken multi-arch URL shape When the user dry-runs installing the SDK for a known family Then the resolved package index is not the broken per-family multi-arch path From 68934fa6d74b8bee8512a4e0cfec15531fdce65f Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Mon, 31 Aug 2026 15:03:15 +0000 Subject: [PATCH 4/4] style: run cargo fmt on therock.rs import list Signed-off-by: Juho Vainio --- apps/rocm/src/therock.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index e2311d53e..316c6d24f 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -7,12 +7,11 @@ use rocm_core::{ AppPaths, ManagedToolConfig, RocmCliConfig, detect_host_gpu_diagnostics, detect_host_therock_family, detect_managed_therock_family, disk_space, ensure_uv_binary, known_therock_families, known_therock_family_device_chips, managed_tools_dir, - normalize_runtime_path_for_host, - normalize_runtime_path_for_storage, normalize_runtime_path_text_for_host, - normalize_runtime_path_text_for_storage, normalize_therock_family, runtime_is_windows, - runtime_os_name, runtime_path_for_windows_child, runtime_path_list_split, - runtime_python_executable_in_env, unix_time_millis, uv_command_env, uv_pip_install_base, - uv_venv_args, verify_rsa_pkcs1_sha256_signature, + normalize_runtime_path_for_host, normalize_runtime_path_for_storage, + normalize_runtime_path_text_for_host, normalize_runtime_path_text_for_storage, + normalize_therock_family, runtime_is_windows, runtime_os_name, runtime_path_for_windows_child, + runtime_path_list_split, runtime_python_executable_in_env, unix_time_millis, uv_command_env, + uv_pip_install_base, uv_venv_args, verify_rsa_pkcs1_sha256_signature, }; #[cfg(test)] use rocm_core::{