From e263ceabd5ca47fc9d8150d0428e7bf4e4c1a2e9 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Tue, 7 Jul 2026 13:56:34 -0500 Subject: [PATCH 1/7] feat: Add rack-aware machine-a-tron simulation --- Cargo.lock | 2 + crates/api-integration-tests/Cargo.toml | 1 + crates/api-integration-tests/tests/lib.rs | 100 +++++++++++++++- crates/api-test-helper/src/api_server.rs | 14 +++ crates/api-test-helper/src/machine_a_tron.rs | 2 + crates/machine-a-tron/Cargo.toml | 1 + crates/machine-a-tron/config/mat.toml | 7 ++ crates/machine-a-tron/src/api_client.rs | 54 ++++++++- crates/machine-a-tron/src/config.rs | 113 ++++++++++++++++++- crates/machine-a-tron/src/lib.rs | 2 +- crates/machine-a-tron/src/machine_a_tron.rs | 40 ++++--- crates/machine-a-tron/src/main.rs | 1 + 12 files changed, 310 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b54028261..4a6f5348b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1357,6 +1357,7 @@ dependencies = [ "carbide-machine-a-tron", "carbide-secrets", "carbide-utils", + "carbide-uuid", "ctor", "eyre", "futures", @@ -2264,6 +2265,7 @@ dependencies = [ "bmc-mock", "carbide-network", "carbide-rpc", + "carbide-test-support", "carbide-tls", "carbide-utils", "carbide-uuid", diff --git a/crates/api-integration-tests/Cargo.toml b/crates/api-integration-tests/Cargo.toml index a46b27f72e..215a191da8 100644 --- a/crates/api-integration-tests/Cargo.toml +++ b/crates/api-integration-tests/Cargo.toml @@ -26,6 +26,7 @@ authors.workspace = true # [local-dependencies] # DO NOT PUT DEPENDENCIES OTHER THAN LOCAL DEPS HERE, THEY SHOULD ALL HAVE 'path =' IN THEM. carbide-machine-a-tron = { path = "../machine-a-tron" } +carbide-uuid = { path = "../uuid" } carbide-utils = { path = "../utils", features = ["sqlx"] } carbide-api-test-helper = { path = "../api-test-helper" } carbide-kms-provider = { path = "../kms-provider" } diff --git a/crates/api-integration-tests/tests/lib.rs b/crates/api-integration-tests/tests/lib.rs index c9d45fb943..c2fdf4a543 100644 --- a/crates/api-integration-tests/tests/lib.rs +++ b/crates/api-integration-tests/tests/lib.rs @@ -22,13 +22,16 @@ use std::sync::Arc; use std::time::{self, Duration}; use ::carbide_utils::HostPortPair; -use ::machine_a_tron::{BmcMockRegistry, HostMachineHandle, MachineATronConfig, MachineConfig}; +use ::machine_a_tron::{ + BmcMockRegistry, HostMachineHandle, MachineATronConfig, MachineConfig, RackConfig, +}; use api_test_helper::{ IntegrationTestEnvironment, domain, instance, machine, metrics, subnet, tenant, utils, vpc, vpc_prefix, }; use bmc_mock::test_support::TEST_MAC_POOL; use bmc_mock::{HostHardwareType, ListenerOrAddress}; +use carbide_uuid::rack::{RackId, RackProfileId}; use eyre::ContextCompat; use futures::FutureExt; use futures::future::join_all; @@ -158,6 +161,12 @@ async fn test_integration() -> eyre::Result<()> { Ipv4Addr::new(172, 20, 0, 2), ) .boxed(), + test_machine_a_tron_rack( + &test_env, + &bmc_address_registry, + Ipv4Addr::new(172, 20, 0, 2), + ) + .boxed(), test_machine_a_tron_zerodpu( HostHardwareType::DellPowerEdgeR750, &test_env, @@ -415,6 +424,7 @@ async fn test_metrics_integration() -> eyre::Result<()> { 1, 1, false, + None, &test_env, &bmc_address_registry, Ipv4Addr::new(172, 20, 0, 1), @@ -562,6 +572,7 @@ async fn test_machine_a_tron_multidpu( 1, 2, false, + None, test_env, bmc_mock_registry, admin_dhcp_relay_address, @@ -633,6 +644,71 @@ async fn test_machine_a_tron_multidpu( .await } +#[derive(Clone)] +struct TestRackConfig { + rack_id: RackId, + rack_profile_id: RackProfileId, +} + +async fn test_machine_a_tron_rack( + test_env: &IntegrationTestEnvironment, + bmc_mock_registry: &BmcMockRegistry, + admin_dhcp_relay_address: Ipv4Addr, +) -> eyre::Result<()> { + let rack_id = RackId::new("machine-a-tron-nvl72"); + run_machine_a_tron_test( + HostHardwareType::WiwynnGB200Nvl, + 18, + 2, + false, + Some(TestRackConfig { + rack_id: rack_id.clone(), + rack_profile_id: RackProfileId::new("NVL72"), + }), + test_env, + bmc_mock_registry, + admin_dhcp_relay_address, + |machine_handle| { + let db_pool = test_env.db_pool.clone(); + let rack_id = rack_id.clone(); + async move { + machine_handle + .wait_until_machine_up_with_api_state("Ready", Duration::from_secs(240)) + .await?; + let machine_id = machine_handle + .observed_machine_id() + .expect("Machine ID should be set if host is ready") + .to_string(); + + let managed_rack_id: Option = + sqlx::query_scalar("SELECT rack_id FROM machines WHERE id = $1") + .bind(machine_id) + .fetch_one(&db_pool) + .await?; + assert_eq!(managed_rack_id.as_deref(), Some(rack_id.as_str())); + + let expected_machine_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM expected_machines WHERE rack_id = $1") + .bind(rack_id.as_str()) + .fetch_one(&db_pool) + .await?; + assert_eq!(expected_machine_count, 18); + + let rack_profile_id: String = sqlx::query_scalar( + "SELECT rack_profile_id FROM expected_racks WHERE rack_id = $1", + ) + .bind(rack_id.as_str()) + .fetch_one(&db_pool) + .await?; + assert_eq!(rack_profile_id, "NVL72"); + + Ok::<(), eyre::Report>(()) + } + }, + ) + .await +} + async fn test_machine_a_tron_zerodpu( hw_type: HostHardwareType, test_env: &IntegrationTestEnvironment, @@ -644,6 +720,7 @@ async fn test_machine_a_tron_zerodpu( 1, 0, false, + None, test_env, bmc_mock_registry, admin_dhcp_relay_address, @@ -704,6 +781,7 @@ async fn test_machine_a_tron_singledpu_nic_mode( 1, 1, true, + None, test_env, bmc_mock_registry, admin_dhcp_relay_address, @@ -778,6 +856,7 @@ async fn test_machine_a_tron_dpu_to_nic_mode_reregistration( // Start in DPU mode: the host comes up as a managed-DPU machine, and we // flip it to NIC mode below. false, + None, test_env, bmc_mock_registry, admin_dhcp_relay_address, @@ -943,6 +1022,7 @@ async fn test_machine_a_tron_dual_stack( 1, 1, false, + None, test_env, bmc_mock_registry, admin_dhcp_relay_address, @@ -1050,6 +1130,7 @@ async fn test_machine_a_tron_dual_stack_l2( 1, 1, false, + None, test_env, bmc_mock_registry, admin_dhcp_relay_address, @@ -1110,6 +1191,7 @@ async fn run_machine_a_tron_test( host_count: u32, dpu_per_host_count: u32, dpus_in_nic_mode: bool, + rack: Option, test_env: &IntegrationTestEnvironment, bmc_mock_registry: &BmcMockRegistry, admin_dhcp_relay_address: Ipv4Addr, @@ -1128,10 +1210,26 @@ where .iter() .map(|a| format!("https://{}:{}", a.ip(), a.port())) .collect(); + let (racks, rack_id) = rack + .map(|rack| { + let rack_id = rack.rack_id; + ( + BTreeMap::from([( + rack_id.clone(), + RackConfig { + rack_profile_id: rack.rack_profile_id, + }, + )]), + Some(rack_id), + ) + }) + .unwrap_or_default(); let mat_config = MachineATronConfig { + racks, machines: BTreeMap::from([( "config".to_string(), Arc::new(MachineConfig { + rack_id, hw_type, host_count, dpu_per_host_count, diff --git a/crates/api-test-helper/src/api_server.rs b/crates/api-test-helper/src/api_server.rs index 16c04a197e..f8a9ea7d55 100644 --- a/crates/api-test-helper/src/api_server.rs +++ b/crates/api-test-helper/src/api_server.rs @@ -245,6 +245,20 @@ pub async fn start( [host_models] + [rack_profiles.NVL72] + product_family = "gb200" + + [rack_profiles.NVL72.rack_capabilities.compute] + name = "GB200" + count = 18 + vendor = "NVIDIA" + + [rack_profiles.NVL72.rack_capabilities.switch] + count = 9 + + [rack_profiles.NVL72.rack_capabilities.power_shelf] + count = 8 + [firmware_global] autoupdate = true host_enable_autoupdate = [] diff --git a/crates/api-test-helper/src/machine_a_tron.rs b/crates/api-test-helper/src/machine_a_tron.rs index 13b1a635cc..1a1be33a6c 100644 --- a/crates/api-test-helper/src/machine_a_tron.rs +++ b/crates/api-test-helper/src/machine_a_tron.rs @@ -44,6 +44,8 @@ pub async fn run_local( bmc_address_registry: Option, mac_address_pool: Arc>, ) -> eyre::Result<(Vec, MachineATronHandle)> { + app_config.validate()?; + let forge_root_ca_path = get_root_ca_path(None, None); // Will get it from the local repo let forge_client_config = ForgeClientConfig::new(forge_root_ca_path.clone(), None); diff --git a/crates/machine-a-tron/Cargo.toml b/crates/machine-a-tron/Cargo.toml index c51bc5f118..5536f03882 100644 --- a/crates/machine-a-tron/Cargo.toml +++ b/crates/machine-a-tron/Cargo.toml @@ -72,6 +72,7 @@ carbide-network = { path = "../network" } carbide-utils = { path = "../utils", features = ["test-support"] } [dev-dependencies] +carbide-test-support = { path = "../test-support" } toml = { workspace = true } [build-dependencies] diff --git a/crates/machine-a-tron/config/mat.toml b/crates/machine-a-tron/config/mat.toml index 1f79b30d61..a4a1bf43bf 100644 --- a/crates/machine-a-tron/config/mat.toml +++ b/crates/machine-a-tron/config/mat.toml @@ -64,7 +64,14 @@ bmc_mock_port = 2000 # host_bmc_password = "vault-password" # dpu_bmc_password = "vault-password" +# Optional rack declarations. Machine groups can reference these declarations +# with rack_id to register rack-aware expected inventory. +# +# [racks.rack-001] +# rack_profile_id = "NVL72" + [machines.config] +# rack_id = "rack-001" host_count = 3 vpc_count = 10 dpu_per_host_count = 1 diff --git a/crates/machine-a-tron/src/api_client.rs b/crates/machine-a-tron/src/api_client.rs index c0997850cc..5eb32ddc13 100644 --- a/crates/machine-a-tron/src/api_client.rs +++ b/crates/machine-a-tron/src/api_client.rs @@ -21,13 +21,14 @@ use bmc_mock::{DUMMY_FACTORY_PASSWORD, DUMMY_FACTORY_USERNAME, MachineInfo}; use carbide_uuid::instance::InstanceId; use carbide_uuid::machine::{MachineId, MachineInterfaceId}; use carbide_uuid::machine_validation::MachineValidationId; +use carbide_uuid::rack::{RackId, RackProfileId}; use mac_address::MacAddress; use rpc::forge::instance_operating_system_config::Variant; use rpc::forge::machine_cleanup_info::CleanupStepResult; use rpc::forge::{ - ConfigSetting, ExpectedMachine, ExpectedPowerShelf, ExpectedSwitch, InlineIpxe, - InstanceOperatingSystemConfig, MachinesByIdsRequest, SetDynamicConfigRequest, - VpcVirtualizationType, + ConfigSetting, ExpectedMachine, ExpectedPowerShelf, ExpectedRack, ExpectedRackRequest, + ExpectedSwitch, InlineIpxe, InstanceOperatingSystemConfig, MachinesByIdsRequest, + SetDynamicConfigRequest, VpcVirtualizationType, }; use rpc::protos::forge_api_client::ForgeApiClient; @@ -498,6 +499,7 @@ impl ApiClient { &self, bmc_mac_address: String, chassis_serial_number: String, + rack_id: Option, dpu_mode: Option, ) -> ClientApiResult<()> { self.0 @@ -511,7 +513,7 @@ impl ApiClient { sku_id: None, id: None, host_nics: vec![], - rack_id: None, + rack_id, default_pause_ingestion_and_poweron: None, #[allow(deprecated)] dpf_enabled: true, @@ -531,6 +533,7 @@ impl ApiClient { &self, bmc_mac_address: String, shelf_serial_number: String, + rack_id: Option, ) -> ClientApiResult<()> { self.0 .add_expected_power_shelf(ExpectedPowerShelf { @@ -541,7 +544,7 @@ impl ApiClient { shelf_serial_number, bmc_ip_address: String::new(), metadata: None, - rack_id: None, + rack_id, bmc_retain_credentials: Some(true), }) .await @@ -554,6 +557,7 @@ impl ApiClient { bmc_mac_address: String, switch_serial_number: String, nvos_mac_addresses: Vec, + rack_id: Option, ) -> ClientApiResult<()> { self.0 .add_expected_switch(ExpectedSwitch { @@ -568,10 +572,48 @@ impl ApiClient { bmc_ip_address: String::new(), nvos_ip_address: None, metadata: None, - rack_id: None, + rack_id, bmc_retain_credentials: None, }) .await .map_err(ClientApiError::InvocationError) } + + pub async fn ensure_expected_rack( + &self, + rack_id: RackId, + rack_profile_id: RackProfileId, + ) -> ClientApiResult<()> { + let expected_rack = ExpectedRack { + rack_id: Some(rack_id.clone()), + rack_profile_id: Some(rack_profile_id.clone()), + metadata: None, + }; + + match self.0.add_expected_rack(expected_rack).await { + Ok(()) => Ok(()), + Err(status) if status.code() == tonic::Code::AlreadyExists => { + let existing = self + .0 + .get_expected_rack(ExpectedRackRequest { + rack_id: rack_id.to_string(), + }) + .await + .map_err(ClientApiError::InvocationError)?; + if existing.rack_profile_id.as_ref() == Some(&rack_profile_id) { + Ok(()) + } else { + let existing_profile_id = existing + .rack_profile_id + .as_ref() + .map(RackProfileId::as_str) + .unwrap_or(""); + Err(ClientApiError::ConfigError(format!( + "Expected rack {rack_id} already exists with rack_profile_id {existing_profile_id}, not {rack_profile_id}" + ))) + } + } + Err(status) => Err(ClientApiError::InvocationError(status)), + } + } } diff --git a/crates/machine-a-tron/src/config.rs b/crates/machine-a-tron/src/config.rs index 86296d78ea..b0bbd6b1fc 100644 --- a/crates/machine-a-tron/src/config.rs +++ b/crates/machine-a-tron/src/config.rs @@ -23,6 +23,7 @@ use std::time::Duration; use bmc_mock::mac_address_pool::MacAddressPool; use bmc_mock::{DpuMachineInfo, DpuSettings, HostHardwareType}; use carbide_uuid::machine::MachineId; +use carbide_uuid::rack::{RackId, RackProfileId}; use clap::Parser; use duration_str::deserialize_duration; use mac_address::MacAddress; @@ -68,6 +69,8 @@ pub struct MachineATronArgs { #[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct MachineConfig { + #[serde(default)] + pub rack_id: Option, #[serde(default = "default_host_hardware_type")] pub hw_type: HostHardwareType, pub host_count: u32, @@ -125,6 +128,11 @@ pub struct MachineConfig { pub dpu_agent_version: Option, } +#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] +pub struct RackConfig { + pub rack_profile_id: RackProfileId, +} + #[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq)] pub struct DpuFirmwareVersions { pub bmc: Option, @@ -177,6 +185,8 @@ impl DpuFirmwareVersions { #[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] pub struct MachineATronConfig { + #[serde(default)] + pub racks: BTreeMap, // note that order is important in machines so that mac addresses are assigned the same way between runs #[serde( deserialize_with = "deserialize_machine_config", @@ -261,6 +271,27 @@ pub struct MachineATronConfig { } impl MachineATronConfig { + pub fn validate(&self) -> eyre::Result<()> { + for (rack_id, rack) in &self.racks { + eyre::ensure!(!rack_id.as_str().is_empty(), "rack ID cannot be empty"); + eyre::ensure!( + !rack.rack_profile_id.as_str().is_empty(), + "rack {rack_id} has an empty rack_profile_id" + ); + } + + for (machine_group, machine) in &self.machines { + if let Some(rack_id) = machine.rack_id.as_ref() { + eyre::ensure!( + self.racks.contains_key(rack_id), + "machines.{machine_group}.rack_id references undeclared rack {rack_id}" + ); + } + } + + Ok(()) + } + pub fn read_persisted_machines( &self, ) -> eyre::Result>>> { @@ -506,11 +537,14 @@ where #[cfg(test)] mod tests { + use carbide_test_support::Outcome::*; + use carbide_test_support::{Case, check_cases}; + use super::*; - #[test] - fn test_serialize_config() { - let cfg_str = r#" + fn rack_config() -> MachineATronConfig { + toml::from_str( + r#" carbide_api_url = "https://carbide-api.forge:443" log_file = "mat.log" interface = "br-77cbb29de011" @@ -523,7 +557,11 @@ mat_api_server_listen_port = 2112 use_single_bmc_mock = true configure_carbide_bmc_proxy_host = "192.168.1.20" +[racks.rack-001] +rack_profile_id = "NVL72" + [machines.config] +rack_id = "rack-001" host_count = 10 dpu_per_host_count = 2 dpu_reboot_delay = 1 # in units of seconds @@ -536,12 +574,77 @@ run_interval_working = "100ms" run_interval_idle = "1s" network_status_run_interval = "5s" scout_run_interval = "5s" - "#; + "#, + ) + .expect("Could not parse config") + } - let cfg = toml::from_str::(cfg_str).expect("Could not parse config"); + #[test] + fn test_serialize_config() { + let cfg = rack_config(); + cfg.validate().expect("Could not validate config"); let serialized = toml::to_string(&cfg).expect("Could not serialize config"); let round_tripped = toml::from_str::(&serialized) .expect("Could not deserialize serialized config"); assert_eq!(round_tripped, cfg); } + + #[test] + fn rack_references_are_validated() { + let valid = rack_config(); + + let mut rackless = valid.clone(); + rackless.racks.clear(); + Arc::make_mut(rackless.machines.get_mut("config").unwrap()).rack_id = None; + + let mut undeclared = valid.clone(); + undeclared.racks.clear(); + + let mut empty_rack_id = valid.clone(); + let rack = empty_rack_id + .racks + .remove(&RackId::new("rack-001")) + .unwrap(); + empty_rack_id.racks.insert(RackId::new(""), rack); + Arc::make_mut(empty_rack_id.machines.get_mut("config").unwrap()).rack_id = + Some(RackId::new("")); + + let mut empty_profile_id = valid.clone(); + empty_profile_id + .racks + .get_mut(&RackId::new("rack-001")) + .unwrap() + .rack_profile_id = RackProfileId::new(""); + + check_cases( + [ + Case { + scenario: "declared rack", + input: valid, + expect: Yields(()), + }, + Case { + scenario: "legacy rackless config", + input: rackless, + expect: Yields(()), + }, + Case { + scenario: "undeclared rack reference", + input: undeclared, + expect: Fails, + }, + Case { + scenario: "empty rack ID", + input: empty_rack_id, + expect: Fails, + }, + Case { + scenario: "empty rack profile ID", + input: empty_profile_id, + expect: Fails, + }, + ], + |config| config.validate().map_err(drop), + ); + } } diff --git a/crates/machine-a-tron/src/lib.rs b/crates/machine-a-tron/src/lib.rs index 9a6c509d0c..a1d4a50628 100644 --- a/crates/machine-a-tron/src/lib.rs +++ b/crates/machine-a-tron/src/lib.rs @@ -37,7 +37,7 @@ use std::time::{Duration, Instant}; pub use bmc_mock_wrapper::BmcMockRegistry; pub use config::{ MachineATronArgs, MachineATronConfig, MachineATronContext, MachineConfig, PersistedDpuMachine, - PersistedHostMachine, + PersistedHostMachine, RackConfig, }; pub use dpu_machine::DpuMachineHandle; pub use host_machine::HostMachineHandle; diff --git a/crates/machine-a-tron/src/machine_a_tron.rs b/crates/machine-a-tron/src/machine_a_tron.rs index ab42d3079b..895ad21a53 100644 --- a/crates/machine-a-tron/src/machine_a_tron.rs +++ b/crates/machine-a-tron/src/machine_a_tron.rs @@ -48,6 +48,8 @@ impl MachineATron { } pub async fn make_machines(&self, paused: bool) -> eyre::Result> { + self.app_context.app_config.validate()?; + let mut persisted_machines = self .app_context .app_config @@ -136,8 +138,22 @@ impl MachineATron { }; if self.app_context.app_config.register_expected_machines { + for (rack_id, rack) in &self.app_context.app_config.racks { + self.app_context + .api_client() + .ensure_expected_rack(rack_id.clone(), rack.rack_profile_id.clone()) + .await?; + } + for machine in &machines { let host_info = machine.host_info(); + let machine_config = self + .app_context + .app_config + .machines + .get(machine.machine_config_section()) + .expect("machine was constructed from a configured machine group"); + let rack_id = machine_config.rack_id.clone(); let result = match host_info.hw_type { HostHardwareType::LiteOnPowerShelf => { self.app_context @@ -145,6 +161,7 @@ impl MachineATron { .add_expected_power_shelf( host_info.bmc_mac_address.to_string(), host_info.serial.clone(), + rack_id, ) .await } @@ -162,6 +179,7 @@ impl MachineATron { .iter() .map(|mac| mac.to_string()) .collect(), + rack_id, ) .await } @@ -172,25 +190,19 @@ impl MachineATron { // else defers to the absolute default (DpuMode). // Site-explorer's ingestion gate requires this explicit // declaration for any host without DPU PCIe devices. - let dpu_mode = self - .app_context - .app_config - .machines - .get(machine.machine_config_section()) - .and_then(|config| { - if config.dpu_per_host_count == 0 { - Some(DpuMode::NoDpu) - } else if config.dpus_in_nic_mode { - Some(DpuMode::NicMode) - } else { - None - } - }); + let dpu_mode = if machine_config.dpu_per_host_count == 0 { + Some(DpuMode::NoDpu) + } else if machine_config.dpus_in_nic_mode { + Some(DpuMode::NicMode) + } else { + None + }; self.app_context .api_client() .add_expected_machine( host_info.bmc_mac_address.to_string(), host_info.serial.clone(), + rack_id, dpu_mode, ) .await diff --git a/crates/machine-a-tron/src/main.rs b/crates/machine-a-tron/src/main.rs index ead0189ed7..d4ac43a937 100644 --- a/crates/machine-a-tron/src/main.rs +++ b/crates/machine-a-tron/src/main.rs @@ -87,6 +87,7 @@ async fn main() -> Result<(), Box> { } let fig = Figment::new().merge(Toml::file(config_path)); let app_config: MachineATronConfig = fig.extract()?; + app_config.validate()?; let tui_host_logs = if app_config.tui_enabled { Some(TuiHostLogs::start_new(100)) } else { From d389a2a3e094bf1867697f3638fa06f5c85626a5 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Tue, 7 Jul 2026 16:58:42 -0500 Subject: [PATCH 2/7] fix: Preserve admin segment lock order during discovery --- crates/api-core/src/handlers/machine_discovery.rs | 6 ++++++ crates/api-db/src/machine_interface.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/api-core/src/handlers/machine_discovery.rs b/crates/api-core/src/handlers/machine_discovery.rs index 04c86e3e7f..ae973b58cd 100644 --- a/crates/api-core/src/handlers/machine_discovery.rs +++ b/crates/api-core/src/handlers/machine_discovery.rs @@ -116,6 +116,12 @@ pub(crate) async fn discover_machine( }; let mut txn = api.txn_begin().await?; + if hardware_info.is_dpu() { + // Discovery updates machine-interface rows before reconciliation. Take the segment locks + // first so the outer transaction preserves the allocator's advisory-lock-before-row-lock + // order instead of relying on reconciliation's inner savepoint to establish it later. + db::machine_interface::load_and_lock_all_admin_segments(&mut txn).await?; + } tracing::debug!( ?remote_ip, ?interface_id, diff --git a/crates/api-db/src/machine_interface.rs b/crates/api-db/src/machine_interface.rs index b9604b85a9..b07fa01dd1 100644 --- a/crates/api-db/src/machine_interface.rs +++ b/crates/api-db/src/machine_interface.rs @@ -2212,7 +2212,7 @@ FOR UPDATE"#; /// This intentionally locks more broadly than the specific host touches so reconciliation follows /// the same high-level lock order as address allocation: segment advisory lock first, then /// machine interface/address row locks. -async fn load_and_lock_all_admin_segments( +pub async fn load_and_lock_all_admin_segments( txn: &mut Transaction<'_>, ) -> DatabaseResult> { let mut segment_ids = From 70d0d09b86e55ed49eed44d4a907d1da03051a81 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Tue, 7 Jul 2026 17:41:58 -0500 Subject: [PATCH 3/7] chore: Ignore local workspace files Signed-off-by: Kyle Felter --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 830826bff2..f97b5fef1f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,11 @@ crates/dpf/doca-platform # Generated plans from Cursor .cursor/ +.kolo +.kolo/ +.agents/ +.trash/ +.secrets # Generated by github runners, ignore them to prevent failures in check-repo-clean.sh /cargo/ From 8b94e54c8c33dd426dcc276ba141b347a9ef82f4 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Tue, 7 Jul 2026 17:42:06 -0500 Subject: [PATCH 4/7] test: Isolate rack-aware simulator validation Signed-off-by: Kyle Felter --- crates/api-integration-tests/tests/lib.rs | 58 ++++++++++++++++++++--- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/crates/api-integration-tests/tests/lib.rs b/crates/api-integration-tests/tests/lib.rs index c2fdf4a543..0ac9050d27 100644 --- a/crates/api-integration-tests/tests/lib.rs +++ b/crates/api-integration-tests/tests/lib.rs @@ -161,12 +161,6 @@ async fn test_integration() -> eyre::Result<()> { Ipv4Addr::new(172, 20, 0, 2), ) .boxed(), - test_machine_a_tron_rack( - &test_env, - &bmc_address_registry, - Ipv4Addr::new(172, 20, 0, 2), - ) - .boxed(), test_machine_a_tron_zerodpu( HostHardwareType::DellPowerEdgeR750, &test_env, @@ -237,6 +231,58 @@ async fn test_integration() -> eyre::Result<()> { Ok(()) } +/// Exercise the rack-aware machine-a-tron path independently from the other parallel scenarios. +#[tokio::test(flavor = "multi_thread")] +async fn test_machine_a_tron_rack_integration() -> eyre::Result<()> { + let Some(test_env) = IntegrationTestEnvironment::try_from_environment( + 1, + "api_server_test_machine_a_tron_rack_integration", + ) + .await? + else { + return Ok(()); + }; + + let bmc_address_registry = BmcMockRegistry::default(); + let certs_dir = test_env.root_dir.join("crates/bmc-mock"); + let server_config = bmc_mock::tls::server_config(Some(certs_dir)).unwrap(); + let mut bmc_mock_handle = bmc_mock::CombinedServer::run( + "bmc-mock", + bmc_address_registry.clone(), + Some(ListenerOrAddress::Listener(TcpListener::bind( + "127.0.0.1:0", + )?)), + server_config, + ); + let empty_firmware_dir = temp_dir::TempDir::with_prefix("firmware")?; + let cancel_token = CancellationToken::new(); + let server_handle = utils::start_api_server( + test_env.clone(), + Some(HostPortPair::HostAndPort( + "127.0.0.1".to_string(), + bmc_mock_handle.address.port(), + )), + empty_firmware_dir.path().to_owned(), + 0, + true, + cancel_token.clone(), + ) + .await?; + + test_machine_a_tron_rack( + &test_env, + &bmc_address_registry, + Ipv4Addr::new(172, 20, 0, 2), + ) + .await?; + + cancel_token.cancel(); + server_handle.wait().await?; + test_env.db_pool.close().await; + bmc_mock_handle.stop().await?; + Ok(()) +} + fn generate_core_metric_docs(metrics_endpoints: &[SocketAddr]) { let mut infos = metrics::collect_metric_infos(metrics_endpoints).unwrap(); retain_existing_core_metric_infos(&mut infos); From f9f7e30d8b7b010054de2e44b023c03224dfc16c Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Wed, 8 Jul 2026 10:24:28 -0500 Subject: [PATCH 5/7] chore: Remove local workspace ignores --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index f97b5fef1f..830826bff2 100644 --- a/.gitignore +++ b/.gitignore @@ -43,11 +43,6 @@ crates/dpf/doca-platform # Generated plans from Cursor .cursor/ -.kolo -.kolo/ -.agents/ -.trash/ -.secrets # Generated by github runners, ignore them to prevent failures in check-repo-clean.sh /cargo/ From e2311c019b48ef3e37d0c66a79f6ed52e035a549 Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Wed, 8 Jul 2026 13:51:55 -0500 Subject: [PATCH 6/7] refactor: Split DPU discovery lock fix --- crates/api-core/src/handlers/machine_discovery.rs | 6 ------ crates/api-db/src/machine_interface.rs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/crates/api-core/src/handlers/machine_discovery.rs b/crates/api-core/src/handlers/machine_discovery.rs index ae973b58cd..04c86e3e7f 100644 --- a/crates/api-core/src/handlers/machine_discovery.rs +++ b/crates/api-core/src/handlers/machine_discovery.rs @@ -116,12 +116,6 @@ pub(crate) async fn discover_machine( }; let mut txn = api.txn_begin().await?; - if hardware_info.is_dpu() { - // Discovery updates machine-interface rows before reconciliation. Take the segment locks - // first so the outer transaction preserves the allocator's advisory-lock-before-row-lock - // order instead of relying on reconciliation's inner savepoint to establish it later. - db::machine_interface::load_and_lock_all_admin_segments(&mut txn).await?; - } tracing::debug!( ?remote_ip, ?interface_id, diff --git a/crates/api-db/src/machine_interface.rs b/crates/api-db/src/machine_interface.rs index e6d237704a..09e1ad1e53 100644 --- a/crates/api-db/src/machine_interface.rs +++ b/crates/api-db/src/machine_interface.rs @@ -2627,7 +2627,7 @@ FOR UPDATE"#; /// This intentionally locks more broadly than the specific host touches so reconciliation follows /// the same high-level lock order as address allocation: segment advisory lock first, then /// machine interface/address row locks. -pub async fn load_and_lock_all_admin_segments( +async fn load_and_lock_all_admin_segments( txn: &mut Transaction<'_>, ) -> DatabaseResult> { let mut segment_ids = From 749668082fe3e59b91f968309f02ea2962e6b8da Mon Sep 17 00:00:00 2001 From: Kyle Felter Date: Fri, 10 Jul 2026 13:23:50 -0500 Subject: [PATCH 7/7] test: Align rack fixture with simulated inventory Signed-off-by: Kyle Felter --- AGENTS.md | 4 ++++ crates/api-test-helper/src/api_server.rs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b30478c4a9..0b05160f92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,6 +101,10 @@ Enumerating a function's input variants as grouped `carbide-test-support` scenar or explicit cases (`check_cases` / `check_values`) is the easiest way to reach thorough coverage of parsers, validators, conversions, and the like. +Keep test rack-profile capability counts aligned with the inventory the fixture +actually instantiates. Use zero for unsupported component types so tests do not +generate expected-but-absent discovery errors. + ### Linting and Formatting ```bash diff --git a/crates/api-test-helper/src/api_server.rs b/crates/api-test-helper/src/api_server.rs index f8a9ea7d55..c639cd8e1d 100644 --- a/crates/api-test-helper/src/api_server.rs +++ b/crates/api-test-helper/src/api_server.rs @@ -254,10 +254,10 @@ pub async fn start( vendor = "NVIDIA" [rack_profiles.NVL72.rack_capabilities.switch] - count = 9 + count = 0 [rack_profiles.NVL72.rack_capabilities.power_shelf] - count = 8 + count = 0 [firmware_global] autoupdate = true