diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index 693aea88732..e1e10f890b6 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -28,6 +28,7 @@ use omicron_sled_agent::sim; use omicron_test_utils::dev; use omicron_test_utils::dev::poll; use omicron_test_utils::dev::poll::CondCheckError; +use omicron_test_utils::dev::poll::wait_for_condition; use omicron_test_utils::dev::poll::wait_for_watch_channel_condition; use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::SledUuid; @@ -242,20 +243,35 @@ impl ControlPlaneTestContext { /// Works both when Dendrite is currently running (will stop and restart) /// or when it was previously stopped via [`Self::stop_dendrite`]. pub async fn restart_dendrite(&self, switch_slot: SwitchSlot) { + use slog::debug; + let log = self.logctx.log.new(slog::o!( + "switch_slot" => format!("{switch_slot:?}"), + )); + debug!(log, "Restarting Dendrite"); + // Get port either from running instance or from stored port after stop // Extract from mutex first to avoid holding lock across await let old = self.dendrite.write().unwrap().remove(&switch_slot); let port = if let Some(mut old) = old { let port = old.port; + debug!( + log, "Shutting down old dpd instance for restart"; + "port" => port, + ); old.cleanup().await.unwrap(); port } else { // Must have been stopped - get stored port - self.stopped_dendrite_ports + let port = self.stopped_dendrite_ports .write() .unwrap() .remove(&switch_slot) - .expect("Dendrite not running and no stored port from stop_dendrite") + .expect("Dendrite not running and no stored port from stop_dendrite"); + debug!( + log, "Reusing port from previously-shut-down dpd instance"; + "port" => port, + ); + port }; let mgs = self.gateway.get(&switch_slot).unwrap(); @@ -287,15 +303,18 @@ impl ControlPlaneTestContext { log: self.logctx.log.clone(), }, ); - loop { - match dpd_client.switch_identifiers().await { - Ok(_) => break, - Err(_) => { - tokio::time::sleep(std::time::Duration::from_millis(50)) - .await; + wait_for_condition( + || async { + match dpd_client.switch_identifiers().await { + Ok(_) => Ok(()), + Err(_) => Err(dev::poll::CondCheckError::<()>::NotYet), } - } - } + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .expect("contacted dpd within timeout after restart"); self.dendrite.write().unwrap().insert(switch_slot, dendrite); }