From 03a67c7cd35c9bbd421e0d94f7503c5c1956535d Mon Sep 17 00:00:00 2001 From: Josh Carp Date: Wed, 21 Jan 2026 13:17:23 -0500 Subject: [PATCH 1/2] Optionally configure crdb listen port in dev. At the moment, omicron-dev starts crdb on port 0, which tells the database to use any available port. This is fine for most use cases, but less fine for running a second nexus alongside an existing run-all: https://github.com/oxidecomputer/omicron/blob/main/docs/how-to-run-simulated.adoc#using-both-omicron-dev-run-all-and-running-nexus-manually. This process involves copying the randomly selected ports from the output of run-all and writing them to a custom nexus config. To make the process simpler, this patch allows the user to configure the crdb listen port, defaulting to the current choice of 0. --- dev-tools/omicron-dev/Cargo.toml | 1 + dev-tools/omicron-dev/src/main.rs | 17 ++++++++++++----- nexus/db-queries/src/db/pub_test_utils/crdb.rs | 10 +++++++++- nexus/test-utils/src/nexus_test.rs | 4 +++- nexus/test-utils/src/starter.rs | 14 +++++++++++++- test-utils/src/dev/db.rs | 2 +- test-utils/src/dev/mod.rs | 5 ++++- test-utils/src/dev/seed.rs | 1 + test-utils/src/lib.rs | 2 ++ 9 files changed, 46 insertions(+), 10 deletions(-) diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 21a4bc7210c..1827d92516c 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -22,6 +22,7 @@ nexus-config.workspace = true nexus-test-interface.workspace = true nexus-test-utils = { workspace = true, features = ["omicron-dev"] } omicron-nexus.workspace = true +omicron-test-utils.workspace = true omicron-workspace-hack.workspace = true oxide-tokio-rt.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. diff --git a/dev-tools/omicron-dev/src/main.rs b/dev-tools/omicron-dev/src/main.rs index 9fa5ac0fc05..959c0c8ed39 100644 --- a/dev-tools/omicron-dev/src/main.rs +++ b/dev-tools/omicron-dev/src/main.rs @@ -11,6 +11,7 @@ use libc::SIGINT; use nexus_config::NexusConfig; use nexus_test_interface::NexusServer; use nexus_test_utils::resource_helpers::DiskTest; +use omicron_test_utils::COCKROACHDB_DEFAULT_LISTEN_PORT; use signal_hook_tokio::Signals; use std::fs; @@ -51,6 +52,9 @@ struct RunAllArgs { /// Nexus external API listen port. Use `0` to request any available port. #[clap(long, action)] nexus_listen_port: Option, + /// CockroachDB listen port. Use `0` to request any available port. + #[clap(long, default_value_t = COCKROACHDB_DEFAULT_LISTEN_PORT)] + db_listen_port: u16, /// Override the gateway server configuration file. #[clap(long, default_value = DEFAULT_SP_SIM_CONFIG)] gateway_config: Utf8PathBuf, @@ -88,11 +92,14 @@ impl RunAllArgs { } println!("omicron-dev: setting up all services ... "); - let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< - omicron_nexus::Server, - >(&mut config, 0, self.gateway_config.clone()) - .await - .context("error setting up services")?; + let cptestctx = + nexus_test_utils::omicron_dev_setup_with_config::< + omicron_nexus::Server, + >( + &mut config, 0, self.gateway_config.clone(), self.db_listen_port + ) + .await + .context("error setting up services")?; println!("omicron-dev: Adding disks to first sled agent"); diff --git a/nexus/db-queries/src/db/pub_test_utils/crdb.rs b/nexus/db-queries/src/db/pub_test_utils/crdb.rs index aa06f0681eb..3aca52f51e6 100644 --- a/nexus/db-queries/src/db/pub_test_utils/crdb.rs +++ b/nexus/db-queries/src/db/pub_test_utils/crdb.rs @@ -35,6 +35,7 @@ pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { dev::test_setup_database( log, dev::StorageSource::CopyFromSeed { input_tar }, + dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT, ) .await } @@ -45,7 +46,12 @@ pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { pub async fn test_setup_database_empty( log: &Logger, ) -> dev::db::CockroachInstance { - dev::test_setup_database(log, dev::StorageSource::DoNotPopulate).await + dev::test_setup_database( + log, + dev::StorageSource::DoNotPopulate, + dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT, + ) + .await } /// Wrapper around [`dev::test_setup_database`] which uses a seed tarball @@ -53,10 +59,12 @@ pub async fn test_setup_database_empty( pub async fn test_setup_database_from_seed( log: &Logger, input_tar: Utf8PathBuf, + listen_port: u16, ) -> dev::db::CockroachInstance { dev::test_setup_database( log, dev::StorageSource::CopyFromSeed { input_tar }, + listen_port, ) .await } diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index a57bf139a9b..dab89b1afbf 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -361,8 +361,10 @@ pub async fn omicron_dev_setup_with_config( config: &mut NexusConfig, extra_sled_agents: u16, gateway_config_file: Utf8PathBuf, + listen_port: u16, ) -> Result> { - let starter = ControlPlaneStarter::::new("omicron-dev", config); + let mut starter = ControlPlaneStarter::::new("omicron-dev", config); + starter.db_listen_port(listen_port); let log = &starter.logctx.log; debug!(log, "Ensuring seed tarball exists"); diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 696a40b8e88..6b5cd81253e 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -166,6 +166,8 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub password: Option, pub simulated_upstairs: Arc, + + db_listen_port: u16, } type StepInitFn<'a, N> = Box< @@ -216,9 +218,14 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { simulated_upstairs: Arc::new(sim::SimulatedUpstairs::new( simulated_upstairs_log, )), + db_listen_port: dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT, } } + pub fn db_listen_port(&mut self, port: u16) { + self.db_listen_port = port; + } + pub async fn init_with_steps( &mut self, steps: Vec<(&str, StepInitFn<'a, N>)>, @@ -266,7 +273,12 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { } #[cfg(feature = "omicron-dev")] PopulateCrdb::FromSeed { input_tar } => { - crdb::test_setup_database_from_seed(log, input_tar).await + crdb::test_setup_database_from_seed( + log, + input_tar, + self.db_listen_port, + ) + .await } PopulateCrdb::Empty => crdb::test_setup_database_empty(log).await, }; diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs index 9398290c7a5..76a6496933f 100644 --- a/test-utils/src/dev/db.rs +++ b/test-utils/src/dev/db.rs @@ -33,7 +33,7 @@ const COCKROACHDB_START_TIMEOUT_DEFAULT: Duration = Duration::from_secs(30); // This is appropriate for the test suite and may be useful in some cases for // omicron-dev. However, omicron-dev by default chooses a specific port so that // we can ship a Nexus configuration that will use the same port. -const COCKROACHDB_DEFAULT_LISTEN_PORT: u16 = 0; +pub const COCKROACHDB_DEFAULT_LISTEN_PORT: u16 = 0; /// CockroachDB database name // This MUST be kept in sync with dbinit.sql and dbwipe.sql. diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index 5a37ac2549e..76081e9b047 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -62,9 +62,10 @@ pub enum StorageSource { pub async fn test_setup_database( log: &Logger, source: StorageSource, + listen_port: u16, ) -> db::CockroachInstance { usdt::register_probes().expect("Failed to register USDT DTrace probes"); - setup_database(log, source).await.unwrap() + setup_database(log, source, listen_port).await.unwrap() } // TODO: switch to anyhow entirely -- this function is currently a mishmash of @@ -72,8 +73,10 @@ pub async fn test_setup_database( async fn setup_database( log: &Logger, storage_source: StorageSource, + listen_port: u16, ) -> Result { let builder = db::CockroachStarterBuilder::new(); + let builder = builder.listen_port(listen_port); let builder = match &storage_source { StorageSource::DoNotPopulate | StorageSource::CopyFromSeed { .. } => { builder diff --git a/test-utils/src/dev/seed.rs b/test-utils/src/dev/seed.rs index 5d5ea5beb44..3287aabff82 100644 --- a/test-utils/src/dev/seed.rs +++ b/test-utils/src/dev/seed.rs @@ -173,6 +173,7 @@ pub async fn test_setup_database_seed( super::StorageSource::PopulateLatest { output_dir: tmp_seed_dir.path().to_owned(), }, + super::db::COCKROACHDB_DEFAULT_LISTEN_PORT, ) .await .context("failed to setup database")?; diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index f035b8132b7..f991dd3a774 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -15,6 +15,8 @@ use headers::authorization::Credentials; pub mod certificates; pub mod dev; +pub use dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT; + #[macro_use] extern crate slog; From cf8c300cbb019d3a4119ab99d8a7eea0db9e6626 Mon Sep 17 00:00:00 2001 From: Josh Carp Date: Fri, 23 Jan 2026 10:17:12 -0500 Subject: [PATCH 2/2] Optionally configure mgs and internal dns ports for omicron-dev. This completes the set of omicron-dev ports used in the simulated workflow that includes `run-all` and a second nexus: https://github.com/oxidecomputer/omicron/blob/main/docs/how-to-run-simulated.adoc#using-both-omicron-dev-run-all-and-running-nexus-manually --- dev-tools/omicron-dev/Cargo.toml | 1 - dev-tools/omicron-dev/src/main.rs | 33 ++++++++++------- .../db-queries/src/db/pub_test_utils/crdb.rs | 11 ++---- nexus/test-utils/src/nexus_test.rs | 8 +++-- nexus/test-utils/src/starter.rs | 35 +++++++++++++++---- test-utils/src/dev/db.rs | 2 +- test-utils/src/dev/mod.rs | 6 ++-- test-utils/src/dev/seed.rs | 2 +- test-utils/src/lib.rs | 2 -- 9 files changed, 64 insertions(+), 36 deletions(-) diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 1827d92516c..21a4bc7210c 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -22,7 +22,6 @@ nexus-config.workspace = true nexus-test-interface.workspace = true nexus-test-utils = { workspace = true, features = ["omicron-dev"] } omicron-nexus.workspace = true -omicron-test-utils.workspace = true omicron-workspace-hack.workspace = true oxide-tokio-rt.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. diff --git a/dev-tools/omicron-dev/src/main.rs b/dev-tools/omicron-dev/src/main.rs index 959c0c8ed39..58dbef377e4 100644 --- a/dev-tools/omicron-dev/src/main.rs +++ b/dev-tools/omicron-dev/src/main.rs @@ -11,7 +11,6 @@ use libc::SIGINT; use nexus_config::NexusConfig; use nexus_test_interface::NexusServer; use nexus_test_utils::resource_helpers::DiskTest; -use omicron_test_utils::COCKROACHDB_DEFAULT_LISTEN_PORT; use signal_hook_tokio::Signals; use std::fs; @@ -49,12 +48,18 @@ enum OmicronDevCmd { #[derive(Clone, Debug, Args)] struct RunAllArgs { - /// Nexus external API listen port. Use `0` to request any available port. + /// Nexus external API listen port. Use `0` to request any available port. #[clap(long, action)] nexus_listen_port: Option, /// CockroachDB listen port. Use `0` to request any available port. - #[clap(long, default_value_t = COCKROACHDB_DEFAULT_LISTEN_PORT)] - db_listen_port: u16, + #[clap(long)] + db_listen_port: Option, + /// Internal DNS listen port. Use `0` to request any available port. + #[clap(long)] + internal_dns_listen_port: Option, + /// Management gateway listen port. Use `0` to request any available port. + #[clap(long)] + mgs_listen_port: Option, /// Override the gateway server configuration file. #[clap(long, default_value = DEFAULT_SP_SIM_CONFIG)] gateway_config: Utf8PathBuf, @@ -92,14 +97,18 @@ impl RunAllArgs { } println!("omicron-dev: setting up all services ... "); - let cptestctx = - nexus_test_utils::omicron_dev_setup_with_config::< - omicron_nexus::Server, - >( - &mut config, 0, self.gateway_config.clone(), self.db_listen_port - ) - .await - .context("error setting up services")?; + let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< + omicron_nexus::Server, + >( + &mut config, + 0, + self.gateway_config.clone(), + self.db_listen_port, + self.internal_dns_listen_port, + self.mgs_listen_port, + ) + .await + .context("error setting up services")?; println!("omicron-dev: Adding disks to first sled agent"); diff --git a/nexus/db-queries/src/db/pub_test_utils/crdb.rs b/nexus/db-queries/src/db/pub_test_utils/crdb.rs index 3aca52f51e6..d0fe4608f12 100644 --- a/nexus/db-queries/src/db/pub_test_utils/crdb.rs +++ b/nexus/db-queries/src/db/pub_test_utils/crdb.rs @@ -35,7 +35,7 @@ pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { dev::test_setup_database( log, dev::StorageSource::CopyFromSeed { input_tar }, - dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT, + None, ) .await } @@ -46,12 +46,7 @@ pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { pub async fn test_setup_database_empty( log: &Logger, ) -> dev::db::CockroachInstance { - dev::test_setup_database( - log, - dev::StorageSource::DoNotPopulate, - dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT, - ) - .await + dev::test_setup_database(log, dev::StorageSource::DoNotPopulate, None).await } /// Wrapper around [`dev::test_setup_database`] which uses a seed tarball @@ -59,7 +54,7 @@ pub async fn test_setup_database_empty( pub async fn test_setup_database_from_seed( log: &Logger, input_tar: Utf8PathBuf, - listen_port: u16, + listen_port: Option, ) -> dev::db::CockroachInstance { dev::test_setup_database( log, diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index dab89b1afbf..c8b470548fb 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -361,10 +361,14 @@ pub async fn omicron_dev_setup_with_config( config: &mut NexusConfig, extra_sled_agents: u16, gateway_config_file: Utf8PathBuf, - listen_port: u16, + db_listen_port: Option, + internal_dns_listen_port: Option, + mgs_listen_port: Option, ) -> Result> { let mut starter = ControlPlaneStarter::::new("omicron-dev", config); - starter.db_listen_port(listen_port); + starter.db_listen_port(db_listen_port); + starter.internal_dns_listen_port(internal_dns_listen_port); + starter.mgs_listen_port(mgs_listen_port); let log = &starter.logctx.log; debug!(log, "Ensuring seed tarball exists"); diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 6b5cd81253e..9cc3d5cb0fa 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -167,7 +167,9 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub simulated_upstairs: Arc, - db_listen_port: u16, + db_listen_port: Option, + internal_dns_listen_port: Option, + mgs_listen_port: Option, } type StepInitFn<'a, N> = Box< @@ -218,14 +220,24 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { simulated_upstairs: Arc::new(sim::SimulatedUpstairs::new( simulated_upstairs_log, )), - db_listen_port: dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT, + db_listen_port: None, + internal_dns_listen_port: None, + mgs_listen_port: None, } } - pub fn db_listen_port(&mut self, port: u16) { + pub fn db_listen_port(&mut self, port: Option) { self.db_listen_port = port; } + pub fn internal_dns_listen_port(&mut self, port: Option) { + self.internal_dns_listen_port = port; + } + + pub fn mgs_listen_port(&mut self, port: Option) { + self.mgs_listen_port = port; + } + pub async fn init_with_steps( &mut self, steps: Vec<(&str, StepInitFn<'a, N>)>, @@ -1212,7 +1224,18 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { /// Set up an internal DNS server on the first sled agent pub async fn start_internal_dns(&mut self) { let log = self.logctx.log.new(o!("component" => "internal_dns_server")); - let dns = dns_server::TransientServer::new(&log).await.unwrap(); + let dns_addr = SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + self.internal_dns_listen_port.unwrap_or(0), + 0, + 0, + ); + let dns = dns_server::TransientServer::new_with_address( + &log, + dns_addr.into(), + ) + .await + .unwrap(); let SocketAddr::V6(dns_address) = dns.dns_server.local_address() else { panic!("Unsupported IPv4 DNS address"); @@ -1633,7 +1656,7 @@ pub(crate) async fn setup_with_config_impl( builder .start_gateway( SwitchLocation::Switch0, - None, + builder.mgs_listen_port, mgs_config, ) .boxed() @@ -1677,7 +1700,7 @@ pub(crate) async fn setup_with_config_impl( builder .start_gateway( SwitchLocation::Switch1, - None, + builder.mgs_listen_port, gateway_config_file, ) .boxed() diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs index 76a6496933f..9398290c7a5 100644 --- a/test-utils/src/dev/db.rs +++ b/test-utils/src/dev/db.rs @@ -33,7 +33,7 @@ const COCKROACHDB_START_TIMEOUT_DEFAULT: Duration = Duration::from_secs(30); // This is appropriate for the test suite and may be useful in some cases for // omicron-dev. However, omicron-dev by default chooses a specific port so that // we can ship a Nexus configuration that will use the same port. -pub const COCKROACHDB_DEFAULT_LISTEN_PORT: u16 = 0; +const COCKROACHDB_DEFAULT_LISTEN_PORT: u16 = 0; /// CockroachDB database name // This MUST be kept in sync with dbinit.sql and dbwipe.sql. diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index 76081e9b047..ec6d9a23a9d 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -62,7 +62,7 @@ pub enum StorageSource { pub async fn test_setup_database( log: &Logger, source: StorageSource, - listen_port: u16, + listen_port: Option, ) -> db::CockroachInstance { usdt::register_probes().expect("Failed to register USDT DTrace probes"); setup_database(log, source, listen_port).await.unwrap() @@ -73,10 +73,10 @@ pub async fn test_setup_database( async fn setup_database( log: &Logger, storage_source: StorageSource, - listen_port: u16, + listen_port: Option, ) -> Result { let builder = db::CockroachStarterBuilder::new(); - let builder = builder.listen_port(listen_port); + let builder = builder.listen_port(listen_port.unwrap_or(0)); let builder = match &storage_source { StorageSource::DoNotPopulate | StorageSource::CopyFromSeed { .. } => { builder diff --git a/test-utils/src/dev/seed.rs b/test-utils/src/dev/seed.rs index 3287aabff82..6eca6daa8b3 100644 --- a/test-utils/src/dev/seed.rs +++ b/test-utils/src/dev/seed.rs @@ -173,7 +173,7 @@ pub async fn test_setup_database_seed( super::StorageSource::PopulateLatest { output_dir: tmp_seed_dir.path().to_owned(), }, - super::db::COCKROACHDB_DEFAULT_LISTEN_PORT, + None, ) .await .context("failed to setup database")?; diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index f991dd3a774..f035b8132b7 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -15,8 +15,6 @@ use headers::authorization::Credentials; pub mod certificates; pub mod dev; -pub use dev::db::COCKROACHDB_DEFAULT_LISTEN_PORT; - #[macro_use] extern crate slog;