diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74fc2af56ca..ee977747a1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -413,6 +413,13 @@ jobs: hard_fork_latest_era_at_epoch: 0 run_id: "#1" extra_args: "--mithril-era-regenesis-on-switch --aggregate-signature-type=Concatenation" + # Include a test for IPFS upload/download support + - mode: "ipfs" + era: ${{ fromJSON(needs.build-ubuntu.outputs.eras)[0] }} + cardano_node_version: "11.0.1" + hard_fork_latest_era_at_epoch: 0 + run_id: "#1" + extra_args: "--use-ipfs --ipfs-devnet-scripts-directory=./mithril-test-lab/ipfs-devnet/" steps: - name: Checkout sources uses: actions/checkout@v6 diff --git a/Cargo.lock b/Cargo.lock index ca6e381fe3b..c71b42b9406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4598,7 +4598,7 @@ dependencies = [ [[package]] name = "mithril-end-to-end" -version = "0.5.15" +version = "0.5.16" dependencies = [ "anyhow", "async-recursion", diff --git a/mithril-test-lab/ipfs-devnet/README.md b/mithril-test-lab/ipfs-devnet/README.md index efd42bb4dc2..82ea4199259 100644 --- a/mithril-test-lab/ipfs-devnet/README.md +++ b/mithril-test-lab/ipfs-devnet/README.md @@ -148,9 +148,9 @@ For node `N`, the default local ports are: | Service | Port formula | Example for node 1 | | ------- | ------------ | ------------------ | -| Swarm | `4000 + N` | `4001` | | API | `5000 + N` | `5001` | -| Gateway | `8080 + N` | `8081` | +| Gateway | `5100 + N` | `5101` | +| Swarm | `5200 + N` | `5201` | #### Re-initialize an existing swarm diff --git a/mithril-test-lab/ipfs-devnet/commands/mkfiles/kubo-configure-swarm.sh b/mithril-test-lab/ipfs-devnet/commands/mkfiles/kubo-configure-swarm.sh index 02da2a318c8..561dfccea43 100755 --- a/mithril-test-lab/ipfs-devnet/commands/mkfiles/kubo-configure-swarm.sh +++ b/mithril-test-lab/ipfs-devnet/commands/mkfiles/kubo-configure-swarm.sh @@ -67,8 +67,8 @@ configure_node() { local api_port gateway_port swarm_port api_port=$((5000 + node_id)) - gateway_port=$((8080 + node_id)) - swarm_port=$((4000 + node_id)) + gateway_port=$((5100 + node_id)) + swarm_port=$((5200 + node_id)) #---------- Write swarm key { diff --git a/mithril-test-lab/mithril-end-to-end/Cargo.toml b/mithril-test-lab/mithril-end-to-end/Cargo.toml index 80ae746e4ee..a748c3b1851 100644 --- a/mithril-test-lab/mithril-end-to-end/Cargo.toml +++ b/mithril-test-lab/mithril-end-to-end/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-end-to-end" -version = "0.5.15" +version = "0.5.16" authors = { workspace = true } edition = { workspace = true } documentation = { workspace = true } diff --git a/mithril-test-lab/mithril-end-to-end/README.md b/mithril-test-lab/mithril-end-to-end/README.md index 6edfaad4b8a..e5dc6af3f2a 100644 --- a/mithril-test-lab/mithril-end-to-end/README.md +++ b/mithril-test-lab/mithril-end-to-end/README.md @@ -49,6 +49,43 @@ To run `mithril-end-to-end` command, you must first compile the Mithril nodes: cargo build --release ``` +## Additional network topology options + +These options can be combined with the base run command shown above (see `./mithril-end-to-end --help` for the full +list of flags). + +### Multiple aggregators + +Run several aggregators behind Mithril relays with `--number-of-aggregators` (requires `--use-relays` as soon as more +than one aggregator is used): + +```bash +./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../cardano-devnet --number-of-aggregators 3 --use-relays +``` + +### DMQ + +Use the DMQ protocol to broadcast signatures with `--use-dmq`: + +```bash +./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../cardano-devnet --use-dmq +``` + +By default, this uses the Haskell DMQ node created within the `cardano-devnet`. Use `--dmq-node-flavor fake` instead to +use a fake DMQ network created by the Mithril relay. + +### IPFS + +Enable upload/download of Cardano DB snapshots through IPFS with `--use-ipfs`: + +```bash +./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../cardano-devnet --use-ipfs +``` + +By default, a fresh IPFS devnet is spawned in the working directory and stopped at the end of the run. To attach to an +already-running IPFS devnet instead (started and left running independently of the test run), use `--ipfs-devnet-to-attach `. +The target swarm must have at least one node per aggregator, plus one for the client. + ### Note for MacOS users #### `sed` compatibility diff --git a/mithril-test-lab/mithril-end-to-end/src/devnet/runner.rs b/mithril-test-lab/mithril-end-to-end/src/devnet/cardano.rs similarity index 98% rename from mithril-test-lab/mithril-end-to-end/src/devnet/runner.rs rename to mithril-test-lab/mithril-end-to-end/src/devnet/cardano.rs index 6fcd047bb7d..ef60976c4ae 100644 --- a/mithril-test-lab/mithril-end-to-end/src/devnet/runner.rs +++ b/mithril-test-lab/mithril-end-to-end/src/devnet/cardano.rs @@ -4,18 +4,14 @@ use std::fs::{self, File, read_to_string}; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Stdio; -use thiserror::Error; use tokio::process::Command; use mithril_common::StdResult; use mithril_common::entities::{BlockHash, PartyId, TransactionHash}; +use crate::RetryableDevnetError; use crate::utils::{ChildLoggerExt, file_utils}; -#[derive(Error, Debug, PartialEq, Eq)] -#[error("Retryable devnet error: `{0}`")] -pub struct RetryableDevnetError(pub String); - #[derive(Debug, Clone, Default)] pub struct Devnet { artifacts_dir: PathBuf, @@ -436,10 +432,10 @@ impl Devnet { #[cfg(test)] mod tests { - use crate::devnet::DevnetTopology; - use crate::devnet::runner::{Devnet, FullNode, PoolNode}; use std::path::PathBuf; + use super::*; + #[test] pub fn yield_empty_topology_with_0_nodes() { let devnet = Devnet::new(PathBuf::new(), 0, 0); diff --git a/mithril-test-lab/mithril-end-to-end/src/devnet/ipfs.rs b/mithril-test-lab/mithril-end-to-end/src/devnet/ipfs.rs new file mode 100644 index 00000000000..70b47f33e53 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/devnet/ipfs.rs @@ -0,0 +1,220 @@ +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use anyhow::{Context, anyhow}; +use reqwest::Url; +use slog_scope::info; +use tokio::process::Command; + +use mithril_common::StdResult; + +use crate::RetryableDevnetError; +use crate::utils::{ChildLoggerExt, file_utils}; + +const IPFS_DEVNET_SCRIPT_NAME: &str = "ipfs-devnet.sh"; + +#[derive(Debug, Copy, Clone, Default)] +pub enum IpfsDevnetMode { + /// Spawn an IPFS devnet (default, only alive during the tests). + #[default] + Spawn, + /// Attach to an existing IPFS devnet. + Detached, +} + +#[derive(Debug, Clone, Default)] +pub struct IpfsDevnet { + devnet_script_path: PathBuf, + swarm_dir: PathBuf, + topology: Vec, + mode: IpfsDevnetMode, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KuboNode { + pub rpc_url: Url, + pub working_dir: PathBuf, +} + +#[derive(Debug, Clone)] +pub struct IpfsDevnetBootstrapArgs { + pub devnet_scripts_dir: PathBuf, + pub number_of_nodes: u8, + pub swarm_target_dir: PathBuf, + pub kubo_version: Option, + pub mode: IpfsDevnetMode, +} + +impl IpfsDevnet { + fn build_command>(&self, sub_command: C) -> StdResult { + let mut command = Command::new(self.devnet_script_path.clone()); + command + .arg(sub_command) + .env("SWARM_DIR", self.swarm_dir.as_os_str()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + Ok(command) + } + + fn build_topology(number_of_nodes: u8, swarm_dir: &Path) -> Vec { + (1..=number_of_nodes) + .map(|n| KuboNode { + rpc_url: Url::parse(&format!("http://127.0.0.1:{}", 5000_u16 + n as u16)).unwrap(), + working_dir: swarm_dir.join(format!("kubo-node-{n}")), + }) + .collect() + } + + pub async fn bootstrap(bootstrap_args: &IpfsDevnetBootstrapArgs) -> StdResult { + match bootstrap_args.mode { + IpfsDevnetMode::Spawn => Self::bootstrap_new_network(bootstrap_args).await, + IpfsDevnetMode::Detached => Self::bootstrap_attached(bootstrap_args), + } + } + + fn bootstrap_attached(bootstrap_args: &IpfsDevnetBootstrapArgs) -> StdResult { + let devnet_script_path = file_utils::get_process_path( + IPFS_DEVNET_SCRIPT_NAME, + &bootstrap_args.devnet_scripts_dir, + )?; + let swarm_dir = bootstrap_args.swarm_target_dir.to_owned(); + let swarm_dir_entries: Vec<_> = swarm_dir + .read_dir() + .with_context(|| format!("Failed to read swarm directory: '{}'", swarm_dir.display()))? + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) + .collect(); + + for i in 1..=bootstrap_args.number_of_nodes { + let expected_node = format!("kubo-node-{i}"); + if !swarm_dir_entries + .iter() + .any(|e| e.file_name().to_string_lossy() == expected_node) + { + anyhow::bail!( + "Expected node '{}' missing in attached swarm directory ('{}'), please re-initialize the devnet with at least {} nodes", + expected_node, + swarm_dir.display(), + bootstrap_args.number_of_nodes + ); + } + } + + Ok(IpfsDevnet { + devnet_script_path, + swarm_dir: bootstrap_args.swarm_target_dir.to_owned(), + topology: Self::build_topology( + bootstrap_args.number_of_nodes, + &bootstrap_args.swarm_target_dir, + ), + mode: bootstrap_args.mode, + }) + } + + async fn bootstrap_new_network( + bootstrap_args: &IpfsDevnetBootstrapArgs, + ) -> StdResult { + let devnet_script_path = file_utils::get_process_path( + IPFS_DEVNET_SCRIPT_NAME, + &bootstrap_args.devnet_scripts_dir, + )?; + + let mut bootstrap_command = Command::new(&devnet_script_path); + bootstrap_command + .arg("init") + .arg("--overwrite") + .arg("--number") + .arg(bootstrap_args.number_of_nodes.to_string()) + .arg("--swarm-dir") + .arg(bootstrap_args.swarm_target_dir.as_os_str()); + + if let Some(kubo_version) = &bootstrap_args.kubo_version { + bootstrap_command.env("KUBO_VERSION", kubo_version.to_string()); + } + + bootstrap_command + .current_dir(&bootstrap_args.devnet_scripts_dir) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + info!("Bootstrapping the IPFS Devnet"; "script" => &devnet_script_path.display(), "cmd" => "init"); + + let exit_status = bootstrap_command + .spawn() + .with_context(|| format!("{IPFS_DEVNET_SCRIPT_NAME} failed to start"))? + .wait_forwarding_output_to_slog_scope(IPFS_DEVNET_SCRIPT_NAME) + .await + .with_context(|| format!("{IPFS_DEVNET_SCRIPT_NAME} failed to run"))?; + match exit_status.code() { + Some(0) => Ok(IpfsDevnet { + devnet_script_path, + swarm_dir: bootstrap_args.swarm_target_dir.to_owned(), + topology: Self::build_topology( + bootstrap_args.number_of_nodes, + &bootstrap_args.swarm_target_dir, + ), + mode: bootstrap_args.mode, + }), + Some(code) => Err(anyhow!(RetryableDevnetError(format!( + "IPFS Bootstrap devnet exited with status code: {code}" + )))), + None => Err(anyhow!("IPFS Bootstrap devnet terminated by signal")), + } + } + + pub fn swarm_dir(&self) -> PathBuf { + self.swarm_dir.clone() + } + + pub fn topology(&self) -> &[KuboNode] { + &self.topology + } + + pub async fn start(&self) -> StdResult<()> { + // Note: running the start command on an already running devnet does nothing, so running it + // against a "Detached" devnet poses no risk (if stopped, it will start, if running, it will do nothing). + let mut run_command = self.build_command("start")?; + + info!("Starting the IPFS devnet"; "script" => &self.devnet_script_path.display(), "cmd" => "start"); + + let status = run_command + .spawn() + .with_context(|| "Failed to start the IPFS devnet")? + .wait_forwarding_output_to_slog_scope(&format!("{IPFS_DEVNET_SCRIPT_NAME} start")) + .await + .with_context(|| "Error while starting the IPFS devnet")?; + match status.code() { + Some(0) => Ok(()), + Some(code) => Err(anyhow!(RetryableDevnetError(format!( + "Run IPFS devnet exited with status code: {code}" + )))), + None => Err(anyhow!("Run IPFS devnet terminated by signal")), + } + } + + pub async fn stop(&self) -> StdResult<()> { + if matches!(self.mode, IpfsDevnetMode::Detached) { + info!("IPFS devnet is in detached mode, leaving it running"); + return Ok(()); + } + + let mut run_command = self.build_command("stop")?; + + info!("Stopping the IPFS devnet"; "script" => &self.devnet_script_path.display(), "cmd" => "stop"); + + let exit_status = run_command + .spawn() + .with_context(|| "Failed to stop the IPFS devnet")? + .wait_forwarding_output_to_slog_scope(&format!("{IPFS_DEVNET_SCRIPT_NAME} stop")) + .await + .with_context(|| "Error while stopping the IPFS devnet")?; + match exit_status.code() { + Some(0) => Ok(()), + Some(code) => Err(anyhow!("Stop IPFS devnet exited with status code: {code}")), + None => Err(anyhow!("Stop IPFS devnet terminated by signal")), + } + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/devnet/mod.rs b/mithril-test-lab/mithril-end-to-end/src/devnet/mod.rs index 249d6073155..9a34dd90621 100644 --- a/mithril-test-lab/mithril-end-to-end/src/devnet/mod.rs +++ b/mithril-test-lab/mithril-end-to-end/src/devnet/mod.rs @@ -1,5 +1,9 @@ -mod runner; +mod cardano; +mod ipfs; -pub use runner::{ - Devnet, DevnetBootstrapArgs, DevnetTopology, FullNode, PoolNode, RetryableDevnetError, -}; +pub use cardano::{Devnet, DevnetBootstrapArgs, DevnetTopology, FullNode, PoolNode}; +pub use ipfs::{IpfsDevnet, IpfsDevnetBootstrapArgs, IpfsDevnetMode, KuboNode}; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +#[error("Retryable devnet error: `{0}`")] +pub struct RetryableDevnetError(pub String); diff --git a/mithril-test-lab/mithril-end-to-end/src/lib.rs b/mithril-test-lab/mithril-end-to-end/src/lib.rs index d0b1194e33f..992631465c8 100644 --- a/mithril-test-lab/mithril-end-to-end/src/lib.rs +++ b/mithril-test-lab/mithril-end-to-end/src/lib.rs @@ -10,6 +10,7 @@ pub use mithril::*; pub use utils::{CompatibilityChecker, CompatibilityCheckerError, NodeVersion}; use clap::ValueEnum; + /// The flavor of DMQ node to use in the tests. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] pub enum DmqNodeFlavor { diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 1a43db67315..4f07eca3feb 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -27,15 +27,13 @@ use mithril_common::{ messages::SignedEntityTypeDiscriminantsMessage, }; use mithril_doc::GenerateDocCommands; +use mithril_end_to_end::scenario::{FullScenario, MinimalScenario, RunOnlyScenario}; use mithril_end_to_end::toolkit::{ScenarioToolkit, ScenarioToolkitContext}; use mithril_end_to_end::{ AggregateSignatureType, Aggregator, Client, CompatibilityChecker, CompatibilityCheckerError, - Devnet, DevnetBootstrapArgs, DmqNodeFlavor, MithrilInfrastructure, MithrilInfrastructureConfig, - NodeVersion, RelaySigner, RetryableDevnetError, Signer, -}; -use mithril_end_to_end::{ - ProtocolConfiguration, - scenario::{FullScenario, MinimalScenario, RunOnlyScenario}, + Devnet, DevnetBootstrapArgs, DmqNodeFlavor, IpfsDevnet, IpfsDevnetBootstrapArgs, + IpfsDevnetMode, MithrilInfrastructure, MithrilInfrastructureConfig, NodeVersion, + ProtocolConfiguration, RelaySigner, RetryableDevnetError, Signer, }; /// Default signed entity types used by scenarios that support multiple entities, such as Full and RunOnly. @@ -187,6 +185,23 @@ struct NetworkTopologyArgs { /// Haskell DMQ node version #[clap(long)] dmq_node_version: Option, + + /// Enable upload and download to an IPFS devnet + #[clap(long)] + use_ipfs: bool, + + /// Path to an existing IPFS devnet swarm to attach to. + /// + /// The target devnet must have at least one node per aggregator plus one for the client. + /// + /// If set, the target devnet will be attached to, started if needed, and left running after the test. + /// If unset, a new IPFS devnet will be created in the end-to-end working directory. + #[clap(long, requires = "use_ipfs")] + ipfs_devnet_to_attach: Option, + + /// Directory containing scripts to bootstrap an IPFS devnet + #[clap(long, default_value = "./ipfs_devnet")] + ipfs_devnet_scripts_directory: PathBuf, } #[derive(Args, Debug, Clone)] @@ -421,6 +436,7 @@ impl From> for AppResult { struct App { devnet: Arc>>, + ipfs_devnet: Arc>>, infrastructure: Arc>>>, } @@ -428,6 +444,7 @@ impl App { fn new() -> Self { Self { devnet: Arc::new(Mutex::new(None)), + ipfs_devnet: Arc::new(Mutex::new(None)), infrastructure: Arc::new(Mutex::new(None)), } } @@ -512,6 +529,29 @@ impl App { .await?; *self.devnet.lock().await = Some(devnet.clone()); + let ipfs_devnet = if args.network_topology.use_ipfs { + let (mode, swarm_target_dir) = + if let Some(swarm_dir) = args.network_topology.ipfs_devnet_to_attach { + (IpfsDevnetMode::Detached, swarm_dir) + } else { + (IpfsDevnetMode::Spawn, work_dir.join("ipfs_devnet")) + }; + + let devnet = IpfsDevnet::bootstrap(&IpfsDevnetBootstrapArgs { + devnet_scripts_dir: args.network_topology.ipfs_devnet_scripts_directory, + // One per aggregator + one for the Mithril Client + number_of_nodes: args.network_topology.number_of_aggregators + 1, + swarm_target_dir, + kubo_version: None, + mode, + }) + .await?; + Some(devnet) + } else { + None + }; + *self.ipfs_devnet.lock().await = ipfs_devnet.clone(); + let startup_protocol_configuration = Self::build_startup_protocol_configuration(&args.mithril.aggregate_signature_type); @@ -523,6 +563,7 @@ impl App { number_of_signers: args.network_topology.number_of_signers, server_port, devnet: devnet.clone(), + ipfs_devnet: ipfs_devnet.clone(), work_dir, store_dir, artifacts_dir, @@ -639,6 +680,7 @@ impl App { struct AppStopper { devnet: Arc>>, + ipfs_devnet: Arc>>, infrastructure: Arc>>>, } @@ -646,6 +688,7 @@ impl AppStopper { pub fn new(app: &App) -> Self { Self { devnet: app.devnet.clone(), + ipfs_devnet: app.ipfs_devnet.clone(), infrastructure: app.infrastructure.clone(), } } @@ -658,7 +701,12 @@ impl AppStopper { } if let Some(devnet) = self.devnet.lock().await.as_ref() { let _ = devnet.stop().await.inspect_err(|e| { - error!("Failed to stop devnet: {}", e); + error!("Failed to stop Cardano devnet: {}", e); + }); + } + if let Some(ipfs_devnet) = self.ipfs_devnet.lock().await.as_ref() { + let _ = ipfs_devnet.stop().await.inspect_err(|e| { + error!("Failed to stop IPFS devnet: {}", e); }); } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 2b8ae3e1a4f..bb91ae19477 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -1,5 +1,4 @@ use std::cmp; -use std::collections::HashMap; use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -16,12 +15,13 @@ use tokio::sync::RwLock; use mithril_cardano_node_chain::chain_observer::{ChainObserver, PallasChainObserver}; use mithril_common::{CardanoNetwork, StdResult, entities}; -use crate::utils::{MithrilCommand, NodeVersion}; +use crate::utils::{EnvVars, MithrilCommand, NodeVersion}; use crate::{ ANCILLARY_MANIFEST_SECRET_KEY, AggregateSignatureType, DEVNET_DMQ_MAGIC_ID, DEVNET_MAGIC_ID, DmqNodeFlavor, ERA_MARKERS_SECRET_KEY, ERA_MARKERS_VERIFICATION_KEY, FullNode, - GENESIS_SECRET_KEY, GENESIS_VERIFICATION_KEY, PROTOCOL_CONFIGURATION_MARKERS_SECRET_KEY, - PROTOCOL_CONFIGURATION_MARKERS_VERIFICATION_KEY, RetryableDevnetError, + GENESIS_SECRET_KEY, GENESIS_VERIFICATION_KEY, KuboNode, + PROTOCOL_CONFIGURATION_MARKERS_SECRET_KEY, PROTOCOL_CONFIGURATION_MARKERS_VERIFICATION_KEY, + RetryableDevnetError, }; #[derive(Debug)] @@ -30,6 +30,7 @@ pub struct AggregatorConfig<'a> { pub name: &'a str, pub server_port: u64, pub full_node: &'a FullNode, + pub ipfs_kubo_node: Option<&'a KuboNode>, pub cardano_cli_path: &'a Path, pub work_dir: &'a Path, pub store_dir: &'a Path, @@ -106,7 +107,7 @@ impl Aggregator { let public_server_url = format!("http://localhost:{server_port_parameter}/aggregator"); let cardano_node_version = aggregator_config.cardano_node_version.to_string(); let aggregate_signature_type = aggregator_config.aggregate_signature_type.to_string(); - let mut env = HashMap::from([ + let mut env = EnvVars::from([ ("NETWORK", "devnet"), ("NETWORK_MAGIC", &magic_id), ("DMQ_NETWORK_MAGIC", &dmq_magic_id), @@ -193,6 +194,16 @@ impl Aggregator { if aggregator_config.use_dmq { env.insert("DMQ_NODE_SOCKET_PATH", dmq_node_socket_path.as_str()); } + if let Some(node) = aggregator_config.ipfs_kubo_node { + env.insert( + "IPFS_RPC_SERVER_CONFIG", + format!( + r#"{{"url": "{}", "mfs_folder_name": "mithril-snapshots"}}"#, + node.rpc_url + ), + ); + } + let args = vec![ "--db-directory", aggregator_config.full_node.db_path.to_str().unwrap(), diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/client.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/client.rs index 3d977691b92..3cb527d6d9f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/client.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/client.rs @@ -1,4 +1,5 @@ use anyhow::{Context, anyhow}; +use reqwest::Url; use slog_scope::warn; use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -18,9 +19,16 @@ pub struct Client { #[derive(Debug)] pub enum CardanoDbV2Command { List, - ListPerEpoch { epoch_specifier: EpochSpecifier }, - Show { hash: String }, - Download { hash: String }, + ListPerEpoch { + epoch_specifier: EpochSpecifier, + }, + Show { + hash: String, + }, + Download { + hash: String, + ipfs_rpc_url: Option, + }, } impl CardanoDbV2Command { @@ -31,7 +39,13 @@ impl CardanoDbV2Command { format!("list-epoch-{epoch_specifier}") } CardanoDbV2Command::Show { hash } => format!("show-{hash}"), - CardanoDbV2Command::Download { hash } => format!("download-{hash}"), + CardanoDbV2Command::Download { hash, ipfs_rpc_url } => { + if ipfs_rpc_url.is_some() { + format!("download-ipfs-{hash}") + } else { + format!("download-{hash}") + } + } } } @@ -51,15 +65,20 @@ impl CardanoDbV2Command { CardanoDbV2Command::Show { hash } => { vec!["snapshot".to_string(), "show".to_string(), hash.clone()] } - CardanoDbV2Command::Download { hash } => { - vec![ + CardanoDbV2Command::Download { hash, ipfs_rpc_url } => { + let mut args = vec![ "download".to_string(), "--include-ancillary".to_string(), "--allow-override".to_string(), "--download-dir".to_string(), "v2".to_string(), hash.clone(), - ] + ]; + if let Some(ipfs_rpc_url) = ipfs_rpc_url { + args.push("--ipfs-rpc-url".to_string()); + args.push(ipfs_rpc_url.to_string()); + } + args } } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index b5e1204648b..3f3f81d7030 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -18,7 +18,8 @@ use crate::mithril::relay_signer::RelaySignerConfiguration; use crate::toolkit::ScenarioToolkit; use crate::{ AggregateSignatureType, Aggregator, AggregatorConfig, Client, DEVNET_MAGIC_ID, Devnet, - DmqNodeFlavor, FullNode, PoolNode, RelayAggregator, RelayPassive, RelaySigner, Signer, + DmqNodeFlavor, FullNode, IpfsDevnet, KuboNode, PoolNode, RelayAggregator, RelayPassive, + RelaySigner, Signer, }; use super::signer::SignerConfig; @@ -37,6 +38,7 @@ pub struct MithrilInfrastructureConfig { pub number_of_signers: u8, pub server_port: u64, pub devnet: Devnet, + pub ipfs_devnet: Option, pub work_dir: PathBuf, pub store_dir: PathBuf, pub artifacts_dir: PathBuf, @@ -78,6 +80,7 @@ impl MithrilInfrastructureConfig { number_of_signers: 1, server_port: 8080, devnet: Devnet::default(), + ipfs_devnet: None, work_dir: PathBuf::from("/tmp/work"), store_dir: PathBuf::from("/tmp/store"), artifacts_dir: PathBuf::from("/tmp/artifacts"), @@ -116,6 +119,7 @@ pub struct MithrilInfrastructure { artifacts_dir: PathBuf, bin_dir: PathBuf, devnet: Devnet, + ipfs_devnet: Option, aggregators: Vec, signers: Vec, relay_aggregators: Vec, @@ -140,6 +144,10 @@ impl MithrilInfrastructure { if config.use_dmq && config.dmq_node_flavor == Some(DmqNodeFlavor::Haskell) { config.devnet.run_dmq().await?; } + if let Some(ipfs_devnet) = &config.ipfs_devnet { + ipfs_devnet.start().await?; + } + let devnet_topology = config.devnet.topology(); let aggregator_cardano_nodes = &devnet_topology.full_nodes; let signer_cardano_nodes = &devnet_topology.pool_nodes; @@ -150,9 +158,13 @@ impl MithrilInfrastructure { let relay_signer_registration_mode = &config.relay_signer_registration_mode; let relay_signature_registration_mode = &config.relay_signature_registration_mode; - let (leader_aggregator, follower_aggregators) = - Self::prepare_aggregators(config, aggregator_cardano_nodes, chain_observer_type) - .await?; + let (leader_aggregator, follower_aggregators) = Self::prepare_aggregators( + config, + aggregator_cardano_nodes, + config.ipfs_devnet.as_ref().map(|d| d.topology()), + chain_observer_type, + ) + .await?; if leader_aggregator.is_reading_protocol_configurations_on_chain() { Self::register_startup_protocol_configurations( @@ -211,6 +223,7 @@ impl MithrilInfrastructure { bin_dir: config.bin_dir.to_path_buf(), artifacts_dir: config.artifacts_dir.to_path_buf(), devnet: config.devnet.clone(), + ipfs_devnet: config.ipfs_devnet.clone(), aggregators: all_aggregators, signers, relay_aggregators, @@ -285,13 +298,21 @@ impl MithrilInfrastructure { async fn prepare_aggregators( config: &MithrilInfrastructureConfig, full_nodes: &[FullNode], + ipfs_devnet: Option<&[KuboNode]>, chain_observer_type: &str, ) -> StdResult<(Aggregator, Vec)> { let [leader_node, follower_nodes @ ..] = full_nodes else { panic!("Can't prepare Aggregators: No full nodes found"); }; - let leader_aggregator = - Self::prepare_aggregator(0, leader_node, config, chain_observer_type, None).await?; + let leader_aggregator = Self::prepare_aggregator( + 0, + leader_node, + config, + chain_observer_type, + ipfs_devnet.and_then(|n| n.first()), + None, + ) + .await?; let mut follower_aggregators = vec![]; for (index, full_node) in follower_nodes.iter().enumerate() { @@ -300,6 +321,7 @@ impl MithrilInfrastructure { full_node, config, chain_observer_type, + ipfs_devnet.and_then(|n| n.get(index + 1)), Some(leader_aggregator.endpoint()), ) .await?; @@ -314,6 +336,7 @@ impl MithrilInfrastructure { full_node: &FullNode, config: &MithrilInfrastructureConfig, chain_observer_type: &str, + ipfs_kubo_node: Option<&KuboNode>, leader_aggregator_endpoint: Option, ) -> StdResult { let aggregator_name = Aggregator::name_suffix(index); @@ -326,6 +349,7 @@ impl MithrilInfrastructure { name: &aggregator_name, server_port: config.server_port + index as u64, full_node, + ipfs_kubo_node, cardano_cli_path: &config.devnet.cardano_cli_path(), work_dir: &config.work_dir, store_dir: &aggregator_store_dir, @@ -582,6 +606,10 @@ impl MithrilInfrastructure { &self.relay_passives } + pub fn client_ipfs_node(&self) -> Option<&KuboNode> { + self.ipfs_devnet.as_ref().and_then(|d| d.topology().last()) + } + pub fn chain_observer(&self) -> Arc { self.cardano_chain_observer.clone() } diff --git a/mithril-test-lab/mithril-end-to-end/src/scenario/full.rs b/mithril-test-lab/mithril-end-to-end/src/scenario/full.rs index afa44d64c8d..f0eea5c8caa 100644 --- a/mithril-test-lab/mithril-end-to-end/src/scenario/full.rs +++ b/mithril-test-lab/mithril-end-to-end/src/scenario/full.rs @@ -307,6 +307,7 @@ impl FullScenario { .is_certified_and_verified( aggregator, &mut client, + infrastructure.client_ipfs_node().map(|n| n.rpc_url.clone()), expected_epoch_min, infrastructure.signers().len(), ) diff --git a/mithril-test-lab/mithril-end-to-end/src/scenario/minimal.rs b/mithril-test-lab/mithril-end-to-end/src/scenario/minimal.rs index e97fc276d8e..2757cb3e66b 100644 --- a/mithril-test-lab/mithril-end-to-end/src/scenario/minimal.rs +++ b/mithril-test-lab/mithril-end-to-end/src/scenario/minimal.rs @@ -185,6 +185,7 @@ impl MinimalScenario { .is_certified_and_verified( aggregator, &mut client, + infrastructure.client_ipfs_node().map(|n| n.rpc_url.clone()), expected_epoch_min, infrastructure.signers().len(), ) diff --git a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs index adc519b1a63..c307ffb729c 100644 --- a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs +++ b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs @@ -30,6 +30,7 @@ pub async fn bootstrap_aggregator( name: "genesis", server_port: args.server_port as u64, full_node: &args.full_node, + ipfs_kubo_node: None, cardano_cli_path: &args.cardano_cli_path, work_dir: &args.work_dir, store_dir: &args.work_dir.join("aggregator_store"), diff --git a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs index 6f4865ab40f..eac8340f64f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs +++ b/mithril-test-lab/mithril-end-to-end/src/toolkit/check/cardano_database.rs @@ -1,4 +1,5 @@ use anyhow::{Context, anyhow}; +use reqwest::Url; use slog_scope::{info, warn}; use mithril_common::{ @@ -29,6 +30,7 @@ impl CheckCardanoDatabaseToolkit { &self, aggregator: &Aggregator, client: &mut Client, + ipfs_rpc_url: Option, expected_epoch_min: Epoch, total_signers_expected: usize, ) -> StdResult<()> { @@ -43,7 +45,7 @@ impl CheckCardanoDatabaseToolkit { ) .await?; self.node_producing_cardano_database_digests_map(aggregator).await?; - self.verify_with_client(client, &artifact.hash).await?; + self.verify_with_client(client, ipfs_rpc_url, &artifact.hash).await?; Ok(()) } @@ -116,7 +118,12 @@ impl CheckCardanoDatabaseToolkit { }) } - pub async fn verify_with_client(&self, client: &mut Client, hash: &str) -> StdResult<()> { + pub async fn verify_with_client( + &self, + client: &mut Client, + ipfs_rpc_url: Option, + hash: &str, + ) -> StdResult<()> { client .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::List)) .await?; @@ -145,9 +152,20 @@ impl CheckCardanoDatabaseToolkit { client .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Download { hash: hash.to_string(), + ipfs_rpc_url: None, })) .await?; - info!("Client downloaded & restored the cardano database snapshot"; "hash" => &hash); + info!("Client downloaded & restored the cardano database snapshot"; "hash" => &hash, "storage_source" => "local"); + + if ipfs_rpc_url.is_some() { + client + .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Download { + hash: hash.to_string(), + ipfs_rpc_url, + })) + .await?; + info!("Client downloaded & restored the cardano database snapshot"; "hash" => &hash, "storage_source" => "ipfs"); + } Ok(()) }