test(e2e): IPFS support

.github/workflows/ci.yml

@@ -413,6 +413,13 @@ jobs:
             hard_fork_latest_era_at_epoch: 0
             run_id: "#1"
             extra_args: "--mithril-era-regenesis-on-switch --aggregate-signature-type=Concatenation"
+          # Include a test for IPFS upload/download support
+          - mode: "ipfs"
+            era: ${{ fromJSON(needs.build-ubuntu.outputs.eras)[0] }}
+            cardano_node_version: "11.0.1"
+            hard_fork_latest_era_at_epoch: 0
+            run_id: "#1"
+            extra_args: "--use-ipfs --ipfs-devnet-scripts-directory=./mithril-test-lab/ipfs-devnet/"
     steps:
       - name: Checkout sources
         uses: actions/checkout@v6

mithril-test-lab/ipfs-devnet/README.md

@@ -148,9 +148,9 @@ For node `N`, the default local ports are:
 
 | Service | Port formula | Example for node 1 |
 | ------- | ------------ | ------------------ |
-| Swarm   | `4000 + N`   | `4001`             |
 | API     | `5000 + N`   | `5001`             |
-| Gateway | `8080 + N`   | `8081`             |
+| Gateway | `5100 + N`   | `5101`             |
+| Swarm   | `5200 + N`   | `5201`             |
 
 #### Re-initialize an existing swarm
 

mithril-test-lab/ipfs-devnet/commands/mkfiles/kubo-configure-swarm.sh

@@ -67,8 +67,8 @@ configure_node() {
 
   local api_port gateway_port swarm_port
   api_port=$((5000 + node_id))
-  gateway_port=$((8080 + node_id))
-  swarm_port=$((4000 + node_id))
+  gateway_port=$((5100 + node_id))
+  swarm_port=$((5200 + node_id))
 
   #---------- Write swarm key
   {

mithril-test-lab/mithril-end-to-end/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-end-to-end"
-version = "0.5.15"
+version = "0.5.16"
 authors = { workspace = true }
 edition = { workspace = true }
 documentation = { workspace = true }

mithril-test-lab/mithril-end-to-end/README.md

@@ -49,6 +49,43 @@ To run `mithril-end-to-end` command, you must first compile the Mithril nodes:
 cargo build --release
 ```
 
+## Additional network topology options
+
+These options can be combined with the base run command shown above (see `./mithril-end-to-end --help` for the full
+list of flags).
+
+### Multiple aggregators
+
+Run several aggregators behind Mithril relays with `--number-of-aggregators` (requires `--use-relays` as soon as more
+than one aggregator is used):
+
+```bash
+./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../cardano-devnet --number-of-aggregators 3 --use-relays
+```
+
+### DMQ
+
+Use the DMQ protocol to broadcast signatures with `--use-dmq`:
+
+```bash
+./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../cardano-devnet --use-dmq
+```
+
+By default, this uses the Haskell DMQ node created within the `cardano-devnet`. Use `--dmq-node-flavor fake` instead to
+use a fake DMQ network created by the Mithril relay.
+
+### IPFS
+
+Enable upload/download of Cardano DB snapshots through IPFS with `--use-ipfs`:
+
+```bash
+./mithril-end-to-end -vvv --work-directory db/ --bin-directory ../../target/release --devnet-scripts-directory=../cardano-devnet --use-ipfs
+```
+
+By default, a fresh IPFS devnet is spawned in the working directory and stopped at the end of the run. To attach to an
+already-running IPFS devnet instead (started and left running independently of the test run), use `--ipfs-devnet-to-attach <path>`.
+The target swarm must have at least one node per aggregator, plus one for the client.
+
 ### Note for MacOS users
 
 #### `sed` compatibility

mithril-test-lab/mithril-end-to-end/src/devnet/runner.rs → mithril-test-lab/mithril-end-to-end/src/devnet/cardano.rs

@@ -4,18 +4,14 @@ use std::fs::{self, File, read_to_string};
 use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
-use thiserror::Error;
 use tokio::process::Command;
 
 use mithril_common::StdResult;
 use mithril_common::entities::{BlockHash, PartyId, TransactionHash};
 
+use crate::RetryableDevnetError;
 use crate::utils::{ChildLoggerExt, file_utils};
 
-#[derive(Error, Debug, PartialEq, Eq)]
-#[error("Retryable devnet error: `{0}`")]
-pub struct RetryableDevnetError(pub String);
-
 #[derive(Debug, Clone, Default)]
 pub struct Devnet {
     artifacts_dir: PathBuf,
@@ -436,10 +432,10 @@ impl Devnet {
 
 #[cfg(test)]
 mod tests {
-    use crate::devnet::DevnetTopology;
-    use crate::devnet::runner::{Devnet, FullNode, PoolNode};
     use std::path::PathBuf;
 
+    use super::*;
+
     #[test]
     pub fn yield_empty_topology_with_0_nodes() {
         let devnet = Devnet::new(PathBuf::new(), 0, 0);

mithril-test-lab/mithril-end-to-end/src/devnet/ipfs.rs

@@ -0,0 +1,220 @@
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+use std::process::Stdio;
+
+use anyhow::{Context, anyhow};
+use reqwest::Url;
+use slog_scope::info;
+use tokio::process::Command;
+
+use mithril_common::StdResult;
+
+use crate::RetryableDevnetError;
+use crate::utils::{ChildLoggerExt, file_utils};
+
+const IPFS_DEVNET_SCRIPT_NAME: &str = "ipfs-devnet.sh";
+
+#[derive(Debug, Copy, Clone, Default)]
+pub enum IpfsDevnetMode {
+    /// Spawn an IPFS devnet (default, only alive during the tests).
+    #[default]
+    Spawn,
+    /// Attach to an existing IPFS devnet.
+    Detached,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct IpfsDevnet {
+    devnet_script_path: PathBuf,
+    swarm_dir: PathBuf,
+    topology: Vec<KuboNode>,
+    mode: IpfsDevnetMode,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct KuboNode {
+    pub rpc_url: Url,
+    pub working_dir: PathBuf,
+}
+
+#[derive(Debug, Clone)]
+pub struct IpfsDevnetBootstrapArgs {
+    pub devnet_scripts_dir: PathBuf,
+    pub number_of_nodes: u8,
+    pub swarm_target_dir: PathBuf,
+    pub kubo_version: Option<semver::Version>,
+    pub mode: IpfsDevnetMode,
+}
+
+impl IpfsDevnet {
+    fn build_command<C: AsRef<OsStr>>(&self, sub_command: C) -> StdResult<Command> {
+        let mut command = Command::new(self.devnet_script_path.clone());
+        command
+            .arg(sub_command)
+            .env("SWARM_DIR", self.swarm_dir.as_os_str())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .kill_on_drop(true);
+        Ok(command)
+    }
+
+    fn build_topology(number_of_nodes: u8, swarm_dir: &Path) -> Vec<KuboNode> {
+        (1..=number_of_nodes)
+            .map(|n| KuboNode {
+                rpc_url: Url::parse(&format!("http://127.0.0.1:{}", 5000_u16 + n as u16)).unwrap(),
+                working_dir: swarm_dir.join(format!("kubo-node-{n}")),
+            })
+            .collect()
+    }
+
+    pub async fn bootstrap(bootstrap_args: &IpfsDevnetBootstrapArgs) -> StdResult<IpfsDevnet> {
+        match bootstrap_args.mode {
+            IpfsDevnetMode::Spawn => Self::bootstrap_new_network(bootstrap_args).await,
+            IpfsDevnetMode::Detached => Self::bootstrap_attached(bootstrap_args),
+        }
+    }
+
+    fn bootstrap_attached(bootstrap_args: &IpfsDevnetBootstrapArgs) -> StdResult<IpfsDevnet> {
+        let devnet_script_path = file_utils::get_process_path(
+            IPFS_DEVNET_SCRIPT_NAME,
+            &bootstrap_args.devnet_scripts_dir,
+        )?;
+        let swarm_dir = bootstrap_args.swarm_target_dir.to_owned();
+        let swarm_dir_entries: Vec<_> = swarm_dir
+            .read_dir()
+            .with_context(|| format!("Failed to read swarm directory: '{}'", swarm_dir.display()))?
+            .filter_map(|e| e.ok())
+            .filter(|e| e.file_type().is_ok_and(|t| t.is_dir()))
+            .collect();
+
+        for i in 1..=bootstrap_args.number_of_nodes {
+            let expected_node = format!("kubo-node-{i}");
+            if !swarm_dir_entries
+                .iter()
+                .any(|e| e.file_name().to_string_lossy() == expected_node)
+            {
+                anyhow::bail!(
+                    "Expected node '{}' missing in attached swarm directory ('{}'), please re-initialize the devnet with at least {} nodes",
+                    expected_node,
+                    swarm_dir.display(),
+                    bootstrap_args.number_of_nodes
+                );
+            }
+        }
+
+        Ok(IpfsDevnet {
+            devnet_script_path,
+            swarm_dir: bootstrap_args.swarm_target_dir.to_owned(),
+            topology: Self::build_topology(
+                bootstrap_args.number_of_nodes,
+                &bootstrap_args.swarm_target_dir,
+            ),
+            mode: bootstrap_args.mode,
+        })
+    }
+
+    async fn bootstrap_new_network(
+        bootstrap_args: &IpfsDevnetBootstrapArgs,
+    ) -> StdResult<IpfsDevnet> {
+        let devnet_script_path = file_utils::get_process_path(
+            IPFS_DEVNET_SCRIPT_NAME,
+            &bootstrap_args.devnet_scripts_dir,
+        )?;
+
+        let mut bootstrap_command = Command::new(&devnet_script_path);
+        bootstrap_command
+            .arg("init")
+            .arg("--overwrite")
+            .arg("--number")
+            .arg(bootstrap_args.number_of_nodes.to_string())
+            .arg("--swarm-dir")
+            .arg(bootstrap_args.swarm_target_dir.as_os_str());
+
+        if let Some(kubo_version) = &bootstrap_args.kubo_version {
+            bootstrap_command.env("KUBO_VERSION", kubo_version.to_string());
+        }
+
+        bootstrap_command
+            .current_dir(&bootstrap_args.devnet_scripts_dir)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .kill_on_drop(true);
+
+        info!("Bootstrapping the IPFS Devnet"; "script" => &devnet_script_path.display(), "cmd" => "init");
+
+        let exit_status = bootstrap_command
+            .spawn()
+            .with_context(|| format!("{IPFS_DEVNET_SCRIPT_NAME} failed to start"))?
+            .wait_forwarding_output_to_slog_scope(IPFS_DEVNET_SCRIPT_NAME)
+            .await
+            .with_context(|| format!("{IPFS_DEVNET_SCRIPT_NAME} failed to run"))?;
+        match exit_status.code() {
+            Some(0) => Ok(IpfsDevnet {
+                devnet_script_path,
+                swarm_dir: bootstrap_args.swarm_target_dir.to_owned(),
+                topology: Self::build_topology(
+                    bootstrap_args.number_of_nodes,
+                    &bootstrap_args.swarm_target_dir,
+                ),
+                mode: bootstrap_args.mode,
+            }),
+            Some(code) => Err(anyhow!(RetryableDevnetError(format!(
+                "IPFS Bootstrap devnet exited with status code: {code}"
+            )))),
+            None => Err(anyhow!("IPFS Bootstrap devnet terminated by signal")),
+        }
+    }
+
+    pub fn swarm_dir(&self) -> PathBuf {
+        self.swarm_dir.clone()
+    }
+
+    pub fn topology(&self) -> &[KuboNode] {
+        &self.topology
+    }
+
+    pub async fn start(&self) -> StdResult<()> {
+        // Note: running the start command on an already running devnet does nothing, so running it
+        // against a "Detached" devnet poses no risk (if stopped, it will start, if running, it will do nothing).
+        let mut run_command = self.build_command("start")?;
+
+        info!("Starting the IPFS devnet"; "script" => &self.devnet_script_path.display(), "cmd" => "start");
+
+        let status = run_command
+            .spawn()
+            .with_context(|| "Failed to start the IPFS devnet")?
+            .wait_forwarding_output_to_slog_scope(&format!("{IPFS_DEVNET_SCRIPT_NAME} start"))
+            .await
+            .with_context(|| "Error while starting the IPFS devnet")?;
+        match status.code() {
+            Some(0) => Ok(()),
+            Some(code) => Err(anyhow!(RetryableDevnetError(format!(
+                "Run IPFS devnet exited with status code: {code}"
+            )))),
+            None => Err(anyhow!("Run IPFS devnet terminated by signal")),
+        }
+    }
+
+    pub async fn stop(&self) -> StdResult<()> {
+        if matches!(self.mode, IpfsDevnetMode::Detached) {
+            info!("IPFS devnet is in detached mode, leaving it running");
+            return Ok(());
+        }
+
+        let mut run_command = self.build_command("stop")?;
+
+        info!("Stopping the IPFS devnet"; "script" => &self.devnet_script_path.display(), "cmd" => "stop");
+
+        let exit_status = run_command
+            .spawn()
+            .with_context(|| "Failed to stop the IPFS devnet")?
+            .wait_forwarding_output_to_slog_scope(&format!("{IPFS_DEVNET_SCRIPT_NAME} stop"))
+            .await
+            .with_context(|| "Error while stopping the IPFS devnet")?;
+        match exit_status.code() {
+            Some(0) => Ok(()),
+            Some(code) => Err(anyhow!("Stop IPFS devnet exited with status code: {code}")),
+            None => Err(anyhow!("Stop IPFS devnet terminated by signal")),
+        }
+    }
+}

mithril-test-lab/mithril-end-to-end/src/devnet/mod.rs

@@ -1,5 +1,9 @@
-mod runner;
+mod cardano;
+mod ipfs;
 
-pub use runner::{
-    Devnet, DevnetBootstrapArgs, DevnetTopology, FullNode, PoolNode, RetryableDevnetError,
-};
+pub use cardano::{Devnet, DevnetBootstrapArgs, DevnetTopology, FullNode, PoolNode};
+pub use ipfs::{IpfsDevnet, IpfsDevnetBootstrapArgs, IpfsDevnetMode, KuboNode};
+
+#[derive(thiserror::Error, Debug, PartialEq, Eq)]
+#[error("Retryable devnet error: `{0}`")]
+pub struct RetryableDevnetError(pub String);

mithril-test-lab/mithril-end-to-end/src/lib.rs

@@ -10,6 +10,7 @@ pub use mithril::*;
 pub use utils::{CompatibilityChecker, CompatibilityCheckerError, NodeVersion};
 
 use clap::ValueEnum;
+
 /// The flavor of DMQ node to use in the tests.
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
 pub enum DmqNodeFlavor {