better ipfs urls + small fixes

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs

@@ -112,7 +112,7 @@ impl ImmutableFilesUploader for IpfsUploader {
 
         Ok(ImmutablesLocation::Ipfs {
             uri: MultiFilesUri::Template(TemplateUri(format!(
-                "{directory_cid}/{{immutable_file_number}}.tar.zst"
+                "ipfs://{directory_cid}/{{immutable_file_number}}.tar.zst"
             ))),
             compression_algorithm,
         })
@@ -299,6 +299,7 @@ mod tests {
     use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
     use mithril_common::{
         entities::TemplateUri,
+        temp_dir_create,
         test::{TempDir, assert_equivalent, equivalent_to},
     };
     use mithril_file_archiver::FileArchiver;
@@ -887,6 +888,8 @@ mod tests {
     }
 
     mod batch_upload {
+        use std::collections::HashMap;
+
         use mithril_common::test::TempDir;
 
         use crate::file_uploaders::FileUploadRetryPolicy;
@@ -971,6 +974,35 @@ mod tests {
                 .await
                 .expect_err("Should return an error when not template found");
         }
+
+        #[tokio::test]
+        async fn ipfs_batch_upload_yield_ipfs_urls() {
+            let test_dir = temp_dir_create!();
+            let uploader = IpfsUploader::new_for_test("dir", move |mock| {
+                mock.expect_create_dir().returning(|_| Ok(()));
+                mock.expect_list_directory_files()
+                    .returning(move |_| Ok(HashMap::new()));
+                mock.expect_file_exists().never();
+                mock.expect_upload_file().returning(|_, _| Ok("file-cid".to_string()));
+                mock.expect_get_dir_cid()
+                    .returning(|_| Ok("directory-cid".to_string()));
+            });
+
+            let archive_1 = create_fake_archive(&test_dir, "00001.tar.zst");
+
+            let location = IpfsUploader::batch_upload(&uploader, &[archive_1], None)
+                .await
+                .unwrap();
+            assert_eq!(
+                ImmutablesLocation::Ipfs {
+                    uri: MultiFilesUri::Template(TemplateUri(
+                        "ipfs://directory-cid/{immutable_file_number}.tar.zst".to_string()
+                    )),
+                    compression_algorithm: None
+                },
+                location
+            );
+        }
     }
 
     mod immutable_file_number_extractor {

mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs

@@ -3,6 +3,7 @@ use semver::Version;
 use std::path::PathBuf;
 use std::sync::Arc;
 
+use mithril_common::StdResult;
 use mithril_common::crypto_helper::ManifestSigner;
 use mithril_file_archiver::FileArchiver;
 
@@ -78,16 +79,33 @@ impl DependenciesBuilder {
             logger,
         ));
 
-        // Compute the cache pool for prover service
+        // Compute the cache pool for both new and legacy prover services
         // This is done here to avoid circular dependencies between the prover service and the signed entity service
         // TODO: Make this part of a warmup phase of the aggregator?
-        if let Some(signed_entity) =
-            signed_entity_service.get_last_cardano_transaction_snapshot().await?
-        {
-            legacy_prover_service
-                .compute_cache(signed_entity.artifact.block_number)
-                .await?;
-        }
+        let warm_up_legacy_cache = async {
+            if let Some(signed_entity) =
+                signed_entity_service.get_last_cardano_transaction_snapshot().await?
+            {
+                legacy_prover_service
+                    .compute_cache(signed_entity.artifact.block_number)
+                    .await?;
+            }
+            StdResult::Ok(())
+        };
+
+        let warm_up_cache = async {
+            if let Some(signed_entity) = signed_entity_service
+                .get_last_cardano_blocks_transactions_snapshot()
+                .await?
+            {
+                prover_service
+                    .compute_cache(signed_entity.artifact.block_number_signed)
+                    .await?;
+            }
+            StdResult::Ok(())
+        };
+
+        tokio::try_join!(warm_up_legacy_cache, warm_up_cache)?;
 
         Ok(signed_entity_service)
     }

mithril-aggregator/src/file_uploaders/ipfs_uploader.rs

@@ -235,7 +235,7 @@ mod tests {
     use super::*;
 
     impl IpfsUploader {
-        fn new_for_test<P: Into<IpfsMfsDirPath>>(
+        pub(crate) fn new_for_test<P: Into<IpfsMfsDirPath>>(
             mfs_dir: P,
             mock_config: impl FnOnce(&mut MockIpfsBackendUploader),
         ) -> Self {

mithril-aggregator/src/multi_signer.rs

@@ -78,7 +78,7 @@ impl MultiSignerImpl {
         protocol_multi_signer
             .verify_single_signature(&message, single_signature)
             .with_context(|| {
-                format!("Multi Signer can not verify single signature for message '{message:?}' and single signature {single_signature:#?}")
+                format!("Multi Signer can not verify single signature for message '{message:?}' from party_id '{}'", single_signature.party_id)
             })
     }
 }

mithril-client/src/file_downloader/ipfs.rs

@@ -76,8 +76,8 @@ impl IpfsFileDownloader {
             .with_context(|| format!("Could not build Kubo RPC endpoint for route '{route}'"))
     }
 
-    /// POST to a Kubo RPC `route` with the given IPFS path (`<directory-CID>/<filename>`) as its
-    /// `arg` query parameter.
+    /// POST to a Kubo RPC `route` with the given IPFS path (`ipfs://<directory-CID>/<filename>`) as
+    /// its `arg` query parameter.
     async fn post(
         &self,
         route: &str,
@@ -135,7 +135,7 @@ impl IpfsFileDownloader {
         let response = self
             .post(
                 "api/v0/files/stat",
-                &with_ipfs_namespace_prefix(ipfs_path),
+                ipfs_path,
                 Some(self.existence_check_timeout),
             )
             .await?;
@@ -163,11 +163,6 @@ impl IpfsFileDownloader {
     }
 }
 
-// files/stat does not work against directory CID without the `/ipfs/` prefix
-fn with_ipfs_namespace_prefix(cid: &str) -> String {
-    format!("/ipfs/{cid}")
-}
-
 #[async_trait]
 impl FileDownloader for IpfsFileDownloader {
     async fn download_unpack(
@@ -178,9 +173,7 @@ impl FileDownloader for IpfsFileDownloader {
         compression_algorithm: Option<CompressionAlgorithm>,
         download_event_type: DownloadEvent,
     ) -> StdResult<()> {
-        // `location` is a path in the form `<directory-CID>/<filename>` (confirmed against a live
-        // Kubo node), which is exactly the `arg` shape `cat` expect and only need to be prefixed
-        // with `/ipfs/` for `files/stat`.
+        // `location` is expected a path in the form `ipfs://<directory-CID>/<filename>`.
         let ipfs_path = location.as_str();
         let downloaded = self.open_stream(ipfs_path).await?;
 
@@ -225,12 +218,12 @@ mod tests {
         let target_dir = temp_dir_create!();
         let content = "Hello, world!";
         let size = content.len() as u64;
-        let ipfs_path = "QmDummyDirCid/00006.tar.zst";
+        let ipfs_path = "ipfs://QmDummyDirCid/00006.tar.zst";
         let server = MockServer::start();
         server.mock(|when, then| {
             when.method(POST)
                 .path("/api/v0/files/stat")
-                .query_param("arg", with_ipfs_namespace_prefix(ipfs_path));
+                .query_param("arg", ipfs_path);
             then.status(200)
                 .json_body(serde_json::json!({"Key": "bafkreidummy", "Size": size}));
         });
@@ -277,10 +270,10 @@ mod tests {
     #[tokio::test]
     async fn missing_block_reported_through_500_body_is_turned_into_a_not_found_error() {
         let target_dir = temp_dir_create!();
-        let ipfs_path = "QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/00007.tar.zst";
+        let ipfs_path = "ipfs://QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/00007.tar.zst";
         let server = MockServer::start();
         server.mock(|when, then| {
-            when.method(POST).path("/api/v0/files/stat").query_param("arg", with_ipfs_namespace_prefix(ipfs_path));
+            when.method(POST).path("/api/v0/files/stat").query_param("arg", ipfs_path);
             then.status(500).json_body(serde_json::json!({
                 "Message": "no link named \"00007.tar.zst\" under QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2",
                 "Code": 0,
@@ -303,23 +296,22 @@ mod tests {
             .unwrap_err();
 
         assert!(
-            error.to_string().contains(&format!(
-                "Location='{}' not found",
-                with_ipfs_namespace_prefix(ipfs_path)
-            )),
+            error
+                .to_string()
+                .contains(&format!("Location='{ipfs_path}' not found")),
             "unexpected error: {error:?}"
         );
     }
 
     #[tokio::test]
     async fn files_stat_timeout_raise_unreachable_file_downloader_error() {
         let target_dir = temp_dir_create!();
-        let ipfs_path = "QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/00007.tar.zst";
+        let ipfs_path = "ipfs://QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/00007.tar.zst";
         let server = MockServer::start();
         server.mock(|when, then| {
             when.method(POST)
                 .path("/api/v0/files/stat")
-                .query_param("arg", with_ipfs_namespace_prefix(ipfs_path));
+                .query_param("arg", ipfs_path);
             then.delay(Duration::from_millis(100));
         });
         let ipfs_file_downloader = downloader(&server, FeedbackSender::new(&[]))
@@ -341,7 +333,7 @@ mod tests {
         assert_eq!(
             Some(&FileDownloaderUnreachable {
                 source: "IPFS",
-                uri: with_ipfs_namespace_prefix(ipfs_path)
+                uri: ipfs_path.to_string(),
             }),
             error.downcast_ref::<FileDownloaderUnreachable>()
         );
@@ -350,13 +342,13 @@ mod tests {
     #[tokio::test]
     async fn cat_does_not_apply_timeout() {
         let target_dir = temp_dir_create!();
-        let ipfs_path = "QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/00007.tar.zst";
+        let ipfs_path = "ipfs://QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/00007.tar.zst";
 
         let server = MockServer::start();
         server.mock(|when, then| {
             when.method(POST)
                 .path("/api/v0/files/stat")
-                .query_param("arg", with_ipfs_namespace_prefix(ipfs_path));
+                .query_param("arg", ipfs_path);
             then.status(200)
                 .json_body(serde_json::json!({"Key": "bafkreidummy", "Size": 1}));
         });
@@ -384,12 +376,12 @@ mod tests {
     #[tokio::test]
     async fn context_deadline_exceeded_is_not_mistaken_for_a_missing_file() {
         let target_dir = temp_dir_create!();
-        let ipfs_path = "QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/fake-cardano-cli.sh";
+        let ipfs_path = "ipfs://QmRoiDvkuGRg4tjWabNp4Y5jxbUS8FFNFn9pDopqfbtfW2/fake-cardano-cli.sh";
         let server = MockServer::start();
         server.mock(|when, then| {
             when.method(POST)
                 .path("/api/v0/files/stat")
-                .query_param("arg", with_ipfs_namespace_prefix(ipfs_path));
+                .query_param("arg", ipfs_path);
             then.status(500).json_body(serde_json::json!({
                 "Message": "context deadline exceeded",
                 "Code": 0,

mithril-test-lab/ipfs-devnet/commands/mkfiles/kubo-configure-swarm.sh

@@ -117,7 +117,7 @@ configure_node_peers() {
     fi
 
     peer_id="${peer_ids[$((peer_node_id - 1))]}"
-    swarm_port=$((4000 + peer_node_id))
+    swarm_port=$((5200 + peer_node_id))
 
     peers_json="${peers_json}${separator}{\"ID\":\"${peer_id}\",\"Addrs\":[\"/ip4/127.0.0.1/tcp/${swarm_port}\"]}"
     separator=","