fix: memory issues when importing giant mrpack files

Cargo.toml

@@ -85,6 +85,7 @@ eyre = "0.6.12"
 flate2 = "1.1.4"
 fs4 = { version = "0.13.1", default-features = false }
 futures = "0.3.31"
+futures-lite = "2.6.1"
 futures-util = "0.3.31"
 heck = "0.5.0"
 hex = "0.4.3"

apps/app-frontend/src/providers/setup/file-picker.ts

@@ -26,20 +26,27 @@ export function setupFilePickerProvider() {
 			const file = await createFileFromPath(path, 'icon')
 			return { file, path, previewUrl: convertFileSrc(path) }
 		},
-		async pickModpackFile() {
+		async pickModpackFile(options) {
 			const result = await open({
 				multiple: false,
 				filters: [{ name: 'Modpack', extensions: ['mrpack'] }],
 			})
 			if (!result) return null
 			const path = result.path ?? result
 			if (!path) return null
-			const file = await createFileFromPath(
+			if (options?.readFile === false) {
+				// Instance imports stream from the native path, keeping large packs out of JS memory.
+				return { path, previewUrl: '' }
+			}
+			return {
+				file: await createFileFromPath(
+					path,
+					'modpack.mrpack',
+					'application/x-modrinth-modpack+zip',
+				),
 				path,
-				'modpack.mrpack',
-				'application/x-modrinth-modpack+zip',
-			)
-			return { file, path, previewUrl: '' }
+				previewUrl: '',
+			}
 		},
 	})
 }

packages/app-lib/Cargo.toml

@@ -41,6 +41,7 @@ eyre = { workspace = true }
 flate2 = { workspace = true }
 fs4 = { workspace = true, features = ["tokio"] }
 futures = { workspace = true, features = ["alloc", "async-await"] }
+futures-lite = { workspace = true }
 heck = { workspace = true }
 hickory-resolver = { workspace = true }
 indicatif = { workspace = true, optional = true }

packages/app-lib/src/api/pack/install_from.rs

@@ -8,10 +8,9 @@ use crate::state::{
     SideType,
 };
 use crate::util::fetch::{
-    DownloadMeta, DownloadReason, fetch, fetch_advanced, write_cached_icon,
+    DownloadMeta, DownloadReason, fetch, fetch_advanced, sha1_file_async,
+    write_cached_icon,
 };
-use crate::util::io;
-
 use path_util::SafeRelativeUtf8UnixPathBuf;
 use reqwest::Method;
 use serde::{Deserialize, Serialize};
@@ -134,12 +133,22 @@ impl Default for CreatePackProfile {
     }
 }
 
+#[derive(Clone)]
+pub enum CreatePackFile {
+    Bytes(bytes::Bytes),
+    // Local packs can be larger than available memory, so keep them file-backed.
+    Path(PathBuf),
+}
+
 #[derive(Clone)]
 pub struct CreatePack {
-    pub file: bytes::Bytes,
+    pub file: CreatePackFile,
     pub description: CreatePackDescription,
 }
 
+// The hash lookup only gates the unknown-pack warning, so avoid a long blocking scan for huge local packs.
+const MAX_LOCAL_FILE_HASH_LOOKUP_SIZE: u64 = 1024 * 1024 * 1024;
+
 #[derive(Clone, Debug)]
 pub struct CreatePackDescription {
     pub icon: Option<PathBuf>,
@@ -176,28 +185,31 @@ pub async fn get_profile_from_pack(
                 .to_string_lossy()
                 .to_string();
 
-            let state = State::get().await?;
-            let file_bytes = io::read(&path).await?;
-            let hash =
-                crate::util::fetch::sha1_async(bytes::Bytes::from(file_bytes))
-                    .await?;
-            let is_known_file = match CachedEntry::get_file_many(
-                &[&hash],
-                Some(CacheBehaviour::StaleWhileRevalidateSkipOffline),
-                &state.pool,
-                &state.api_semaphore,
-            )
-            .await
+            let is_known_file = if tokio::fs::metadata(&path).await?.len()
+                <= MAX_LOCAL_FILE_HASH_LOOKUP_SIZE
             {
-                Ok(files) => !files.is_empty(),
-                Err(err) => {
-                    tracing::warn!(
-                        "Failed to check Modrinth file hash for {}: {}",
-                        path.display(),
-                        err
-                    );
-                    false
+                let state = State::get().await?;
+                let (_, hash) = sha1_file_async(&path).await?;
+                match CachedEntry::get_file_many(
+                    &[&hash],
+                    Some(CacheBehaviour::StaleWhileRevalidateSkipOffline),
+                    &state.pool,
+                    &state.api_semaphore,
+                )
+                .await
+                {
+                    Ok(files) => !files.is_empty(),
+                    Err(err) => {
+                        tracing::warn!(
+                            "Failed to check Modrinth file hash for {}: {}",
+                            path.display(),
+                            err
+                        );
+                        false
+                    }
                 }
+            } else {
+                false
             };
 
             Ok(CreatePackProfile {
@@ -380,7 +392,7 @@ pub async fn generate_pack_from_version_id(
     }
 
     Ok(CreatePack {
-        file,
+        file: CreatePackFile::Bytes(file),
         description: CreatePackDescription {
             icon,
             override_title: Some(title),
@@ -398,9 +410,8 @@ pub async fn generate_pack_from_file(
     path: PathBuf,
     profile_path: String,
 ) -> crate::Result<CreatePack> {
-    let file = io::read(&path).await?;
     Ok(CreatePack {
-        file: bytes::Bytes::from(file),
+        file: CreatePackFile::Path(path),
         description: CreatePackDescription {
             icon: None,
             override_title: None,