Bundler output

.github/workflows/test.yml

@@ -74,3 +74,43 @@ jobs:
 
       - name: Run integration tests
         run: xvfb-run just test-integrations
+
+  cli-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly
+          targets: wasm32-unknown-unknown
+      - uses: cargo-bins/cargo-binstall@v1.15.9
+      - run: cargo binstall just
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            .
+            integration
+            test-runner
+
+      - name: Run cli tests
+        run: just test-cli
+
+  webpack-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly
+          targets: wasm32-unknown-unknown
+      - uses: cargo-bins/cargo-binstall@v1.15.9
+      - run: cargo binstall just
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            .
+            integration
+            test-runner
+
+      - name: Run webpack tests
+        run: just webpack::all-tests

CHANGELOG.md

@@ -3,6 +3,9 @@
 - Experimental support to emit debug sections. By default enabled via environment variable
   `WASM_SPLIT_CLI_ENABLE_DWARF`. The current setup duplicates the DWARF information into
   all output modules, which can lead to large split files.
+- Support for webpack bundled output with target `bundler`. You must enable the `sourceImport`
+  experiment in webpack to consume the output, see also source-phase-imports which is a
+  - currently stage 3 - TC39 proposal.
 
 ## wasm_split_helpers v0.2.3
 

crates/wasm_split_cli/src/bin/wasm_split_cli.rs

@@ -1,9 +1,16 @@
-use clap::Parser;
+use clap::{Parser, ValueEnum};
 use eyre::Result;
 use std::path::Path;
 
 use wasm_split_cli_support as this;
 
+#[derive(Debug, ValueEnum, Clone, Copy, PartialEq, Eq, Default)]
+enum Target {
+    #[default]
+    Web,
+    Bundler,
+}
+
 #[derive(Debug, Parser)]
 #[command(name = "wasm-split")]
 struct Cli {
@@ -13,6 +20,14 @@ struct Cli {
     /// Output directory.
     output: Box<Path>,
 
+    /// Output target
+    #[arg(long)]
+    target: Option<Target>,
+
+    /// Output module name
+    #[arg(long)]
+    out_name: Option<String>,
+
     /// Print verbose split information.
     #[arg(short, long)]
     verbose: bool,
@@ -23,12 +38,33 @@ fn main() -> Result<()> {
 
     let args = Cli::parse();
     let input_wasm = std::fs::read(args.input)?;
-    let main_out_path = args.output.join("main.wasm");
+    let stem = match args.out_name.as_ref() {
+        Some(name) => name,
+        None => "main",
+    };
+    let main_module;
+    let main_out_path = args.output.join(&format!("{stem}.wasm"));
     let _ = this::transform({
         let mut opts = this::Options::new(&input_wasm);
         opts.verbose = args.verbose;
         opts.output_dir = &args.output;
         opts.main_out_path = &main_out_path;
+        if let Some(name) = args.out_name.as_ref() {
+            main_module = match args.target {
+                None | Some(Target::Web) => format!("./{name}.js"),
+                Some(Target::Bundler) => format!("./{name}_bg.wasm"),
+            };
+            opts.main_module = &main_module;
+        }
+        match args.target {
+            Some(Target::Web) => {
+                opts.target.web();
+            }
+            Some(Target::Bundler) => {
+                opts.target.bundler();
+            }
+            None => {}
+        }
         opts
     })?;
     Ok(())

crates/wasm_split_cli/src/dep_graph.rs

@@ -161,7 +161,7 @@ fn iter_functions_with_relocs<'m>(
         );
         function_index += found_index;
         if function_index >= module.defined_funcs.len() {
-            bail!("Invalid relocation entry {entry:?}, no function contains its relocation range")
+            bail!("Invalid relocation entry {entry:?}, no function contains its relocation range");
         }
         let func_index = module.imported_funcs.len() + function_index;
         Ok((func_index, entry))

crates/wasm_split_cli/src/emit.rs

@@ -78,7 +78,10 @@ impl<'a> EmitState<'a> {
             .map(|name| name.as_ref())
             .collect::<HashSet<_>>();
         if unique_names.len() != shared_names.len() {
-            bail!("Failed to generate unique names for some exports. This is a bug in wasm-split, please report this as an issue.")
+            bail!(
+                "Failed to generate unique names for some exports. \
+                This is a bug in wasm-split, please report this as an issue."
+            );
         }
 
         Ok(EmitState {
@@ -93,9 +96,12 @@ impl<'a> EmitState<'a> {
         })
     }
 
-    pub(crate) fn input(&self) -> &InputModule<'_> {
+    pub(crate) fn input(&self) -> &'a InputModule<'a> {
         self.input_module
     }
+    pub(crate) fn input_options(&self) -> &'a crate::Options<'a> {
+        self.input_options
+    }
 
     fn get_indirect_function_table_type(&self) -> wasmparser::TableType {
         // + 1 due to empty entry at index 0
@@ -659,7 +665,7 @@ impl RelocTarget for ModuleEmitState<'_> {
                 Ok(Some(index))
             }
             RelocDetails::RelTableIndex(_details) => {
-                bail!("Unsupported relocation type: relative table index")
+                bail!("Unsupported relocation type: relative table index");
             }
             RelocDetails::FunctionIndex(details) => {
                 let input_func_id = details.index;
@@ -677,25 +683,27 @@ impl RelocTarget for ModuleEmitState<'_> {
             }
             RelocDetails::TableNumber(details) => {
                 if !self.is_main() && details.index != self.input_module.reloc_info.indirect_table {
-                    bail!("Relocation of tables not supported in split modules.")
+                    bail!("Relocation of tables not supported in split modules.");
                 }
                 // TODO: check that table indices do not get confused by the generate logic below
                 Ok(Some(0))
             }
             RelocDetails::GlobalIndex(details) => {
                 if !self.is_main() && details.index != self.input_module.reloc_info.stack_pointer {
-                    bail!("Relocation of globals not supported in split modules.")
+                    bail!("Relocation of globals not supported in split modules.");
                 }
                 // TODO: check that global indices do not get confused by the generate logic below
                 Ok(Some(0))
             }
             RelocDetails::TagIndex(_details) => {
                 if !self.is_main() {
-                    bail!("Exception handling in split modules not supported yet")
+                    bail!("Exception handling in split modules not supported yet");
                 }
                 Ok(None)
             }
-            _ => bail!("unexpected relocation {reloc:?} in code/data section"),
+            _ => {
+                bail!("unexpected relocation {reloc:?} in code/data section");
+            }
         }
     }
 }
@@ -1043,7 +1051,9 @@ impl<'a> ModuleEmitState<'a> {
                             })
                         }
                         _ => {
-                            bail!("Expected a i32/i64.const expression for an exported data symbol")
+                            bail!(
+                                "Expected a i32/i64.const expression for an exported data symbol"
+                            );
                         }
                     };
                 let fake_reloc = wasmparser::RelocationEntry {
@@ -1123,7 +1133,7 @@ impl<'a> ModuleEmitState<'a> {
                         .get(&DepNode::Function(input_func_id))
                 });
                 let Some(&output_func_id) = output_func_id else {
-                    bail!("No output function corresponding to input function {input_func_id:?}")
+                    bail!("No output function corresponding to input function {input_func_id:?}");
                 };
                 Ok(output_func_id as u32)
             })

crates/wasm_split_cli/src/emit/dwarf.rs

@@ -70,7 +70,9 @@ impl RelocTarget for DwarfRelocTarget<'_, '_> {
                 let _ = details;
                 None
             }
-            _ => bail!("unexpected reloc in debug section: {:?}", reloc),
+            _ => {
+                bail!("unexpected reloc in debug section: {:?}", reloc);
+            }
         };
         Ok(reloc)
     }

crates/wasm_split_cli/src/js.rs

@@ -11,6 +11,7 @@ use crate::{
 type PrefetchMap = HashMap<String, Vec<String>>;
 pub struct LinkModuleWriter<'p> {
     input_module: &'p InputModule<'p>,
+    input_options: &'p crate::Options<'p>,
     program_info: &'p SplitProgramInfo,
     javascript: String,
     prefetch_map: PrefetchMap,
@@ -21,6 +22,7 @@ impl<'p> LinkModuleWriter<'p> {
         Self {
             program_info,
             input_module: emit_state.input(),
+            input_options: emit_state.input_options(),
             javascript: String::new(),
             prefetch_map: HashMap::new(),
         }
@@ -29,26 +31,39 @@ impl<'p> LinkModuleWriter<'p> {
         self.program_info.canary_export_name()
     }
     fn write_main_import(&mut self, mod_path: &str) -> Result<()> {
-        Ok(writeln!(
-            &mut self.javascript,
-            r#"import {{ initSync }} from "{}";"#,
-            mod_path
-        )?)
+        match self.input_options.target {
+            crate::OutputTarget::Web(_) => writeln!(
+                &mut self.javascript,
+                r#"import {{ initSync }} from "{}";
+"#,
+                mod_path
+            )?,
+            crate::OutputTarget::Bundler(_) => writeln!(
+                &mut self.javascript,
+                r#"import * as __wasm from "{}";
+"#,
+                mod_path
+            )?,
+        }
+        Ok(())
     }
     fn write_get_shared_imports(&mut self, main_shares: &str) -> Result<()> {
         let canary_props = if self.input_module.options.debug_assertions {
             format!("{}: ~0xdead,", self.canary_name())
         } else {
             String::new()
         };
+        let main_exports = match self.input_options.target {
+            crate::OutputTarget::Web(_) => "initSync(undefined, undefined)",
+            crate::OutputTarget::Bundler(_) => "__wasm",
+        };
         Ok(write!(
             &mut self.javascript,
             r#"let sharedImports = undefined;
 function getSharedImports() {{
     if (sharedImports === undefined) {{
         sharedImports = {{ __wasm_split: {{ {canary_props} }} }};
-        const mainExports = initSync(undefined, undefined);
-        const {{ {main_shares} }} = mainExports;
+        const {{ {main_shares} }} = {main_exports};
         Object.assign(sharedImports.__wasm_split, {{ {main_shares} }});
     }}
     return sharedImports;
@@ -59,12 +74,10 @@ function getSharedImports() {{
     fn write_runtime(&mut self) -> Result<()> {
         self.javascript
             .push_str(include_str!("./snippets/split_wasm.js"));
-        self.javascript
-            .push_str(if self.input_module.options.debug_assertions {
-                include_str!("./snippets/makeFetch.web.debug.js")
-            } else {
-                include_str!("./snippets/makeFetch.web.js")
-            });
+        if self.input_module.options.debug_assertions {
+            self.javascript
+                .push_str(include_str!("./snippets/instantiate.debug.js"));
+        }
         Ok(())
     }
     fn write_export_const(&mut self, name: &str, def: &impl std::fmt::Display) -> Result<()> {
@@ -73,20 +86,36 @@ function getSharedImports() {{
             "export const {name} = {def};"
         )?)
     }
-    fn fetch_opts<'pth>(&self, empty: bool, file_path: impl 'pth + std::fmt::Display) -> String {
+    fn fetcher<'pth>(&self, empty: bool, file_path: impl 'pth + std::fmt::Display) -> String {
         if empty {
-            "undefined".to_string()
+            return "() => async (_imp) => ({})".to_string();
+        }
+        let wrap = if self.input_module.options.debug_assertions {
+            "debugWrap"
         } else {
-            // Note: the expression returned from here should:
-            // - allow lazily fetching the wasm module (no top-level import)
-            // - allow bundlers and downstream code to recognize it as an expression to a path ("relocate" the import)
-            // TODO: try other syntax for different targets:
-            // - `import.source(<file_path>)`
-            // - `URL.resolve(<file_path)` (support is not as good as for new URL)
+            ""
+        };
+        // Note: the expression returned from here should:
+        // - allow lazily fetching the wasm module (no top-level import)
+        // - allow bundlers and downstream code to recognize it as an expression to a path ("relocate" the import)
+        match self.input_options.target {
             // Note: we use the form `new URL(<string literal>, import.meta.url)` which is understood by some
             // bundlers as syntax that can get rewritten if the path from where the file gets fetched is changed
             // (for example due to attaching a has of its contents).
-            format!("new URL({}, import.meta.url)", file_path)
+            crate::OutputTarget::Web(_) => format!(
+                r#"() => {{
+    const src = fetch(new URL({file_path}, import.meta.url));
+    return async (imports) => {wrap}(WebAssembly.instantiateStreaming(src, imports));
+}}
+"#
+            ),
+            crate::OutputTarget::Bundler(_) => format!(
+                r#"() => {{
+    const module = import.source({file_path});
+    return async (imports) => {wrap}(new WebAssembly.Instance(await module, imports));
+}}
+"#
+            ),
         }
     }
     fn write_loaders(&mut self, program: &SplitProgramInfo) -> Result<()> {
@@ -100,10 +129,10 @@ function getSharedImports() {{
             let var_name = format!("__chunk_{module_index}");
             let splits_dbg = splits.iter().cloned().collect::<Vec<_>>().join(", ");
             writeln!(&mut self.javascript, "/* {splits_dbg} */")?;
-            let fetch_opts = self.fetch_opts(is_empty, format_args!("\"./{file_name}.wasm\""));
+            let fetcher = self.fetcher(is_empty, format_args!("\"./{file_name}.wasm\""));
             writeln!(
                 &mut self.javascript,
-                "const {var_name} = makeLoad({fetch_opts}, []);"
+                "const {var_name} = makeLoad({fetcher}, []);"
             )?;
             for split in splits {
                 split_deps
@@ -130,7 +159,7 @@ function getSharedImports() {{
             let loader_name = identifier.loader_name();
             let deps = split_deps.remove(split).unwrap_or_default();
             let deps = deps.join(", ");
-            let fetch_opts = self.fetch_opts(is_empty, format_args!("\"./{file_name}.wasm\""));
+            let fetch_opts = self.fetcher(is_empty, format_args!("\"./{file_name}.wasm\""));
             self.write_export_const(
                 &loader_name,
                 &format_args!("wrapAsyncCb(makeLoad({fetch_opts}, [{deps}]))"),