diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cb8443b..9e5fbbd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,7 @@ jobs: test: runs-on: ubuntu-latest env: - RUST_BACKTRACE: "1" + RUST_BACKTRACE: full strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1c7334..50c29da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: test: runs-on: ubuntu-latest env: - RUST_BACKTRACE: "1" + RUST_BACKTRACE: full strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] diff --git a/README.md b/README.md index 59a2744..dee672c 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,9 @@ from micropython_wasm import run result = run( "print(sum(range(10)))", memory_bytes=16 * 1024 * 1024, - fuel=5_000_000, + fuel=20_000_000, wall_timeout_seconds=1.0, + host_result_bytes=256 * 1024, ) ``` @@ -130,6 +131,8 @@ Arguments: `/input`, with read-only WASI directory and file permissions. - `host_functions`: optional mapping of host function names to Python callables. This enables the low-level `host.call(name, payload_json)` bridge. +- `host_result_bytes`: maximum serialized host callback response size. The + default is `256 * 1024`. ### `run_micropython_wasi(code, wasm_path, ...)` @@ -176,9 +179,10 @@ function arguments as `run()`: ```python session = MicroPythonSession( memory_bytes=16 * 1024 * 1024, - fuel=5_000_000, + fuel=20_000_000, readonly_dir="fixtures", host_functions={"add": lambda a, b: a + b}, + host_result_bytes=256 * 1024, ) ``` @@ -234,9 +238,10 @@ session.close() ```python session = MicroPythonReplaySession( memory_bytes=16 * 1024 * 1024, - fuel=5_000_000, + fuel=20_000_000, wall_timeout_seconds=1.0, readonly_dir="fixtures", + host_result_bytes=256 * 1024, ) ``` diff --git a/micropython_wasm/__init__.py b/micropython_wasm/__init__.py index c45e0b4..6cf3638 100644 --- a/micropython_wasm/__init__.py +++ b/micropython_wasm/__init__.py @@ -8,6 +8,9 @@ from pathlib import Path from typing import Callable, Mapping, Optional +DEFAULT_HOST_RESULT_BYTES = 256 * 1024 +DEFAULT_FUEL = 20_000_000 + __all__ = [ "MicroPythonWasmError", "MicroPythonWasmArtifactNotFound", @@ -97,10 +100,11 @@ def __init__( wasm_path: str | Path | None = None, *, memory_bytes: int = 16 * 1024 * 1024, - fuel: int = 5_000_000, + fuel: int = DEFAULT_FUEL, wall_timeout_seconds: Optional[float] = 1.0, readonly_dir: str | Path | None = None, host_functions: Mapping[str, Callable[..., object]] | None = None, + host_result_bytes: int = DEFAULT_HOST_RESULT_BYTES, ) -> None: self.wasm_path = ( Path(wasm_path) if wasm_path is not None else default_wasm_path() @@ -109,6 +113,7 @@ def __init__( self.fuel = fuel self.wall_timeout_seconds = wall_timeout_seconds self.readonly_dir = readonly_dir + self.host_result_bytes = host_result_bytes self._snippets: list[str] = [] self._preamble: list[str] = [] self._host_functions: dict[str, Callable[..., object]] = {} @@ -170,6 +175,7 @@ def run(self, code: str) -> RunResult: wall_timeout_seconds=self.wall_timeout_seconds, readonly_dir=self.readonly_dir, host_functions=self._host_functions, + host_result_bytes=self.host_result_bytes, ) marker_with_newline = marker + "\n" if marker_with_newline not in result.stdout: @@ -208,10 +214,11 @@ def __init__( wasm_path: str | Path | None = None, *, memory_bytes: int = 16 * 1024 * 1024, - fuel: int = 5_000_000, + fuel: int = DEFAULT_FUEL, wall_timeout_seconds: Optional[float] = None, readonly_dir: str | Path | None = None, host_functions: Mapping[str, Callable[..., object]] | None = None, + host_result_bytes: int = DEFAULT_HOST_RESULT_BYTES, ) -> None: self.wasm_path = ( Path(wasm_path) if wasm_path is not None else default_wasm_path() @@ -220,6 +227,7 @@ def __init__( self.fuel = fuel self.wall_timeout_seconds = wall_timeout_seconds self.readonly_dir = readonly_dir + self.host_result_bytes = host_result_bytes self._host_functions: dict[str, Callable[..., object]] = {} self._pending_preamble: list[str] = [] self._closed = False @@ -365,6 +373,9 @@ def _thread_main(self) -> None: try: self._run_bootstrap() except BaseException as exc: + exc.__traceback__ = None + exc.__cause__ = None + exc.__context__ = None self._thread_error = exc def _run_bootstrap(self) -> None: @@ -392,6 +403,7 @@ def _run_bootstrap(self) -> None: self.memory_bytes, self.fuel, self.wall_timeout_seconds, + self.host_result_bytes, ) cfg = Config() @@ -451,7 +463,15 @@ def capture_stderr(data: bytes) -> None: with self._callback_lock: host_functions.update(self._host_functions) self._thread_host_functions = host_functions - _define_host_call(linker, store, host_functions, Func, FuncType, ValType) + _define_host_call( + linker, + store, + host_functions, + self.host_result_bytes, + Func, + FuncType, + ValType, + ) try: try: @@ -543,10 +563,11 @@ def run( wasm_path: str | Path | None = None, *, memory_bytes: int = 16 * 1024 * 1024, - fuel: int = 5_000_000, + fuel: int = DEFAULT_FUEL, wall_timeout_seconds: Optional[float] = 1.0, readonly_dir: str | Path | None = None, host_functions: Mapping[str, Callable[..., object]] | None = None, + host_result_bytes: int = DEFAULT_HOST_RESULT_BYTES, ) -> RunResult: """Run MicroPython code in a fresh WASI WebAssembly instance.""" @@ -558,6 +579,7 @@ def run( wall_timeout_seconds=wall_timeout_seconds, readonly_dir=readonly_dir, host_functions=host_functions, + host_result_bytes=host_result_bytes, ) @@ -566,10 +588,11 @@ def run_micropython_wasi( wasm_path: str | Path, *, memory_bytes: int = 16 * 1024 * 1024, - fuel: int = 5_000_000, + fuel: int = DEFAULT_FUEL, wall_timeout_seconds: Optional[float] = 1.0, readonly_dir: str | Path | None = None, host_functions: Mapping[str, Callable[..., object]] | None = None, + host_result_bytes: int = DEFAULT_HOST_RESULT_BYTES, ) -> RunResult: """ Run code through a WASI MicroPython command module. @@ -580,7 +603,9 @@ def run_micropython_wasi( """ wasm_path = Path(wasm_path) - _validate_execution_options(wasm_path, memory_bytes, fuel, wall_timeout_seconds) + _validate_execution_options( + wasm_path, memory_bytes, fuel, wall_timeout_seconds, host_result_bytes + ) try: from wasmtime import ( @@ -643,7 +668,13 @@ def capture_stderr(data: bytes) -> None: linker = Linker(engine) linker.define_wasi() _define_host_call( - linker, store, dict(host_functions or {}), Func, FuncType, ValType + linker, + store, + dict(host_functions or {}), + host_result_bytes, + Func, + FuncType, + ValType, ) timer: threading.Timer | None = None @@ -685,6 +716,7 @@ def _validate_execution_options( memory_bytes: int, fuel: int, wall_timeout_seconds: Optional[float], + host_result_bytes: int, ) -> None: if not wasm_path.exists(): raise MicroPythonWasmArtifactNotFound( @@ -701,6 +733,10 @@ def _validate_execution_options( raise ValueError("fuel must be greater than zero") if wall_timeout_seconds is not None and wall_timeout_seconds <= 0: raise ValueError("wall_timeout_seconds must be greater than zero or None") + if host_result_bytes <= 0: + raise ValueError("host_result_bytes must be greater than zero") + if host_result_bytes > 2_147_483_647: + raise ValueError("host_result_bytes must fit in a WebAssembly i32") def _configure_readonly_dir(wasi, readonly_dir: str | Path | None, error_cls) -> None: @@ -723,7 +759,13 @@ def _configure_readonly_dir(wasi, readonly_dir: str | Path | None, error_cls) -> def _define_host_call( - linker, store, host_functions, func_cls, func_type_cls, val_type_cls + linker, + store, + host_functions, + host_result_bytes, + func_cls, + func_type_cls, + val_type_cls, ) -> None: def host_call( caller, @@ -790,3 +832,10 @@ def host_call( "host_call", func_cls(store, ty, host_call, access_caller=True), ) + cap_ty = func_type_cls([], [val_type_cls.i32()]) + linker.define( + store, + "micropython_wasm", + "host_result_cap", + func_cls(store, cap_ty, lambda: host_result_bytes), + ) diff --git a/micropython_wasm/artifacts/micropython-wasi.wasm b/micropython_wasm/artifacts/micropython-wasi.wasm index b388d13..4144a54 100644 Binary files a/micropython_wasm/artifacts/micropython-wasi.wasm and b/micropython_wasm/artifacts/micropython-wasi.wasm differ diff --git a/micropython_wasm/usercmodule/host/hostmodule.c b/micropython_wasm/usercmodule/host/hostmodule.c index 4b63294..2657af0 100644 --- a/micropython_wasm/usercmodule/host/hostmodule.c +++ b/micropython_wasm/usercmodule/host/hostmodule.c @@ -1,10 +1,14 @@ #include #include +#include "py/mpstate.h" #include "py/objstr.h" #include "py/runtime.h" -#define HOST_RESULT_CAP (64 * 1024) +#define DEFAULT_HOST_RESULT_CAP (256 * 1024) + +__attribute__((import_module("micropython_wasm"), import_name("host_result_cap"))) +extern int32_t micropython_wasm_host_result_cap(void); __attribute__((import_module("micropython_wasm"), import_name("host_call"))) extern int32_t micropython_wasm_host_call( @@ -21,7 +25,21 @@ static mp_obj_t host_call(mp_obj_t name_obj, mp_obj_t payload_obj) { size_t payload_len; const char *name = mp_obj_str_get_data(name_obj, &name_len); const char *payload = mp_obj_str_get_data(payload_obj, &payload_len); - char *result = m_new(char, HOST_RESULT_CAP); + int32_t result_cap = micropython_wasm_host_result_cap(); + if (result_cap <= 0) { + result_cap = DEFAULT_HOST_RESULT_CAP; + } + char *result = MP_STATE_VM(host_result_buffer); + if ((size_t)result_cap > MP_STATE_VM(host_result_buffer_cap)) { + result = m_renew( + char, + result, + MP_STATE_VM(host_result_buffer_cap), + (size_t)result_cap + ); + MP_STATE_VM(host_result_buffer) = result; + MP_STATE_VM(host_result_buffer_cap) = (size_t)result_cap; + } int32_t result_len = micropython_wasm_host_call( name, @@ -29,20 +47,17 @@ static mp_obj_t host_call(mp_obj_t name_obj, mp_obj_t payload_obj) { payload, payload_len, result, - HOST_RESULT_CAP + (size_t)result_cap ); if (result_len < 0) { - m_del(char, result, HOST_RESULT_CAP); mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("host callback failed")); } - if ((size_t)result_len > HOST_RESULT_CAP) { - m_del(char, result, HOST_RESULT_CAP); + if ((size_t)result_len > (size_t)result_cap) { mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("host callback result too large")); } mp_obj_t out = mp_obj_new_str(result, (size_t)result_len); - m_del(char, result, HOST_RESULT_CAP); return out; } static MP_DEFINE_CONST_FUN_OBJ_2(host_call_obj, host_call); @@ -59,3 +74,5 @@ const mp_obj_module_t host_user_cmodule = { }; MP_REGISTER_MODULE(MP_QSTR_host, host_user_cmodule); +MP_REGISTER_ROOT_POINTER(char *host_result_buffer); +MP_REGISTER_ROOT_POINTER(size_t host_result_buffer_cap); diff --git a/tests/test_host_functions.py b/tests/test_host_functions.py index 08ebe43..8e94efc 100644 --- a/tests/test_host_functions.py +++ b/tests/test_host_functions.py @@ -4,7 +4,7 @@ import pytest -from micropython_wasm import MicroPythonSession, default_wasm_path, run +from micropython_wasm import MicroPythonSession, MicroPythonWasmError, default_wasm_path, run pytestmark = pytest.mark.skipif( not default_wasm_path().exists(), @@ -89,6 +89,51 @@ def describe(value): assert result.stdout == "4\n8\nhost,python\n" +def test_run_host_function_allows_256k_results_by_default(): + result = run( + """ +import host +print(len(host.call("large", '{"args": [], "kwargs": {}}'))) +""", + host_functions={"large": lambda: "x" * (128 * 1024)}, + wall_timeout_seconds=None, + ) + + assert result.stdout == "131094\n" + + +def test_run_host_result_bytes_can_lower_result_limit(): + result = run( + """ +import host +try: + host.call("large", '{"args": [], "kwargs": {}}') +except ValueError as ex: + print(str(ex)) +""", + host_functions={"large": lambda: "x" * 2048}, + host_result_bytes=1024, + wall_timeout_seconds=None, + ) + + assert result.stdout == "host callback result too large\n" + + +def test_session_host_result_bytes_can_lower_result_limit(): + session = MicroPythonSession( + host_functions={"large": lambda: "x" * 2048}, + host_result_bytes=1024, + wall_timeout_seconds=None, + ) + try: + with pytest.raises( + MicroPythonWasmError, match="ValueError: host callback result too large" + ): + session.run("large()") + finally: + session.close() + + def test_session_host_function_result_can_be_used_as_session_state(): session = MicroPythonSession( host_functions={"add": lambda a, b: a + b}, wall_timeout_seconds=None diff --git a/tests/test_micropython_wasm.py b/tests/test_micropython_wasm.py index 526b4a9..4505745 100644 --- a/tests/test_micropython_wasm.py +++ b/tests/test_micropython_wasm.py @@ -80,6 +80,9 @@ def test_run_micropython_wasi_validates_resource_limits_before_wasmtime(tmp_path with pytest.raises(ValueError, match="wall_timeout_seconds"): run_micropython_wasi("print(1)", wasm_path, wall_timeout_seconds=0) + with pytest.raises(ValueError, match="host_result_bytes"): + run_micropython_wasi("print(1)", wasm_path, host_result_bytes=0) + def test_build_script_finds_and_copies_unix_wasi_artifact(tmp_path): build_script = load_build_script() diff --git a/tests/test_persistent_session.py b/tests/test_persistent_session.py index 7d31d83..d879377 100644 --- a/tests/test_persistent_session.py +++ b/tests/test_persistent_session.py @@ -148,5 +148,9 @@ def test_persistent_session_wall_timeout_interrupts_infinite_loop(): try: with pytest.raises(MicroPythonWasmError, match="guest trapped"): session.run("while True:\n pass") + assert session._thread_error is not None + assert session._thread_error.__traceback__ is None + assert session._thread_error.__cause__ is None + assert session._thread_error.__context__ is None finally: session.close()