From a0eabb638ed9df93870f9577d7996c47207c41a0 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 18 Sep 2026 12:23:10 -0700 Subject: [PATCH 1/2] fix: Allow pywrangler sync to build path-based dependencies We added --allow-build but this is broken because if version 1.0 of pkg-a is available from pyodide-index but version 1.1 is on pypyi and 1.1 has no emscripten wheel, with --allow-build will try and fail to build 1.1 from pypi rather than installing 1.0 from the pyodide index. This just takes note of which packages are local paths and allows building exactly those. --- packages/cli/src/pywrangler/cli.py | 2 +- packages/cli/src/pywrangler/resolve.py | 25 +++++-- packages/cli/src/pywrangler/sync.py | 22 +++--- packages/cli/tests/test_cli.py | 90 +++++++++---------------- packages/cli/tests/test_version_sync.py | 80 ++++++++++++++++++++++ 5 files changed, 141 insertions(+), 78 deletions(-) diff --git a/packages/cli/src/pywrangler/cli.py b/packages/cli/src/pywrangler/cli.py index f196f03b..c05f6e63 100644 --- a/packages/cli/src/pywrangler/cli.py +++ b/packages/cli/src/pywrangler/cli.py @@ -136,7 +136,7 @@ def types_command(outdir: str | None, config: str | None) -> Never: "--allow-build/--no-allow-build", default=None, help=( - "Allow building source distributions and local directory sources. " + "Allow building any source distribution, not just local path sources. " "Defaults to the [tool.pywrangler] allow-build setting in pyproject.toml." ), ) diff --git a/packages/cli/src/pywrangler/resolve.py b/packages/cli/src/pywrangler/resolve.py index ae2aff7c..50b9105a 100644 --- a/packages/cli/src/pywrangler/resolve.py +++ b/packages/cli/src/pywrangler/resolve.py @@ -30,6 +30,9 @@ def __init__(self, lockfile: Path) -> None: # Names of packages sourced from a local path. They need refreshing when # rebuilt. self.local_packages: list[str] = [] + # Subset of `local_packages` that have no wheel and so must be built + # (directories, sdists). Local `.whl` files are excluded. + self.local_build_packages: list[str] = [] with open(lockfile, "rb") as f: data = tomllib.load(f) @@ -39,18 +42,26 @@ def __init__(self, lockfile: Path) -> None: if not name: logger.warning("Skipping malformed lockfile entry: %s", pkg) continue - if any(self._is_local_source(pkg, key) for key in self._LOCAL_SOURCE_KEYS): + local_paths = [ + path + for key in self._LOCAL_SOURCE_KEYS + if (path := self._local_source_path(pkg, key)) is not None + ] + if local_paths: self.local_packages.append(name) + if not all(path.endswith(".whl") for path in local_paths): + self.local_build_packages.append(name) self.requirements.append((name, pkg.get("version"))) @staticmethod - def _is_local_source(pkg: dict, key: str) -> bool: + def _local_source_path(pkg: dict, key: str) -> str | None: source = pkg.get(key) if not isinstance(source, dict): - return False + return None # A local reference has a `path` - return "path" in source + path = source.get("path") + return path if isinstance(path, str) else None def parse_requirements() -> list[str]: @@ -79,9 +90,9 @@ def _compile_lockfile( are preserved across re-runs (no silent upgrades). By default ``--no-build`` is passed so only prebuilt wheels are used. This - is because building a Pyodide platformed wheel will fail. Set *allow_build* - to permit building source distributions / local directory sources. This is - useful for testing against local checkouts of pure Python packages. + is because building a Pyodide platformed wheel will fail. Local directory + sources still resolve because uv reads their static metadata without + building. Set *allow_build* to permit building source distributions. """ project_root = get_project_root() with temp_requirements_file(supplemental_requirements) as req_in_path: diff --git a/packages/cli/src/pywrangler/sync.py b/packages/cli/src/pywrangler/sync.py index 97a87884..53636cf2 100644 --- a/packages/cli/src/pywrangler/sync.py +++ b/packages/cli/src/pywrangler/sync.py @@ -169,9 +169,10 @@ def _install_requirements_to_vendor( ) -> str | None: """Install packages to the Pyodide vendor directory from pylock.toml. - By default ``--no-build`` is passed so only prebuilt wheels install. When - *allow_build* is True, source distributions / local directory sources are - allowed to build. + By default only prebuilt wheels install, except for local path sources such + as directories and sdists, which have no wheel and are explicitly requested + by the project, so they are always built. When *allow_build* is True, every + source distribution is allowed to build. Returns: Error message string if installation failed, None if successful. @@ -206,13 +207,14 @@ def _install_requirements_to_vendor( install_cmd = ["uv", "pip", "install"] if not allow_build: - install_cmd.append("--no-build") - else: - # uv caches built wheels for local sources keyed on their path, so edits - # to local checkouts wouldn't be picked up. Refresh the build cache for - # those packages so `sync` always rebuilds them. - for name in plan.local_packages: - install_cmd += ["--refresh-package", name] + install_cmd += ["--only-binary", ":all:"] + for name in plan.local_build_packages: + install_cmd += ["--no-binary", name] + # uv caches built wheels for local sources keyed on their path, so edits to + # local checkouts wouldn't be picked up. Refresh the build cache for those + # packages so `sync` always rebuilds them. + for name in plan.local_packages: + install_cmd += ["--refresh-package", name] install_cmd += ["-r", str(plan.lockfile), "--preview-features", "pylock"] result = run_command( install_cmd, diff --git a/packages/cli/tests/test_cli.py b/packages/cli/tests/test_cli.py index 65c44513..71c578fb 100644 --- a/packages/cli/tests/test_cli.py +++ b/packages/cli/tests/test_cli.py @@ -392,9 +392,9 @@ def create_dummy_build_dep(parent_dir: Path, name: str = "dummy-build-dep") -> P """Create a tiny pure-Python dependency that must be built from source. The package only exists as a local directory (no prebuilt wheel on any - index), so installing it requires ``uv`` to run its build backend. This is - exactly what ``--allow-build`` gates: with ``--no-build`` (the default) the - resolver refuses to build it, and with ``--allow-build`` it succeeds. + index), so installing it requires ``uv`` to run its build backend. Local + path sources are the one kind of source ``sync`` builds even without + ``--allow-build``. Returns the path to the created dependency directory. """ @@ -428,8 +428,8 @@ def create_worker_pyproject_with_local_dep( The dependency is expressed as a PEP 508 direct reference to the local directory (``name @ file://...``) so the resolver treats it as a - ``directory`` source that must be built from source. ``allow_build_config`` - toggles the ``[tool.pywrangler] allow-build`` key. + ``directory`` source that must be built. ``allow_build_config`` toggles the + ``[tool.pywrangler] allow-build`` key. """ pywrangler_table = ( "[tool.pywrangler]\nallow-build = true\n" if allow_build_config else "" @@ -455,13 +455,17 @@ def create_worker_pyproject_with_local_dep( sys.platform == "win32", reason="FIXME Pyodide WASM interpreter cannot run setuptools build backends on Windows", ) -def test_sync_allow_build_local_dependency(test_dir): - """End-to-end test for --allow-build with a local source dependency. +@pytest.mark.parametrize( + "extra_args", [[], ["--no-allow-build"]], ids=["default", "explicit"] +) +def test_sync_builds_local_directory_dependency(test_dir, extra_args): + """End-to-end test: local directory sources are built without --allow-build. A tiny dummy package that only exists as a local directory (and therefore - must be built from source) is added to the worker's pyproject.toml. Syncing - without --allow-build must fail (default is --no-build), while syncing with - --allow-build must succeed and vendor the built package. + must be built from source) is added to the worker's pyproject.toml. Unlike + source distributions from an index, which stay blocked so uv can't pick a + newer sdist over an older Pyodide wheel, a local path source is something + the project explicitly asked for, so `sync` builds it. """ dep_name = "dummy-build-dep" dep_dir = create_dummy_build_dep(test_dir, dep_name) @@ -469,51 +473,21 @@ def test_sync_allow_build_local_dependency(test_dir): create_test_wrangler_jsonc(test_dir, "src/worker.py") vendor_path = test_dir / "python_modules" - sync_cmd = ["uv", "run", "pywrangler", "sync"] - - # Without --allow-build: the default --no-build rejects the local source. - result = subprocess.run( - [*sync_cmd, "--no-allow-build"], - capture_output=True, - text=True, - cwd=test_dir, - check=False, - ) - assert result.returncode != 0, ( - "sync should fail without --allow-build because the local dependency " - "must be built from source" - ) - assert not is_package_installed(vendor_path, dep_name), ( - "dummy build dep should not be vendored when the build was rejected" - ) - - # With --allow-build: uv is allowed to build the local source. result = subprocess.run( - [*sync_cmd, "--force", "--allow-build"], + ["uv", "run", "pywrangler", "sync", *extra_args], capture_output=True, text=True, cwd=test_dir, check=False, ) - assert result.returncode == 0, ( - f"sync --allow-build failed: {result.stdout}\n{result.stderr}" - ) + assert result.returncode == 0, f"sync failed: {result.stdout}\n{result.stderr}" assert is_package_installed(vendor_path, dep_name), ( - f"{dep_name} should be built and vendored into python_modules " - "when --allow-build is passed" + f"{dep_name} should be built and vendored into python_modules" ) -@pytest.mark.skipif( - sys.platform == "win32", - reason="FIXME Pyodide WASM interpreter cannot run setuptools build backends on Windows", -) -def test_sync_allow_build_via_pyproject_config(test_dir): - """End-to-end test for the [tool.pywrangler] allow-build config fallback. - - When no CLI flag is passed, sync should honor `allow-build = true` in the - [tool.pywrangler] table of pyproject.toml. - """ +def test_sync_reads_allow_build_from_pyproject_config(test_dir): + """`sync()` falls back to `[tool.pywrangler] allow-build` when no flag is passed.""" dep_name = "dummy-build-dep" dep_dir = create_dummy_build_dep(test_dir, dep_name) create_worker_pyproject_with_local_dep( @@ -521,21 +495,17 @@ def test_sync_allow_build_via_pyproject_config(test_dir): ) create_test_wrangler_jsonc(test_dir, "src/worker.py") - vendor_path = test_dir / "python_modules" - result = subprocess.run( - ["uv", "run", "pywrangler", "sync"], - capture_output=True, - text=True, - cwd=test_dir, - check=False, - ) - assert result.returncode == 0, ( - f"sync failed with [tool.pywrangler] allow-build = true: " - f"{result.stdout}\n{result.stderr}" - ) - assert is_package_installed(vendor_path, dep_name), ( - f"{dep_name} should be vendored when allow-build is enabled via config" - ) + with ( + patch.object(pywrangler_sync, "check_requirements_txt"), + patch.object(pywrangler_sync, "create_workers_venv"), + patch.object(pywrangler_sync, "create_pyodide_venv"), + patch.object(pywrangler_sync, "resolve_requirements") as mock_resolve, + patch.object(pywrangler_sync, "install_requirements") as mock_install, + ): + pywrangler_sync.sync(force=True) + + assert mock_resolve.call_args.kwargs["allow_build"] is True + assert mock_install.call_args.kwargs["allow_build"] is True def test_sync_command_handles_missing_pyproject(): diff --git a/packages/cli/tests/test_version_sync.py b/packages/cli/tests/test_version_sync.py index 5d62a877..3338f7fe 100644 --- a/packages/cli/tests/test_version_sync.py +++ b/packages/cli/tests/test_version_sync.py @@ -68,6 +68,66 @@ def test_get_vendor_package_versions_disables_color(): assert env.get("VIRTUAL_ENV") == str(Path("pyodide-venv")) +class TestInstallToVendorCommand: + """Check the `uv pip install` flags built by `_install_requirements_to_vendor`.""" + + @pytest.fixture + def plan(self, tmp_path: Path) -> InstallPlan: + lockfile = tmp_path / "pylock.toml" + lockfile.write_text( + 'lock-version = "1.0"\n' + '[[packages]]\nname = "click"\nversion = "8.1.7"\n' + '[[packages]]\nname = "local-dir"\n' + 'directory = { path = "../local-dir", editable = false }\n' + '[[packages]]\nname = "local-wheel"\nversion = "3.0"\n' + 'archive = { path = "dist/local_wheel-3.0-py3-none-any.whl" }\n' + ) + return InstallPlan(lockfile) + + def _run(self, plan: InstallPlan, tmp_path: Path, *, allow_build: bool) -> list: + with ( + patch.object(pywrangler_sync, "run_command") as mock_run, + patch.object(pywrangler_sync, "get_project_root", return_value=tmp_path), + patch.object( + pywrangler_sync, + "get_vendor_modules_path", + return_value=tmp_path / "python_modules", + ), + patch.object( + pywrangler_sync, + "get_pyodide_venv_path", + return_value=tmp_path / "pyodide-venv", + ), + patch.object(pywrangler_sync, "get_python_version", return_value="3.12"), + patch.object(pywrangler_sync, "_write_sync_token"), + patch.object(pywrangler_sync, "get_vendor_token_path"), + ): + mock_run.return_value.returncode = 1 + mock_run.return_value.stdout = "boom" + pywrangler_sync._install_requirements_to_vendor( + plan, allow_build=allow_build + ) + return mock_run.call_args[0][0] + + def test_default_builds_only_local_sources(self, plan, tmp_path): + command = self._run(plan, tmp_path, allow_build=False) + assert command[:3] == ["uv", "pip", "install"] + assert "--no-build" not in command + assert command[3:5] == ["--only-binary", ":all:"] + assert ["--no-binary", "local-dir"] == command[5:7] + # Local wheels must not be marked --no-binary or uv refuses them. + assert "local-wheel" not in command[: command.index("--refresh-package")] + assert command.count("--refresh-package") == 2 + assert "-r" in command + + def test_allow_build_lifts_binary_restriction(self, plan, tmp_path): + command = self._run(plan, tmp_path, allow_build=True) + assert "--only-binary" not in command + assert "--no-binary" not in command + assert "--no-build" not in command + assert command.count("--refresh-package") == 2 + + class TestInstallRequirements: @patch.object(pywrangler_sync, "_install_requirements_to_vendor") @patch.object(pywrangler_sync, "_get_vendor_package_versions") @@ -337,6 +397,26 @@ def test_empty_packages(self, tmp_path): plan = InstallPlan(lockfile) assert plan.requirements == [] + def test_classifies_local_sources(self, tmp_path): + """Local directories and sdists must be built; local wheels must not.""" + lockfile = tmp_path / "pylock.toml" + lockfile.write_text( + 'lock-version = "1.0"\n' + '[[packages]]\nname = "from-pypi"\nversion = "1.0"\n' + 'wheels = [{ url = "https://example.invalid/from_pypi-1.0-py3-none-any.whl" }]\n' + '[[packages]]\nname = "local-dir"\n' + 'directory = { path = "../local-dir", editable = false }\n' + '[[packages]]\nname = "local-sdist"\nversion = "2.0"\n' + 'sdist = { path = "dist/local_sdist-2.0.tar.gz" }\n' + '[[packages]]\nname = "local-wheel"\nversion = "3.0"\n' + 'archive = { path = "dist/local_wheel-3.0-py3-none-any.whl" }\n' + '[[packages]]\nname = "remote-archive"\nversion = "4.0"\n' + 'archive = { url = "https://example.invalid/remote_archive-4.0.tar.gz" }\n' + ) + plan = InstallPlan(lockfile) + assert plan.local_packages == ["local-dir", "local-sdist", "local-wheel"] + assert plan.local_build_packages == ["local-dir", "local-sdist"] + class TestResolveRequirements: @patch.object(pywrangler_resolve, "_compile_lockfile") From d2add47f1cab952b872028d66e5043b5c63868d4 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 18 Sep 2026 12:39:37 -0700 Subject: [PATCH 2/2] chore: Tidy up testlib; use paths to install runtime-sdk, django-cf & testlib --- packages/django-cf/tests/conftest.py | 61 ++++--------------- .../tests/in_worker/worker/pyproject.toml | 9 ++- .../tests/bindings-test/pyproject.toml | 4 +- packages/runtime-sdk/tests/conftest.py | 32 +++------- packages/runtime-sdk/tests/test_in_workerd.py | 21 ++----- .../django-async/pyproject.toml | 4 +- .../fastapi-tests/pyproject.toml | 4 +- .../flask-tests/pyproject.toml | 5 +- .../asgi-ws-disconnect/pyproject.toml | 5 +- .../tests/workerd-test/asgi/pyproject.toml | 5 +- .../durable-object-abort/pyproject.toml | 5 +- .../durable-object-inheritance/pyproject.toml | 5 +- .../durable-object-websocket/pyproject.toml | 5 +- .../durable-object/pyproject.toml | 5 +- .../entropy-patches/pyproject.toml | 4 ++ .../workerd-test/http-client/pyproject.toml | 5 +- .../workerd-test/python-rpc/pyproject.toml | 5 +- .../tests/workerd-test/sdk/pyproject.toml | 5 +- .../tests/workerd-test/wsgi/pyproject.toml | 5 +- packages/testlib/testlib/host.py | 45 +++++++++++++- 20 files changed, 134 insertions(+), 105 deletions(-) diff --git a/packages/django-cf/tests/conftest.py b/packages/django-cf/tests/conftest.py index ebd4244b..3a76d5d5 100644 --- a/packages/django-cf/tests/conftest.py +++ b/packages/django-cf/tests/conftest.py @@ -4,7 +4,6 @@ import os import shutil -import subprocess from collections.abc import Generator from dataclasses import dataclass from pathlib import Path @@ -15,6 +14,8 @@ COMPAT_CONFIGS, CompatConfig, configure_compatibility, + link_packages, + pywrangler_sync, ) from testlib.host import ( dev_server as run_dev_server, @@ -25,9 +26,6 @@ TEST_DIR: Path = Path(__file__).parent PACKAGE_DIR: Path = TEST_DIR.parent -WORKERS_PY: Path = PACKAGE_DIR.parent / "cli" -WORKERS_RUNTIME_SDK: Path = PACKAGE_DIR.parent / "runtime-sdk" / "src" -TESTLIB: Path = PACKAGE_DIR.parent / "testlib" DJANGO_CF_SRC: Path = PACKAGE_DIR / "django_cf" D1_PROJECT: Path = PACKAGE_DIR / "templates" / "d1" @@ -80,24 +78,11 @@ def _serve(project_dir: Path, tmp_path: Path) -> Generator[DevServer]: target = tmp_path / project_dir.name shutil.copytree(project_dir, target, ignore=GENERATED) - pywrangler = ["uv", "run", "--with", str(WORKERS_PY), "pywrangler"] env = os.environ | {"WORKERS_CI": "1"} + pywrangler_sync(target, env) - sync = subprocess.run( - [*pywrangler, "sync"], - cwd=target, - env=env, - capture_output=True, - text=True, - check=False, - ) - if sync.returncode != 0: - pytest.fail( - f"pywrangler sync failed for {project_dir.name}\n{sync.stdout}\n{sync.stderr}" - ) - - # `sync` vendors the released django-cf from PyPI; tests must exercise the - # working tree instead. + # These are deployable example apps, so their pyproject.toml depends on the + # released django-cf from PyPI. Tests must exercise the working tree instead. vendored = target / "python_modules" / "django_cf" shutil.rmtree(vendored, ignore_errors=True) shutil.copytree( @@ -108,7 +93,6 @@ def _serve(project_dir: Path, tmp_path: Path) -> Generator[DevServer]: target, tmp_path, env, - pywrangler, startup_timeout=DEV_STARTUP_TIMEOUT, ) as (base_url, log_path): _seed(base_url, log_path) @@ -134,11 +118,6 @@ def r2_web_server(tmp_path_factory: pytest.TempPathFactory) -> Generator[DevServ yield from _serve(R2_PROJECT, tmp_path_factory.mktemp("r2")) -@pytest.fixture(scope="session", autouse=True) -def build_testlib(): - subprocess.run(["uv", "build"], cwd=TESTLIB, check=True) - - @pytest.fixture( scope="module", params=COMPAT_CONFIGS, @@ -154,44 +133,26 @@ def dev_server( ) -> Generator[str]: """Serve ``tests/in_worker/worker``, once per compat config. - Unlike the app fixtures above, this one runs ``uv run --no-project`` and - vendors the runtime SDK and django-cf working trees by hand: the worker has - no Django project to build, it only needs the two libraries importable. + Unlike the app fixtures above, the worker's pyproject.toml depends on the + working-tree django-cf, runtime-sdk and testlib via ``../packages/...`` + path sources, which the ``packages`` symlink makes resolvable, so ``sync`` + vendors them directly. """ tmp_path = tmp_path_factory.mktemp("in_worker") target = tmp_path / IN_WORKER_PROJECT.name shutil.copytree(IN_WORKER_PROJECT, target, ignore=GENERATED) - shutil.copytree(TESTLIB, tmp_path / "testlib", ignore=GENERATED) + link_packages(tmp_path) wrangler_jsonc = target / "wrangler.jsonc" configure_compatibility(wrangler_jsonc, compat_config) - pywrangler = [ - "uv", - "run", - "--frozen", - "--no-project", - "--with", - str(WORKERS_PY), - "pywrangler", - ] env = os.environ | {"_PYODIDE_EXTRA_MOUNTS": str(tmp_path)} - - subprocess.run([*pywrangler, "sync"], cwd=target, check=True, env=env) - - shutil.copytree(WORKERS_RUNTIME_SDK, target / "python_modules", dirs_exist_ok=True) - shutil.copytree( - DJANGO_CF_SRC, - target / "python_modules" / "django_cf", - dirs_exist_ok=True, - ignore=shutil.ignore_patterns("__pycache__"), - ) + pywrangler_sync(target, env) with run_dev_server( target, tmp_path, env, - pywrangler, startup_timeout=DEV_STARTUP_TIMEOUT, ) as (base_url, _): yield base_url diff --git a/packages/django-cf/tests/in_worker/worker/pyproject.toml b/packages/django-cf/tests/in_worker/worker/pyproject.toml index ca2edead..368cea73 100644 --- a/packages/django-cf/tests/in_worker/worker/pyproject.toml +++ b/packages/django-cf/tests/in_worker/worker/pyproject.toml @@ -4,11 +4,18 @@ version = "0.0.0" requires-python = ">=3.12" dependencies = [ "django<6.1", + "django-cf", "pytest", "pytest-asyncio<1.2.0", "sqlparse", "testlib", + "workers-runtime-sdk", ] +# `../packages` is a symlink to the monorepo's `packages/` directory that the +# host-side `dev_server` fixture creates next to its copy of this project, so +# the tests exercise the working-tree checkouts rather than PyPI releases. [tool.uv.sources] -testlib = { path = "../testlib/dist/testlib-0.0.0-py3-none-any.whl" } +django-cf = { path = "../packages/django-cf" } +testlib = { path = "../packages/testlib" } +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/bindings-test/pyproject.toml b/packages/runtime-sdk/tests/bindings-test/pyproject.toml index 0db629d3..3c80a23d 100644 --- a/packages/runtime-sdk/tests/bindings-test/pyproject.toml +++ b/packages/runtime-sdk/tests/bindings-test/pyproject.toml @@ -13,7 +13,9 @@ dependencies = [ "asyncpg; python_version >= '3.14'", "cryptography", "testlib", + "workers-runtime-sdk", ] [tool.uv.sources] -testlib = { path = "../testlib/dist/testlib-0.0.0-py3-none-any.whl" } +testlib = { path = "../packages/testlib" } +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/conftest.py b/packages/runtime-sdk/tests/conftest.py index 10ed8d9b..845af20b 100644 --- a/packages/runtime-sdk/tests/conftest.py +++ b/packages/runtime-sdk/tests/conftest.py @@ -2,7 +2,6 @@ import os import shutil -import subprocess from collections.abc import Generator from pathlib import Path from typing import Any @@ -12,6 +11,8 @@ COMPAT_CONFIGS, CompatConfig, configure_compatibility, + link_packages, + pywrangler_sync, ) from testlib.host import ( dev_server as run_dev_server, @@ -21,9 +22,6 @@ ) TEST_DIR: Path = Path(__file__).parent -WORKERS_PY: Path = TEST_DIR.parent.parent / "cli" -WORKERS_RUNTIME_SDK: Path = TEST_DIR.parent / "src" -TESTLIB: Path = TEST_DIR.parent.parent / "testlib" DEV_STARTUP_TIMEOUT: int = 120 OPT_IN_MARKERS: tuple[str, ...] = ("hyperdrive",) @@ -43,11 +41,6 @@ def pytest_collection_modifyitems( item.add_marker(skip) -@pytest.fixture(scope="session", autouse=True) -def build_testlib(): - subprocess.run(["uv", "build"], cwd=TESTLIB, check=True) - - @pytest.fixture( scope="module", params=COMPAT_CONFIGS, @@ -74,32 +67,27 @@ def dev_server( worker_project_dir: Path, compat_config: CompatConfig, ) -> Generator[str]: - """Start a pywrangler dev server on a free port and yield its base URL.""" + """Start a pywrangler dev server on a free port and yield its base URL. + + The project is copied next to a ``packages`` symlink so that its + ``../packages/...`` sources resolve and ``sync`` vendors the working-tree + testlib and runtime-sdk. + """ tmp_path = tmp_path_factory.mktemp(f"{worker_project_dir.name}_dev") target = tmp_path / worker_project_dir.name shutil.copytree(worker_project_dir, target, ignore=shutil.ignore_patterns(".venv")) - shutil.copytree(TESTLIB, tmp_path / "testlib") + link_packages(tmp_path) env = os.environ | {"_PYODIDE_EXTRA_MOUNTS": str(tmp_path)} wrangler_jsonc = target / "wrangler.jsonc" configure_compatibility(wrangler_jsonc, compat_config) - pywrangler_cmd = ["uv", "run", "--no-project", "--with", WORKERS_PY, "pywrangler"] - - subprocess.run( - [*pywrangler_cmd, "sync"], - cwd=target, - check=True, - env=env, - ) - - shutil.copytree(WORKERS_RUNTIME_SDK, target / "python_modules", dirs_exist_ok=True) + pywrangler_sync(target, env) with run_dev_server( target, tmp_path, env, - pywrangler_cmd, startup_timeout=DEV_STARTUP_TIMEOUT, readiness_path="/health", require_success=True, diff --git a/packages/runtime-sdk/tests/test_in_workerd.py b/packages/runtime-sdk/tests/test_in_workerd.py index cbfce3b7..462d1c72 100644 --- a/packages/runtime-sdk/tests/test_in_workerd.py +++ b/packages/runtime-sdk/tests/test_in_workerd.py @@ -8,12 +8,12 @@ COMPAT_CONFIGS, CompatConfig, configure_compatibility, + link_packages, + pywrangler_sync, ) TEST_DIR = Path(__file__).parent WORKERD_TESTS = TEST_DIR / "workerd-test" -WORKERS_PY = TEST_DIR.parent.parent / "cli" -WORKERS_RUNTIME_SDK = TEST_DIR.parent / "src" DISK_SERVICE_NAME = "TEST_TMPDIR" @@ -110,23 +110,14 @@ def test_in_workerd( # noqa: PLR0913, PLR0917 (too-many-arguments) target = tmp_path / test_dir.name disk_service_dir = target / DISK_SERVICE_NAME shutil.copytree(test_dir, target, ignore=shutil.ignore_patterns(".venv")) + # Makes the project's `../packages/runtime-sdk` source resolve so `sync` + # vendors the working-tree SDK. + link_packages(tmp_path) disk_service_dir.mkdir(exist_ok=True) configure_compatibility(target / "wrangler.jsonc", compat_config) - pywrangler_cmd = ["uv", "run", "--no-project", "--with", WORKERS_PY, "pywrangler"] - - subprocess.run( - [*pywrangler_cmd, "sync"], - cwd=target, - check=True, - env=os.environ | {"_PYODIDE_EXTRA_MOUNTS": str(tmp_path)}, - ) - - # Copy runtime-sdk to the python modules as well - # FIXME: remove this and pass runtime-sdk as a dependency explicitly after - # https://github.com/cloudflare/workers-py/pull/81 is merged - shutil.copytree(WORKERS_RUNTIME_SDK, target / "python_modules", dirs_exist_ok=True) + pywrangler_sync(target, os.environ | {"_PYODIDE_EXTRA_MOUNTS": str(tmp_path)}) modules = embed(target / "python_modules", target, level=1) + embed( target / "tests", target, level=1 diff --git a/packages/runtime-sdk/tests/web-frameworks-test/django-async/pyproject.toml b/packages/runtime-sdk/tests/web-frameworks-test/django-async/pyproject.toml index f6a3ecfc..6793d371 100644 --- a/packages/runtime-sdk/tests/web-frameworks-test/django-async/pyproject.toml +++ b/packages/runtime-sdk/tests/web-frameworks-test/django-async/pyproject.toml @@ -11,7 +11,9 @@ dependencies = [ "pytest", "pytest-asyncio<1.2.0", "testlib", + "workers-runtime-sdk", ] [tool.uv.sources] -testlib = { path = "../testlib/dist/testlib-0.0.0-py3-none-any.whl" } +testlib = { path = "../packages/testlib" } +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/pyproject.toml b/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/pyproject.toml index 69808178..9f7548c9 100644 --- a/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/pyproject.toml +++ b/packages/runtime-sdk/tests/web-frameworks-test/fastapi-tests/pyproject.toml @@ -8,7 +8,9 @@ dependencies = [ "pytest", "pytest-asyncio<1.2.0", "testlib", + "workers-runtime-sdk", ] [tool.uv.sources] -testlib = { path = "../testlib/dist/testlib-0.0.0-py3-none-any.whl" } +testlib = { path = "../packages/testlib" } +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/web-frameworks-test/flask-tests/pyproject.toml b/packages/runtime-sdk/tests/web-frameworks-test/flask-tests/pyproject.toml index b3d7af83..93514e48 100644 --- a/packages/runtime-sdk/tests/web-frameworks-test/flask-tests/pyproject.toml +++ b/packages/runtime-sdk/tests/web-frameworks-test/flask-tests/pyproject.toml @@ -2,7 +2,8 @@ name = "flask-tests" version = "0.1.0" requires-python = ">=3.12" -dependencies = ["flask", "pytest", "testlib"] +dependencies = ["flask", "pytest", "testlib", "workers-runtime-sdk"] [tool.uv.sources] -testlib = { path = "../testlib/dist/testlib-0.0.0-py3-none-any.whl" } +testlib = { path = "../packages/testlib" } +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/asgi-ws-disconnect/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/asgi-ws-disconnect/pyproject.toml index f7d33188..7fcc72fb 100644 --- a/packages/runtime-sdk/tests/workerd-test/asgi-ws-disconnect/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/asgi-ws-disconnect/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = ["pytest", "pytest-asyncio<1.2.0"] +dependencies = ["pytest", "pytest-asyncio<1.2.0", "workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/asgi/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/asgi/pyproject.toml index f7d33188..7fcc72fb 100644 --- a/packages/runtime-sdk/tests/workerd-test/asgi/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/asgi/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = ["pytest", "pytest-asyncio<1.2.0"] +dependencies = ["pytest", "pytest-asyncio<1.2.0", "workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/durable-object-abort/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/durable-object-abort/pyproject.toml index 072f326e..704ab9e4 100644 --- a/packages/runtime-sdk/tests/workerd-test/durable-object-abort/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/durable-object-abort/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = [] +dependencies = ["workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/durable-object-inheritance/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/durable-object-inheritance/pyproject.toml index 072f326e..704ab9e4 100644 --- a/packages/runtime-sdk/tests/workerd-test/durable-object-inheritance/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/durable-object-inheritance/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = [] +dependencies = ["workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/durable-object-websocket/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/durable-object-websocket/pyproject.toml index 072f326e..704ab9e4 100644 --- a/packages/runtime-sdk/tests/workerd-test/durable-object-websocket/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/durable-object-websocket/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = [] +dependencies = ["workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/durable-object/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/durable-object/pyproject.toml index 072f326e..704ab9e4 100644 --- a/packages/runtime-sdk/tests/workerd-test/durable-object/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/durable-object/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = [] +dependencies = ["workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/entropy-patches/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/entropy-patches/pyproject.toml index ff72ddb6..087a4b9d 100644 --- a/packages/runtime-sdk/tests/workerd-test/entropy-patches/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/entropy-patches/pyproject.toml @@ -20,4 +20,8 @@ dependencies = [ "litestar", "opentelemetry-api", "mcp; python_version >= '3.14'", + "workers-runtime-sdk", ] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/http-client/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/http-client/pyproject.toml index 33345d4c..1178cd11 100644 --- a/packages/runtime-sdk/tests/workerd-test/http-client/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/http-client/pyproject.toml @@ -2,4 +2,7 @@ name = "http-client-compatibility-test" version = "0.0.0" requires-python = ">=3.14" -dependencies = ["pytest"] +dependencies = ["pytest", "workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/python-rpc/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/python-rpc/pyproject.toml index 072f326e..704ab9e4 100644 --- a/packages/runtime-sdk/tests/workerd-test/python-rpc/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/python-rpc/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = [] +dependencies = ["workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/sdk/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/sdk/pyproject.toml index f7d33188..7fcc72fb 100644 --- a/packages/runtime-sdk/tests/workerd-test/sdk/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/sdk/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = ["pytest", "pytest-asyncio<1.2.0"] +dependencies = ["pytest", "pytest-asyncio<1.2.0", "workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/runtime-sdk/tests/workerd-test/wsgi/pyproject.toml b/packages/runtime-sdk/tests/workerd-test/wsgi/pyproject.toml index f7d33188..7fcc72fb 100644 --- a/packages/runtime-sdk/tests/workerd-test/wsgi/pyproject.toml +++ b/packages/runtime-sdk/tests/workerd-test/wsgi/pyproject.toml @@ -2,4 +2,7 @@ name = "test" version = "0.0.0" requires-python = ">=3.12" -dependencies = ["pytest", "pytest-asyncio<1.2.0"] +dependencies = ["pytest", "pytest-asyncio<1.2.0", "workers-runtime-sdk"] + +[tool.uv.sources] +workers-runtime-sdk = { path = "../packages/runtime-sdk" } diff --git a/packages/testlib/testlib/host.py b/packages/testlib/testlib/host.py index 095c7eba..252168fe 100644 --- a/packages/testlib/testlib/host.py +++ b/packages/testlib/testlib/host.py @@ -19,6 +19,48 @@ SUITE_CONNECT_TIMEOUT = 10 SUITE_READ_TIMEOUT = 300 +# The monorepo's `packages/` directory. +PACKAGES: Path = Path(__file__).parents[2] +WORKERS_PY: Path = PACKAGES / "cli" +PY_WRANGLER_CMD: list[str] = [ + "uv", + "run", + "--no-project", + "--with", + str(WORKERS_PY), + "pywrangler", +] + + +def link_packages(tmp_path: Path) -> Path: + """Symlink the monorepo's ``packages/`` directory into *tmp_path*. + + Worker test projects are copied to ``tmp_path/`` before being synced, + so their ``[tool.uv.sources]`` entries refer to the working-tree checkouts + as ``../packages/``. This makes those paths resolve, so + ``pywrangler sync`` builds and vendors the local testlib, runtime-sdk and + django-cf instead of the PyPI releases. + """ + link = tmp_path / "packages" + link.symlink_to(PACKAGES, target_is_directory=True) + return link + + +def pywrangler_sync(cwd: Path, env: dict[str, str]) -> None: + """Run ``pywrangler sync`` in *cwd*, failing the test with its output on error.""" + result = subprocess.run( + [*PY_WRANGLER_CMD, "sync"], + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + pytest.fail( + f"pywrangler sync failed in {cwd}\n{result.stdout}\n{result.stderr}" + ) + @dataclass(frozen=True) class CompatConfig: @@ -130,7 +172,6 @@ def dev_server( target: Path, tmp_path: Path, env: dict[str, str], - pywrangler: list[str], *, startup_timeout: int, readiness_path: str = "", @@ -146,7 +187,7 @@ def dev_server( with log_path.open("w") as log_file: process = subprocess.Popen( [ - *pywrangler, + *PY_WRANGLER_CMD, "dev", "--port", str(port),