From 28fe46fa32deb836e1ba2945cab901c52d0d06ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 22 Sep 2026 15:17:26 +0200 Subject: [PATCH 1/8] WIP: GH-38536: [Python][Packaging] PoC to split libarrow_s3 into its own wheel --- ci/scripts/python_wheel_unix_test.sh | 3 +- ci/scripts/python_wheel_validate_contents.py | 3 +- ci/scripts/python_wheel_xlinux_build.sh | 9 +++- python/CMakeLists.txt | 8 +--- python/pyarrow-s3/CMakeLists.txt | 31 +++++++++++++ python/pyarrow-s3/pyarrow_s3/__init__.py | 31 +++++++++++++ python/pyarrow-s3/pyproject.toml | 47 ++++++++++++++++++++ python/pyarrow/fs.py | 24 +++++++--- 8 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 python/pyarrow-s3/CMakeLists.txt create mode 100644 python/pyarrow-s3/pyarrow_s3/__init__.py create mode 100644 python/pyarrow-s3/pyproject.toml diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh index cb445611e233..04d0400ee033 100755 --- a/ci/scripts/python_wheel_unix_test.sh +++ b/ci/scripts/python_wheel_unix_test.sh @@ -78,7 +78,8 @@ import pyarrow.parquet python -c "import pyarrow._gcsfs" fi if [ "${PYARROW_TEST_S3}" == "ON" ]; then - python -c "import pyarrow._s3fs" + # S3 ships in the separate pyarrow-s3 wheel; Load via pyarrow.fs. + python -c "import pyarrow.fs; pyarrow.fs.S3FileSystem" fi if [ "${PYARROW_TEST_FLIGHT}" == "ON" ]; then python -c "import pyarrow.flight" diff --git a/ci/scripts/python_wheel_validate_contents.py b/ci/scripts/python_wheel_validate_contents.py index 36956cf0b294..7ff157c706c7 100644 --- a/ci/scripts/python_wheel_validate_contents.py +++ b/ci/scripts/python_wheel_validate_contents.py @@ -40,7 +40,8 @@ def _count_docstrings(source): # TODO(GH-48970): Check stubs ARE present once annotations are complete def validate_wheel(path): p = Path(path) - wheels = list(p.glob('*.whl')) + # Only the core pyarrow wheel; pyarrow_s3-*.whl sits alongside it. + wheels = list(p.glob('pyarrow-*.whl')) error_msg = f"{len(wheels)} wheels found but only 1 expected ({wheels})" assert len(wheels) == 1, error_msg with zipfile.ZipFile(wheels[0]) as wheel_zip: diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index 0a5d63affc2e..c9aac14afabd 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -213,6 +213,13 @@ popd rm -rf dist/temp-fix-wheel +echo "=== (${PYTHON_VERSION}) Building pyarrow-s3 wheel ===" +# CMake pulls libarrow_s3 from /tmp/arrow-dist via CMAKE_PREFIX_PATH. +python -m build --wheel --no-isolation --outdir dist pyarrow-s3 + echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ===" -auditwheel repair dist/pyarrow-*.whl -w repaired_wheels +# libarrow ships in pyarrow and libarrow_s3 in pyarrow-s3: neither wheel +# may graft the other's library. +auditwheel repair --exclude 'libarrow_s3.so.*' dist/pyarrow-*.whl -w repaired_wheels +auditwheel repair --exclude 'libarrow.so.*' dist/pyarrow_s3-*.whl -w repaired_wheels popd diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7f6f1aebeef8..d0e3e9c77a17 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -746,12 +746,8 @@ if(PYARROW_BUILD_S3) endif() find_package(ArrowS3 REQUIRED) if(ARROW_BUILD_SHARED) - if(PYARROW_BUNDLE_ARROW_CPP) - bundle_arrow_lib(${ARROW_S3_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) - if(MSVC) - bundle_arrow_import_lib(${ARROW_S3_IMPORT_LIB}) - endif() - endif() + # We do not bundle libarrow_s3 as part of the main PyArrow wheel + # It ships in the separate pyarrow-s3 wheel (python/pyarrow-s3). set(S3_LINK_LIBS ArrowS3::arrow_s3_shared) else() set(S3_LINK_LIBS ArrowS3::arrow_s3_static) diff --git a/python/pyarrow-s3/CMakeLists.txt b/python/pyarrow-s3/CMakeLists.txt new file mode 100644 index 000000000000..3986f59f46ef --- /dev/null +++ b/python/pyarrow-s3/CMakeLists.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.25) + +# Nothing is compiled, but Arrow's CMake package expects a language enabled. +project(pyarrow_s3 LANGUAGES CXX) + +find_package(ArrowS3 REQUIRED) + +# Ship only the SONAME file, named the same way bundle_arrow_lib names it in +# python/CMakeLists.txt, so pyarrow's _s3fs resolves it by SONAME. +get_filename_component(ARROW_S3_LIB_REAL ${ARROW_S3_SHARED_LIB} REALPATH) +get_filename_component(ARROW_S3_LIB_NAME ${ARROW_S3_SHARED_LIB} NAME_WE) +install(FILES ${ARROW_S3_LIB_REAL} + DESTINATION pyarrow_s3 + RENAME ${ARROW_S3_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}) diff --git a/python/pyarrow-s3/pyarrow_s3/__init__.py b/python/pyarrow-s3/pyarrow_s3/__init__.py new file mode 100644 index 000000000000..930f2e2cbac6 --- /dev/null +++ b/python/pyarrow-s3/pyarrow_s3/__init__.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ctypes +import glob +import os + +# libarrow_s3 depends on libarrow, which ships in the pyarrow wheel. +# Importing pyarrow first loads libarrow into the process, so the dynamic +# loader resolves libarrow_s3's dependency on it by SONAME. +import pyarrow # noqa: F401 + +# Keep a reference so the library stays loaded for the process lifetime. +_libarrow_s3 = ctypes.CDLL( + glob.glob(os.path.join(os.path.dirname(__file__), "libarrow_s3.so.*"))[0], + mode=ctypes.RTLD_GLOBAL, +) diff --git a/python/pyarrow-s3/pyproject.toml b/python/pyarrow-s3/pyproject.toml new file mode 100644 index 000000000000..2f298acf0814 --- /dev/null +++ b/python/pyarrow-s3/pyproject.toml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["scikit-build-core>=1.0", "setuptools_scm[toml]>=8"] +build-backend = "scikit_build_core.build" + +[project] +name = "pyarrow-s3" +dynamic = ["version", "dependencies"] +requires-python = ">=3.11" +description = "S3 filesystem support for PyArrow" +license = "Apache-2.0" +# TODO: Add the license files + +[tool.scikit-build] +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" +wheel.packages = ["pyarrow_s3"] +# The wheel ships a shared library but no CPython extension: one wheel per +# platform (py3-none-) installed to platlib, next to pyarrow. +wheel.py-api = "py3" +wheel.platlib = true + +# Pin pyarrow to exactly this package's version. +[tool.scikit-build.metadata.dependencies] +provider = "scikit_build_core.metadata.template" +result = ["pyarrow=={project[version]}"] + +[tool.setuptools_scm] +root = '../..' +version_scheme = 'guess-next-dev' +git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' +fallback_version = '26.0.0a0' diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index 670ccaaf2455..7c8fb1f63cf3 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -39,6 +39,7 @@ FileStats = FileInfo _not_imported = [] +_not_imported_reasons = {} try: from pyarrow._azurefs import AzureFileSystem # noqa except ImportError: @@ -54,13 +55,26 @@ except ImportError: _not_imported.append("GcsFileSystem") +try: + # S3 support can be installed separately, as the pyarrow-s3 package. + # Importing it loads libarrow_s3 so that pyarrow._s3fs can be imported. + import pyarrow_s3 # noqa: F401 +except ModuleNotFoundError: + pass + try: from pyarrow._s3fs import ( # noqa AwsDefaultS3RetryStrategy, AwsStandardS3RetryStrategy, S3FileSystem, S3LogLevel, S3RetryStrategy, ensure_s3_initialized, finalize_s3, ensure_s3_finalized, initialize_s3, resolve_s3_region) -except ImportError: +except ImportError as exc: _not_imported.append("S3FileSystem") + if not isinstance(exc, ModuleNotFoundError): + # pyarrow._s3fs exists, but libarrow_s3 could not be loaded. + _not_imported_reasons["S3FileSystem"] = ( + f"{exc}. If pyarrow was installed from PyPI, S3 support is " + "provided by the separate 'pyarrow-s3' package" + ) else: # GH-38364: we don't initialize S3 eagerly as that could lead # to crashes at shutdown even when S3 isn't used. @@ -72,10 +86,10 @@ def __getattr__(name): if name in _not_imported: - raise ImportError( - "The pyarrow installation is not built with support for " - f"'{name}'" - ) + msg = f"The pyarrow installation is not built with support for '{name}'" + if name in _not_imported_reasons: + msg += f" ({_not_imported_reasons[name]})" + raise ImportError(msg) raise AttributeError( f"module 'pyarrow.fs' has no attribute '{name}'" From 84f8c7e041710333f13468332e763d5cfdac21bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 22 Sep 2026 15:19:35 +0200 Subject: [PATCH 2/8] Allow PyArrow to have optional dependencies --- python/pyproject.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 68c1a807dd51..936709274f6d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -29,7 +29,7 @@ build-backend = "scikit_build_core.build" [project] name = "pyarrow" -dynamic = ["version"] +dynamic = ["version", "optional-dependencies"] requires-python = ">=3.11" description = "Python library for Apache Arrow" readme = {file = "README.md", content-type = "text/markdown"} @@ -104,6 +104,11 @@ PYARROW_GENERATE_COVERAGE = {env = "PYARROW_GENERATE_COVERAGE", default = "OFF"} PYARROW_CXXFLAGS = {env = "PYARROW_CXXFLAGS", default = ""} PYARROW_REQUIRE_STUB_DOCSTRINGS = {env = "PYARROW_REQUIRE_STUB_DOCSTRINGS", default = "OFF"} +# Optional components shipped as separate wheels, pinned to this exact version. +[tool.scikit-build.metadata.optional-dependencies] +provider = "scikit_build_core.metadata.template" +result = { s3 = ["pyarrow-s3=={project[version]}"] } + [tool.setuptools_scm] root = '..' version_file = 'pyarrow/_generated_version.py' From db8b996a786c603ad51997ab59b1e01a67c77231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 22 Sep 2026 15:59:10 +0200 Subject: [PATCH 3/8] Cover from_uri case when pyarrow_s3 not present --- ci/scripts/python_wheel_unix_test.sh | 27 +++++++++++++++++++++++-- ci/scripts/python_wheel_xlinux_build.sh | 12 +++++++---- python/pyarrow/_fs.pyx | 18 +++++++++++++++-- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh index 04d0400ee033..fa4c0289f15c 100755 --- a/ci/scripts/python_wheel_unix_test.sh +++ b/ci/scripts/python_wheel_unix_test.sh @@ -58,8 +58,31 @@ export ARROW_TEST_DATA=${source_dir}/testing/data export PARQUET_TEST_DATA=${source_dir}/cpp/submodules/parquet-testing/data if [ "${INSTALL_PYARROW}" == "ON" ]; then - # Install the built wheels - python -m pip install "${source_dir}"/python/repaired_wheels/*.whl + # TODO: We probably want to cover this case in isolation on it's own internal unit test. + # Install the core wheel alone first: S3 ships in the separate pyarrow-s3 + # wheel, so check that S3 fails with an actionable message without it. + python -m pip install "${source_dir}"/python/repaired_wheels/pyarrow-*.whl + if [ "${ARROW_S3}" == "ON" ]; then + python -c " +import pyarrow.fs as fs +try: + fs.S3FileSystem +except ImportError as e: + assert 'pyarrow-s3' in str(e), e +else: + raise AssertionError('S3FileSystem available without pyarrow-s3') +try: + fs.FileSystem.from_uri('s3://bucket/key?region=us-east-1') +except ValueError as e: + assert 'pyarrow-s3' in str(e), e +else: + raise AssertionError('from_uri resolved s3:// without pyarrow-s3') +" + python -m pip install "${source_dir}"/python/repaired_wheels/pyarrow_s3-*.whl + # With pyarrow-s3 installed, from_uri must work even if pyarrow.fs was + # never imported (it loads pyarrow-s3 on demand). + python -c "from pyarrow._fs import FileSystem; FileSystem.from_uri('s3://bucket/key?region=us-east-1')" + fi fi if [ "${CHECK_IMPORTS}" == "ON" ]; then diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index c9aac14afabd..c3dcfc1afd68 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -213,13 +213,17 @@ popd rm -rf dist/temp-fix-wheel -echo "=== (${PYTHON_VERSION}) Building pyarrow-s3 wheel ===" -# CMake pulls libarrow_s3 from /tmp/arrow-dist via CMAKE_PREFIX_PATH. -python -m build --wheel --no-isolation --outdir dist pyarrow-s3 +if [ "${ARROW_S3}" == "ON" ]; then + echo "=== (${PYTHON_VERSION}) Building pyarrow-s3 wheel ===" + # CMake pulls libarrow_s3 from /tmp/arrow-dist via CMAKE_PREFIX_PATH. + python -m build --wheel --no-isolation --outdir dist pyarrow-s3 +fi echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ===" # libarrow ships in pyarrow and libarrow_s3 in pyarrow-s3: neither wheel # may graft the other's library. auditwheel repair --exclude 'libarrow_s3.so.*' dist/pyarrow-*.whl -w repaired_wheels -auditwheel repair --exclude 'libarrow.so.*' dist/pyarrow_s3-*.whl -w repaired_wheels +if [ "${ARROW_S3}" == "ON" ]; then + auditwheel repair --exclude 'libarrow.so.*' dist/pyarrow_s3-*.whl -w repaired_wheels +fi popd diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index 0739b6acba32..59ac8aaebb09 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -21,7 +21,7 @@ from cpython.datetime cimport datetime, PyDateTime_DateTime from pyarrow.includes.common cimport * from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint -from pyarrow.lib import _detect_compression, frombytes, tobytes +from pyarrow.lib import ArrowInvalid, _detect_compression, frombytes, tobytes from pyarrow.lib cimport * from pyarrow.util import _stringify_path @@ -499,8 +499,22 @@ cdef class FileSystem(_Weakrefable): """ if isinstance(uri, str) and uri.startswith(("fsspec+", "hf://")): return FileSystem._fsspec_from_uri(uri) - else: + try: return FileSystem._native_from_uri(uri) + except ArrowInvalid as exc: + if not (isinstance(uri, str) and uri.startswith("s3://") + and "Unrecognized filesystem type" in str(exc)): + raise + # S3 support can ship separately, as the pyarrow-s3 package. + # Importing pyarrow.fs loads it if installed, which registers + # the s3 scheme. + import pyarrow.fs + if "S3FileSystem" not in pyarrow.fs._not_imported: + return FileSystem._native_from_uri(uri) + reason = pyarrow.fs._not_imported_reasons.get("S3FileSystem") + if reason is None: + raise + raise ArrowInvalid(f"{exc} ({reason})") from None cdef init(self, const shared_ptr[CFileSystem]& wrapped): self.wrapped = wrapped From f98afb6142486a32664c24c45f30fcbeabfb0cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 22 Sep 2026 16:44:49 +0200 Subject: [PATCH 4/8] Several macOS fixes --- ci/scripts/python_wheel_macos_build.sh | 8 ++++++++ python/pyarrow-s3/CMakeLists.txt | 14 +++++++++++--- python/pyarrow-s3/pyarrow_s3/__init__.py | 8 +++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 4724f8e15246..d71df665c93e 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -193,3 +193,11 @@ fi # Move the verified wheels mkdir -p "${source_dir}/python/repaired_wheels" mv "${source_dir}"/python/dist/*.whl "${source_dir}"/python/repaired_wheels/ + +if [ "${ARROW_S3}" == "ON" ]; then + echo "=== (${PYTHON_VERSION}) Building pyarrow-s3 wheel ===" + # CMake pulls libarrow_s3 from the Arrow install via CMAKE_PREFIX_PATH. + python -m build --wheel --no-isolation \ + --outdir "${source_dir}/python/repaired_wheels" \ + "${source_dir}/python/pyarrow-s3" +fi diff --git a/python/pyarrow-s3/CMakeLists.txt b/python/pyarrow-s3/CMakeLists.txt index 3986f59f46ef..aaea4ddfb45d 100644 --- a/python/pyarrow-s3/CMakeLists.txt +++ b/python/pyarrow-s3/CMakeLists.txt @@ -22,10 +22,18 @@ project(pyarrow_s3 LANGUAGES CXX) find_package(ArrowS3 REQUIRED) -# Ship only the SONAME file, named the same way bundle_arrow_lib names it in -# python/CMakeLists.txt, so pyarrow's _s3fs resolves it by SONAME. +# Ship only the versioned library, named the same way bundle_arrow_lib names +# it in python/CMakeLists.txt, so pyarrow's _s3fs resolves it by its SONAME +# (Linux) or install name (macOS). get_filename_component(ARROW_S3_LIB_REAL ${ARROW_S3_SHARED_LIB} REALPATH) get_filename_component(ARROW_S3_LIB_NAME ${ARROW_S3_SHARED_LIB} NAME_WE) +if(APPLE) + set(ARROW_S3_LIB_INSTALLED_NAME + ${ARROW_S3_LIB_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}) +else() + set(ARROW_S3_LIB_INSTALLED_NAME + ${ARROW_S3_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}) +endif() install(FILES ${ARROW_S3_LIB_REAL} DESTINATION pyarrow_s3 - RENAME ${ARROW_S3_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION}) + RENAME ${ARROW_S3_LIB_INSTALLED_NAME}) diff --git a/python/pyarrow-s3/pyarrow_s3/__init__.py b/python/pyarrow-s3/pyarrow_s3/__init__.py index 930f2e2cbac6..fc1c3cb28df7 100644 --- a/python/pyarrow-s3/pyarrow_s3/__init__.py +++ b/python/pyarrow-s3/pyarrow_s3/__init__.py @@ -18,14 +18,20 @@ import ctypes import glob import os +import sys # libarrow_s3 depends on libarrow, which ships in the pyarrow wheel. # Importing pyarrow first loads libarrow into the process, so the dynamic # loader resolves libarrow_s3's dependency on it by SONAME. import pyarrow # noqa: F401 +if sys.platform == "darwin": + _pattern = "libarrow_s3.*.dylib" +else: + _pattern = "libarrow_s3.so.*" + # Keep a reference so the library stays loaded for the process lifetime. _libarrow_s3 = ctypes.CDLL( - glob.glob(os.path.join(os.path.dirname(__file__), "libarrow_s3.so.*"))[0], + glob.glob(os.path.join(os.path.dirname(__file__), _pattern))[0], mode=ctypes.RTLD_GLOBAL, ) From 2647a431354c4664fa9825b258408a2ee0a84653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 22 Sep 2026 19:51:15 +0200 Subject: [PATCH 5/8] Some Windows fixes --- ci/scripts/python_wheel_windows_build.bat | 19 ++++++++++++++++++- ci/scripts/python_wheel_windows_test.bat | 9 ++++++++- python/pyarrow-s3/CMakeLists.txt | 11 ++++++++++- python/pyarrow-s3/pyarrow_s3/__init__.py | 4 +++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 3805f750d450..7889ec997f89 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -151,8 +151,25 @@ pushd C:\arrow\python for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1 echo "Wheel name: %WHEEL_NAME%" +@REM arrow_s3.dll ships in the separate pyarrow-s3 wheel. %PYTHON_CMD% -m delvewheel repair -vv ^ - --ignore-existing --with-mangle ^ + --ignore-existing --with-mangle --exclude arrow_s3.dll ^ -w repaired_wheels %WHEEL_NAME% || exit /B 1 +if not "%ARROW_S3%"=="ON" goto :skip_pyarrow_s3 + +@REM Build pyarrow-s3: CMake pulls arrow_s3.dll from C:\arrow-dist via +@REM CMAKE_PREFIX_PATH. +%PYTHON_CMD% -m build --wheel --no-isolation --outdir dist-s3 pyarrow-s3 || exit /B 1 + +for /f %%i in ('dir dist-s3\pyarrow_s3-*.whl /B') do (set S3_WHEEL_NAME=%cd%\dist-s3\%%i) || exit /B 1 +echo "pyarrow-s3 wheel name: %S3_WHEEL_NAME%" + +@REM arrow.dll ships in the pyarrow wheel. +%PYTHON_CMD% -m delvewheel repair -vv ^ + --ignore-existing --with-mangle --exclude arrow.dll ^ + -w repaired_wheels %S3_WHEEL_NAME% || exit /B 1 + +:skip_pyarrow_s3 + popd diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat index 1e9cacac8bfa..b9f7c7009faa 100755 --- a/ci/scripts/python_wheel_windows_test.bat +++ b/ci/scripts/python_wheel_windows_test.bat @@ -42,12 +42,19 @@ py -0p @REM Install the built wheels %PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\repaired_wheels pyarrow || exit /B 1 +@REM S3 ships in the separate pyarrow-s3 wheel: with only pyarrow installed, +@REM check that S3 fails with an actionable message, then install it. +if "%PYARROW_TEST_S3%"=="ON" %PYTHON_CMD% C:\arrow\ci\scripts\python_wheel_check_s3_split.py || exit /B 1 +if "%PYARROW_TEST_S3%"=="ON" %PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\repaired_wheels pyarrow-s3 || exit /B 1 +@REM from_uri must work even if pyarrow.fs was never imported. +if "%PYARROW_TEST_S3%"=="ON" %PYTHON_CMD% -c "from pyarrow._fs import FileSystem; FileSystem.from_uri('s3://bucket/key?region=us-east-1')" || exit /B 1 + @REM Test that the modules are importable %PYTHON_CMD% -c "import pyarrow" || exit /B 1 %PYTHON_CMD% -c "import pyarrow._azurefs" || exit /B 1 %PYTHON_CMD% -c "import pyarrow._gcsfs" || exit /B 1 %PYTHON_CMD% -c "import pyarrow._hdfs" || exit /B 1 -%PYTHON_CMD% -c "import pyarrow._s3fs" || exit /B 1 +%PYTHON_CMD% -c "import pyarrow.fs; pyarrow.fs.S3FileSystem" || exit /B 1 %PYTHON_CMD% -c "import pyarrow.csv" || exit /B 1 %PYTHON_CMD% -c "import pyarrow.dataset" || exit /B 1 %PYTHON_CMD% -c "import pyarrow.flight" || exit /B 1 diff --git a/python/pyarrow-s3/CMakeLists.txt b/python/pyarrow-s3/CMakeLists.txt index aaea4ddfb45d..72af49c3c807 100644 --- a/python/pyarrow-s3/CMakeLists.txt +++ b/python/pyarrow-s3/CMakeLists.txt @@ -27,7 +27,9 @@ find_package(ArrowS3 REQUIRED) # (Linux) or install name (macOS). get_filename_component(ARROW_S3_LIB_REAL ${ARROW_S3_SHARED_LIB} REALPATH) get_filename_component(ARROW_S3_LIB_NAME ${ARROW_S3_SHARED_LIB} NAME_WE) -if(APPLE) +if(MSVC) + set(ARROW_S3_LIB_INSTALLED_NAME ${ARROW_S3_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) +elseif(APPLE) set(ARROW_S3_LIB_INSTALLED_NAME ${ARROW_S3_LIB_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}) else() @@ -37,3 +39,10 @@ endif() install(FILES ${ARROW_S3_LIB_REAL} DESTINATION pyarrow_s3 RENAME ${ARROW_S3_LIB_INSTALLED_NAME}) +if(MSVC) + # The import library, previously bundled in the pyarrow wheel, for C++ + # extensions that link against Arrow S3. + install(FILES ${ARROW_S3_IMPORT_LIB} + DESTINATION pyarrow_s3 + RENAME ${ARROW_S3_LIB_NAME}.lib) +endif() diff --git a/python/pyarrow-s3/pyarrow_s3/__init__.py b/python/pyarrow-s3/pyarrow_s3/__init__.py index fc1c3cb28df7..5078e79ee8bf 100644 --- a/python/pyarrow-s3/pyarrow_s3/__init__.py +++ b/python/pyarrow-s3/pyarrow_s3/__init__.py @@ -25,7 +25,9 @@ # loader resolves libarrow_s3's dependency on it by SONAME. import pyarrow # noqa: F401 -if sys.platform == "darwin": +if sys.platform == "win32": + _pattern = "arrow_s3.dll" +elif sys.platform == "darwin": _pattern = "libarrow_s3.*.dylib" else: _pattern = "libarrow_s3.so.*" From 495c04868e6d8333be20fda993731919b24d17e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Sep 2026 11:53:51 +0200 Subject: [PATCH 6/8] Add missing local file to commit --- ci/scripts/python_wheel_check_s3_split.py | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ci/scripts/python_wheel_check_s3_split.py diff --git a/ci/scripts/python_wheel_check_s3_split.py b/ci/scripts/python_wheel_check_s3_split.py new file mode 100644 index 000000000000..ad4248cd86b5 --- /dev/null +++ b/ci/scripts/python_wheel_check_s3_split.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Run with pyarrow installed but not pyarrow-s3: S3 use must fail with a +# message pointing at the pyarrow-s3 package. + +import pyarrow.fs as fs + +try: + fs.S3FileSystem +except ImportError as e: + assert "pyarrow-s3" in str(e), e +else: + raise AssertionError("S3FileSystem available without pyarrow-s3") + +try: + fs.FileSystem.from_uri("s3://bucket/key?region=us-east-1") +except ValueError as e: + assert "pyarrow-s3" in str(e), e +else: + raise AssertionError("from_uri resolved s3:// without pyarrow-s3") From 70bc497e7aee4387d7edbea460bd665d25a414bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Sep 2026 11:58:15 +0200 Subject: [PATCH 7/8] Add missing analyze-existing argument to delvewheel --- ci/scripts/python_wheel_windows_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 7889ec997f89..13925fe5ef4f 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -167,7 +167,7 @@ echo "pyarrow-s3 wheel name: %S3_WHEEL_NAME%" @REM arrow.dll ships in the pyarrow wheel. %PYTHON_CMD% -m delvewheel repair -vv ^ - --ignore-existing --with-mangle --exclude arrow.dll ^ + --ignore-existing --analyze-existing --with-mangle --exclude arrow.dll ^ -w repaired_wheels %S3_WHEEL_NAME% || exit /B 1 :skip_pyarrow_s3 From f7dce07ffc3f83bfeab4b9afc8130191f9c510a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 23 Sep 2026 13:25:20 +0200 Subject: [PATCH 8/8] Fix musllinux by adding INSTALL_RPATH /../pyarrow_s3 to _s3fs --- python/CMakeLists.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index d0e3e9c77a17..401038074803 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -119,10 +119,11 @@ if(UNIX) # loaded properly if(APPLE) set(CMAKE_INSTALL_NAME_DIR "@rpath") - set(CMAKE_INSTALL_RPATH "@loader_path/") + set(PYARROW_RPATH_ORIGIN "@loader_path") else() - set(CMAKE_INSTALL_RPATH "\$ORIGIN") + set(PYARROW_RPATH_ORIGIN "\$ORIGIN") endif() + set(CMAKE_INSTALL_RPATH "${PYARROW_RPATH_ORIGIN}/") endif() find_program(CCACHE_FOUND ccache) @@ -1050,6 +1051,15 @@ endif() if(PYARROW_BUILD_S3) target_link_libraries(_s3fs PRIVATE ${S3_LINK_LIBS}) + if(PYARROW_BUNDLE_ARROW_CPP AND UNIX) + # libarrow_s3 ships in the sibling pyarrow-s3 wheel, so it is not in any + # directory this extension already searches. Preloading it from + # pyarrow_s3/__init__.py is not enough: musl's loader does not match a + # DT_NEEDED entry against a library that was opened by an explicit path. + set_property(TARGET _s3fs + APPEND + PROPERTY INSTALL_RPATH "${PYARROW_RPATH_ORIGIN}/../pyarrow_s3") + endif() endif() #