GH-38536: [Python][Packaging] PoC to split libarrow_s3 into its own wheel

ci/scripts/python_wheel_check_s3_split.py

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Run with pyarrow installed but not pyarrow-s3: S3 use must fail with a
+# message pointing at the pyarrow-s3 package.
+
+import pyarrow.fs as fs
+
+try:
+    fs.S3FileSystem
+except ImportError as e:
+    assert "pyarrow-s3" in str(e), e
+else:
+    raise AssertionError("S3FileSystem available without pyarrow-s3")
+
+try:
+    fs.FileSystem.from_uri("s3://bucket/key?region=us-east-1")
+except ValueError as e:
+    assert "pyarrow-s3" in str(e), e
+else:
+    raise AssertionError("from_uri resolved s3:// without pyarrow-s3")

ci/scripts/python_wheel_macos_build.sh

@@ -193,3 +193,11 @@ fi
 # Move the verified wheels
 mkdir -p "${source_dir}/python/repaired_wheels"
 mv "${source_dir}"/python/dist/*.whl "${source_dir}"/python/repaired_wheels/
+
+if [ "${ARROW_S3}" == "ON" ]; then
+  echo "=== (${PYTHON_VERSION}) Building pyarrow-s3 wheel ==="
+  # CMake pulls libarrow_s3 from the Arrow install via CMAKE_PREFIX_PATH.
+  python -m build --wheel --no-isolation \
+    --outdir "${source_dir}/python/repaired_wheels" \
+    "${source_dir}/python/pyarrow-s3"
+fi

ci/scripts/python_wheel_unix_test.sh

@@ -58,8 +58,31 @@ export ARROW_TEST_DATA=${source_dir}/testing/data
 export PARQUET_TEST_DATA=${source_dir}/cpp/submodules/parquet-testing/data
 
 if [ "${INSTALL_PYARROW}" == "ON" ]; then
-  # Install the built wheels
-  python -m pip install "${source_dir}"/python/repaired_wheels/*.whl
+  # TODO: We probably want to cover this case in isolation on it's own internal unit test.
+  # Install the core wheel alone first: S3 ships in the separate pyarrow-s3
+  # wheel, so check that S3 fails with an actionable message without it.
+  python -m pip install "${source_dir}"/python/repaired_wheels/pyarrow-*.whl
+  if [ "${ARROW_S3}" == "ON" ]; then
+    python -c "
+import pyarrow.fs as fs
+try:
+    fs.S3FileSystem
+except ImportError as e:
+    assert 'pyarrow-s3' in str(e), e
+else:
+    raise AssertionError('S3FileSystem available without pyarrow-s3')
+try:
+    fs.FileSystem.from_uri('s3://bucket/key?region=us-east-1')
+except ValueError as e:
+    assert 'pyarrow-s3' in str(e), e
+else:
+    raise AssertionError('from_uri resolved s3:// without pyarrow-s3')
+"
+    python -m pip install "${source_dir}"/python/repaired_wheels/pyarrow_s3-*.whl
+    # With pyarrow-s3 installed, from_uri must work even if pyarrow.fs was
+    # never imported (it loads pyarrow-s3 on demand).
+    python -c "from pyarrow._fs import FileSystem; FileSystem.from_uri('s3://bucket/key?region=us-east-1')"
+  fi
 fi
 
 if [ "${CHECK_IMPORTS}" == "ON" ]; then
@@ -78,7 +101,8 @@ import pyarrow.parquet
     python -c "import pyarrow._gcsfs"
   fi
   if [ "${PYARROW_TEST_S3}" == "ON" ]; then
-    python -c "import pyarrow._s3fs"
+    # S3 ships in the separate pyarrow-s3 wheel; Load via pyarrow.fs.
+    python -c "import pyarrow.fs; pyarrow.fs.S3FileSystem"
   fi
   if [ "${PYARROW_TEST_FLIGHT}" == "ON" ]; then
     python -c "import pyarrow.flight"

ci/scripts/python_wheel_validate_contents.py

@@ -40,7 +40,8 @@ def _count_docstrings(source):
 # TODO(GH-48970): Check stubs ARE present once annotations are complete
 def validate_wheel(path):
     p = Path(path)
-    wheels = list(p.glob('*.whl'))
+    # Only the core pyarrow wheel; pyarrow_s3-*.whl sits alongside it.
+    wheels = list(p.glob('pyarrow-*.whl'))
     error_msg = f"{len(wheels)} wheels found but only 1 expected ({wheels})"
     assert len(wheels) == 1, error_msg
     with zipfile.ZipFile(wheels[0]) as wheel_zip:

ci/scripts/python_wheel_windows_build.bat

@@ -151,8 +151,25 @@ pushd C:\arrow\python
 for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1
 echo "Wheel name: %WHEEL_NAME%"
 
+@REM arrow_s3.dll ships in the separate pyarrow-s3 wheel.
 %PYTHON_CMD% -m delvewheel repair -vv ^
-    --ignore-existing --with-mangle ^
+    --ignore-existing --with-mangle --exclude arrow_s3.dll ^
     -w repaired_wheels %WHEEL_NAME% || exit /B 1
 
+if not "%ARROW_S3%"=="ON" goto :skip_pyarrow_s3
+
+@REM Build pyarrow-s3: CMake pulls arrow_s3.dll from C:\arrow-dist via
+@REM CMAKE_PREFIX_PATH.
+%PYTHON_CMD% -m build --wheel --no-isolation --outdir dist-s3 pyarrow-s3 || exit /B 1
+
+for /f %%i in ('dir dist-s3\pyarrow_s3-*.whl /B') do (set S3_WHEEL_NAME=%cd%\dist-s3\%%i) || exit /B 1
+echo "pyarrow-s3 wheel name: %S3_WHEEL_NAME%"
+
+@REM arrow.dll ships in the pyarrow wheel.
+%PYTHON_CMD% -m delvewheel repair -vv ^
+    --ignore-existing --analyze-existing --with-mangle --exclude arrow.dll ^
+    -w repaired_wheels %S3_WHEEL_NAME% || exit /B 1
+
+:skip_pyarrow_s3
+
 popd

ci/scripts/python_wheel_windows_test.bat

@@ -42,12 +42,19 @@ py -0p
 @REM Install the built wheels
 %PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\repaired_wheels pyarrow || exit /B 1
 
+@REM S3 ships in the separate pyarrow-s3 wheel: with only pyarrow installed,
+@REM check that S3 fails with an actionable message, then install it.
+if "%PYARROW_TEST_S3%"=="ON" %PYTHON_CMD% C:\arrow\ci\scripts\python_wheel_check_s3_split.py || exit /B 1
+if "%PYARROW_TEST_S3%"=="ON" %PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\repaired_wheels pyarrow-s3 || exit /B 1
+@REM from_uri must work even if pyarrow.fs was never imported.
+if "%PYARROW_TEST_S3%"=="ON" %PYTHON_CMD% -c "from pyarrow._fs import FileSystem; FileSystem.from_uri('s3://bucket/key?region=us-east-1')" || exit /B 1
+
 @REM Test that the modules are importable
 %PYTHON_CMD% -c "import pyarrow" || exit /B 1
 %PYTHON_CMD% -c "import pyarrow._azurefs" || exit /B 1
 %PYTHON_CMD% -c "import pyarrow._gcsfs" || exit /B 1
 %PYTHON_CMD% -c "import pyarrow._hdfs" || exit /B 1
-%PYTHON_CMD% -c "import pyarrow._s3fs" || exit /B 1
+%PYTHON_CMD% -c "import pyarrow.fs; pyarrow.fs.S3FileSystem" || exit /B 1
 %PYTHON_CMD% -c "import pyarrow.csv" || exit /B 1
 %PYTHON_CMD% -c "import pyarrow.dataset" || exit /B 1
 %PYTHON_CMD% -c "import pyarrow.flight" || exit /B 1

ci/scripts/python_wheel_xlinux_build.sh

@@ -213,6 +213,17 @@ popd
 
 rm -rf dist/temp-fix-wheel
 
+if [ "${ARROW_S3}" == "ON" ]; then
+  echo "=== (${PYTHON_VERSION}) Building pyarrow-s3 wheel ==="
+  # CMake pulls libarrow_s3 from /tmp/arrow-dist via CMAKE_PREFIX_PATH.
+  python -m build --wheel --no-isolation --outdir dist pyarrow-s3
+fi
+
 echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ==="
-auditwheel repair dist/pyarrow-*.whl -w repaired_wheels
+# libarrow ships in pyarrow and libarrow_s3 in pyarrow-s3: neither wheel
+# may graft the other's library.
+auditwheel repair --exclude 'libarrow_s3.so.*' dist/pyarrow-*.whl -w repaired_wheels
+if [ "${ARROW_S3}" == "ON" ]; then
+  auditwheel repair --exclude 'libarrow.so.*' dist/pyarrow_s3-*.whl -w repaired_wheels
+fi
 popd

python/CMakeLists.txt

@@ -119,10 +119,11 @@ if(UNIX)
   # loaded properly
   if(APPLE)
     set(CMAKE_INSTALL_NAME_DIR "@rpath")
-    set(CMAKE_INSTALL_RPATH "@loader_path/")
+    set(PYARROW_RPATH_ORIGIN "@loader_path")
   else()
-    set(CMAKE_INSTALL_RPATH "\$ORIGIN")
+    set(PYARROW_RPATH_ORIGIN "\$ORIGIN")
   endif()
+  set(CMAKE_INSTALL_RPATH "${PYARROW_RPATH_ORIGIN}/")
 endif()
 
 find_program(CCACHE_FOUND ccache)
@@ -746,12 +747,8 @@ if(PYARROW_BUILD_S3)
   endif()
   find_package(ArrowS3 REQUIRED)
   if(ARROW_BUILD_SHARED)
-    if(PYARROW_BUNDLE_ARROW_CPP)
-      bundle_arrow_lib(${ARROW_S3_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION})
-      if(MSVC)
-        bundle_arrow_import_lib(${ARROW_S3_IMPORT_LIB})
-      endif()
-    endif()
+    # We do not bundle libarrow_s3 as part of the main PyArrow wheel
+    # It ships in the separate pyarrow-s3 wheel (python/pyarrow-s3).
     set(S3_LINK_LIBS ArrowS3::arrow_s3_shared)
   else()
     set(S3_LINK_LIBS ArrowS3::arrow_s3_static)
@@ -1054,6 +1051,15 @@ endif()
 
 if(PYARROW_BUILD_S3)
   target_link_libraries(_s3fs PRIVATE ${S3_LINK_LIBS})
+  if(PYARROW_BUNDLE_ARROW_CPP AND UNIX)
+    # libarrow_s3 ships in the sibling pyarrow-s3 wheel, so it is not in any
+    # directory this extension already searches. Preloading it from
+    # pyarrow_s3/__init__.py is not enough: musl's loader does not match a
+    # DT_NEEDED entry against a library that was opened by an explicit path.
+    set_property(TARGET _s3fs
+                 APPEND
+                 PROPERTY INSTALL_RPATH "${PYARROW_RPATH_ORIGIN}/../pyarrow_s3")
+  endif()
 endif()
 
 #

python/pyarrow-s3/CMakeLists.txt

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.25)
+
+# Nothing is compiled, but Arrow's CMake package expects a language enabled.
+project(pyarrow_s3 LANGUAGES CXX)
+
+find_package(ArrowS3 REQUIRED)
+
+# Ship only the versioned library, named the same way bundle_arrow_lib names
+# it in python/CMakeLists.txt, so pyarrow's _s3fs resolves it by its SONAME
+# (Linux) or install name (macOS).
+get_filename_component(ARROW_S3_LIB_REAL ${ARROW_S3_SHARED_LIB} REALPATH)
+get_filename_component(ARROW_S3_LIB_NAME ${ARROW_S3_SHARED_LIB} NAME_WE)
+if(MSVC)
+  set(ARROW_S3_LIB_INSTALLED_NAME ${ARROW_S3_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+elseif(APPLE)
+  set(ARROW_S3_LIB_INSTALLED_NAME
+      ${ARROW_S3_LIB_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX})
+else()
+  set(ARROW_S3_LIB_INSTALLED_NAME
+      ${ARROW_S3_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_SO_VERSION})
+endif()
+install(FILES ${ARROW_S3_LIB_REAL}
+        DESTINATION pyarrow_s3
+        RENAME ${ARROW_S3_LIB_INSTALLED_NAME})
+if(MSVC)
+  # The import library, previously bundled in the pyarrow wheel, for C++
+  # extensions that link against Arrow S3.
+  install(FILES ${ARROW_S3_IMPORT_LIB}
+          DESTINATION pyarrow_s3
+          RENAME ${ARROW_S3_LIB_NAME}.lib)
+endif()

python/pyarrow-s3/pyarrow_s3/__init__.py

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import ctypes
+import glob
+import os
+import sys
+
+# libarrow_s3 depends on libarrow, which ships in the pyarrow wheel.
+# Importing pyarrow first loads libarrow into the process, so the dynamic
+# loader resolves libarrow_s3's dependency on it by SONAME.
+import pyarrow  # noqa: F401
+
+if sys.platform == "win32":
+    _pattern = "arrow_s3.dll"
+elif sys.platform == "darwin":
+    _pattern = "libarrow_s3.*.dylib"
+else:
+    _pattern = "libarrow_s3.so.*"
+
+# Keep a reference so the library stays loaded for the process lifetime.
+_libarrow_s3 = ctypes.CDLL(
+    glob.glob(os.path.join(os.path.dirname(__file__), _pattern))[0],
+    mode=ctypes.RTLD_GLOBAL,
+)

python/pyarrow-s3/pyproject.toml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[build-system]
+requires = ["scikit-build-core>=1.0", "setuptools_scm[toml]>=8"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "pyarrow-s3"
+dynamic = ["version", "dependencies"]
+requires-python = ">=3.11"
+description = "S3 filesystem support for PyArrow"
+license = "Apache-2.0"
+# TODO: Add the license files
+
+[tool.scikit-build]
+metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
+wheel.packages = ["pyarrow_s3"]
+# The wheel ships a shared library but no CPython extension: one wheel per
+# platform (py3-none-<platform>) installed to platlib, next to pyarrow.
+wheel.py-api = "py3"
+wheel.platlib = true
+
+# Pin pyarrow to exactly this package's version.
+[tool.scikit-build.metadata.dependencies]
+provider = "scikit_build_core.metadata.template"
+result = ["pyarrow=={project[version]}"]
+
+[tool.setuptools_scm]
+root = '../..'
+version_scheme = 'guess-next-dev'
+git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"'
+fallback_version = '26.0.0a0'

python/pyarrow/_fs.pyx

@@ -21,7 +21,7 @@ from cpython.datetime cimport datetime, PyDateTime_DateTime
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint
-from pyarrow.lib import _detect_compression, frombytes, tobytes
+from pyarrow.lib import ArrowInvalid, _detect_compression, frombytes, tobytes
 from pyarrow.lib cimport *
 from pyarrow.util import _stringify_path
 
@@ -499,8 +499,22 @@ cdef class FileSystem(_Weakrefable):
         """
         if isinstance(uri, str) and uri.startswith(("fsspec+", "hf://")):
             return FileSystem._fsspec_from_uri(uri)
-        else:
+        try:
             return FileSystem._native_from_uri(uri)
+        except ArrowInvalid as exc:
+            if not (isinstance(uri, str) and uri.startswith("s3://")
+                    and "Unrecognized filesystem type" in str(exc)):
+                raise
+            # S3 support can ship separately, as the pyarrow-s3 package.
+            # Importing pyarrow.fs loads it if installed, which registers
+            # the s3 scheme.
+            import pyarrow.fs
+            if "S3FileSystem" not in pyarrow.fs._not_imported:
+                return FileSystem._native_from_uri(uri)
+            reason = pyarrow.fs._not_imported_reasons.get("S3FileSystem")
+            if reason is None:
+                raise
+            raise ArrowInvalid(f"{exc} ({reason})") from None
 
     cdef init(self, const shared_ptr[CFileSystem]& wrapped):
         self.wrapped = wrapped