Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions c_glib/arrow-glib/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,9 @@ if not gio.found()
gio = dependency('gio-2.0')
endif
dependencies = [arrow_acero, arrow_compute, arrow, gobject, gio]
if arrow_s3.found()
dependencies += arrow_s3
endif
libarrow_glib = library(
'arrow-glib',
sources: sources + enums,
Expand Down
12 changes: 12 additions & 0 deletions c_glib/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,13 @@ if arrow_cpp_build_lib_dir == ''
modules: ['ArrowDataset::arrow_dataset_shared'],
required: false,
)
arrow_s3 = dependency(
'arrow-s3',
'ArrowS3',
kwargs: common_args,
modules: ['ArrowS3::arrow_s3_shared'],
required: false,
)
arrow_flight = dependency(
'arrow-flight',
'ArrowFlight',
Expand Down Expand Up @@ -235,6 +242,11 @@ main(void)
dirs: [arrow_cpp_build_lib_dir],
required: false,
)
arrow_s3 = cpp_compiler.find_library(
'arrow_s3',
dirs: [arrow_cpp_build_lib_dir],
required: false,
)
arrow_flight = cpp_compiler.find_library(
'arrow_flight',
dirs: [arrow_cpp_build_lib_dir],
Expand Down
1 change: 0 additions & 1 deletion ci/docker/conda-cpp.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ ENV ARROW_ACERO=ON \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_S3=ON \
ARROW_S3_MODULE=ON \
ARROW_SUBSTRAIT=ON \
ARROW_USE_CCACHE=ON \
ARROW_USE_MOLD=ON \
Expand Down
2 changes: 0 additions & 2 deletions ci/scripts/cpp_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
ARROW_JEMALLOC=OFF
ARROW_MIMALLOC=OFF
ARROW_S3=OFF
ARROW_S3_MODULE=OFF
ARROW_WITH_OPENTELEMETRY=OFF
fi

Expand Down Expand Up @@ -237,7 +236,6 @@ else
-DARROW_PARQUET="${ARROW_PARQUET:-OFF}" \
-DARROW_RUNTIME_SIMD_LEVEL="${ARROW_RUNTIME_SIMD_LEVEL:-MAX}" \
-DARROW_S3="${ARROW_S3:-OFF}" \
-DARROW_S3_MODULE="${ARROW_S3_MODULE:-OFF}" \
-DARROW_SIMD_LEVEL="${ARROW_SIMD_LEVEL:-DEFAULT}" \
-DARROW_SUBSTRAIT="${ARROW_SUBSTRAIT:-OFF}" \
-DARROW_TEST_LINKAGE="${ARROW_TEST_LINKAGE:-shared}" \
Expand Down
9 changes: 9 additions & 0 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,15 @@ if(ARROW_BUILD_STATIC)
endif()
set(ARROW_FLIGHT_PC_REQUIRES_PRIVATE "")

# For arrow-s3.pc.
set(ARROW_S3_PC_CFLAGS "")
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
if(ARROW_BUILD_STATIC)
string(APPEND ARROW_S3_PC_CFLAGS_PRIVATE " -DARROW_S3_STATIC")
endif()
set(ARROW_S3_PC_LIBS_PRIVATE "")
set(ARROW_S3_PC_REQUIRES_PRIVATE "")

# For arrow-substrait.pc.
set(ARROW_SUBSTRAIT_PC_CFLAGS "")
set(ARROW_SUBSTRAIT_PC_CFLAGS_PRIVATE "")
Expand Down
8 changes: 1 addition & 7 deletions cpp/cmake_modules/DefineOptions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -396,17 +396,11 @@ takes precedence over ccache if a storage backend is configured" ON)
ARROW_JSON)

define_option(ARROW_S3
"Build Arrow with S3 support (requires the AWS SDK for C++)"
"Build Arrow S3 Module (requires the AWS SDK for C++)"
OFF
DEPENDS
ARROW_FILESYSTEM)
Comment thread
raulcd marked this conversation as resolved.

define_option(ARROW_S3_MODULE
"Build the Arrow S3 filesystem as a dynamic module"
OFF
DEPENDS
ARROW_S3)

define_option(ARROW_SUBSTRAIT
"Build the Arrow Substrait Consumer Module"
OFF
Expand Down
12 changes: 8 additions & 4 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -4244,6 +4244,10 @@ if(ARROW_S3)
endif()
# Keep this in sync with s3fs.cc
resolve_dependency(AWSSDK
ARROW_CMAKE_PACKAGE_NAME
ArrowS3
ARROW_PC_PACKAGE_NAME
arrow-s3
HAVE_ALT
TRUE
REQUIRED_VERSION
Expand All @@ -4255,15 +4259,15 @@ if(ARROW_S3)
if(ARROW_BUILD_STATIC)
if(${AWSSDK_SOURCE} STREQUAL "SYSTEM")
foreach(AWSSDK_LINK_LIBRARY ${AWSSDK_LINK_LIBRARIES})
string(APPEND ARROW_PC_LIBS_PRIVATE " $<TARGET_FILE:${AWSSDK_LINK_LIBRARY}>")
string(APPEND ARROW_S3_PC_LIBS_PRIVATE " $<TARGET_FILE:${AWSSDK_LINK_LIBRARY}>")
endforeach()
else()
if(UNIX)
string(APPEND ARROW_PC_REQUIRES_PRIVATE " libcurl")
string(APPEND ARROW_S3_PC_REQUIRES_PRIVATE " libcurl")
endif()
string(APPEND ARROW_PC_REQUIRES_PRIVATE " openssl")
string(APPEND ARROW_S3_PC_REQUIRES_PRIVATE " openssl")
if(APPLE)
string(APPEND ARROW_PC_LIBS_PRIVATE " -framework Security")
string(APPEND ARROW_S3_PC_LIBS_PRIVATE " -framework Security")
endif()
endif()
endif()
Expand Down
44 changes: 44 additions & 0 deletions cpp/src/arrow/ArrowS3Config.cmake.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This config sets the following variables in your project::
#
# ArrowS3_FOUND - true if Arrow S3 found on the system
#
# This config sets the following targets in your project::
#
# ArrowS3::arrow_s3_shared - for linked as shared library if shared library is built
# ArrowS3::arrow_s3_static - for linked as static library if static library is built

@PACKAGE_INIT@

set(ARROW_S3_SYSTEM_DEPENDENCIES "@ARROW_S3_SYSTEM_DEPENDENCIES@")

include(CMakeFindDependencyMacro)
find_dependency(Arrow CONFIG)

if(ARROW_BUILD_STATIC)
arrow_find_dependencies("${ARROW_S3_SYSTEM_DEPENDENCIES}")
endif()

include("${CMAKE_CURRENT_LIST_DIR}/ArrowS3Targets.cmake")

arrow_keep_backward_compatibility(ArrowS3 arrow_s3)

check_required_components(ArrowS3)

arrow_show_details(ArrowS3 ARROW_S3)
65 changes: 47 additions & 18 deletions cpp/src/arrow/CMakeLists.txt

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this remaining ARROW_S3 section get an update now? I've tried it locally on top of PR changes, seems OK;

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
@@ -111,22 +111,6 @@ if(ARROW_USE_GLOG)
   endif()
 endif()
 
-if(ARROW_S3)
-  if(AWSSDK_SOURCE STREQUAL "SYSTEM")
-    list(APPEND
-         ARROW_STATIC_INSTALL_INTERFACE_LIBS
-         aws-cpp-sdk-identity-management
-         aws-cpp-sdk-sts
-         aws-cpp-sdk-cognito-identity
-         aws-cpp-sdk-s3
-         aws-cpp-sdk-core)
-  elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
-    if(UNIX)
-      list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
-    endif()
-  endif()
-endif()
 if(ARROW_WITH_OPENTELEMETRY)

@@ -1070,6 +1054,10 @@ if(ARROW_FILESYSTEM)
     if(AWSSDK_SOURCE STREQUAL "SYSTEM")
       list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS ${AWSSDK_LINK_LIBRARIES})
+    elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
+      if(UNIX)
+        list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
+      endif()
     endif()
     add_arrow_lib(arrow_s3

Original file line number Diff line number Diff line change
Expand Up @@ -1018,11 +1018,6 @@ if(ARROW_FILESYSTEM)
if(ARROW_HDFS)
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc)
endif()
if(ARROW_S3)
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/s3fs.cc)
set_source_files_properties(filesystem/s3fs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION
ON)
endif()

arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
if(ARROW_AZURE)
Expand All @@ -1042,21 +1037,55 @@ if(ARROW_FILESYSTEM)
endforeach()
endif()
if(ARROW_S3)
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE ${AWSSDK_LINK_LIBRARIES})
# If libarrow_s3.a is only built, "pkg-config --cflags --libs
# arrow-s3" outputs build flags for static linking not shared
# linking. ARROW_S3_PC_* except ARROW_S3_PC_*_PRIVATE are for the
# static linking case.
if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
string(APPEND ARROW_S3_PC_CFLAGS "${ARROW_S3_PC_CFLAGS_PRIVATE}")
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
Comment thread
raulcd marked this conversation as resolved.
set(ARROW_S3_PC_LIBS "${ARROW_S3_PC_LIBS_PRIVATE}")
set(ARROW_S3_PC_LIBS_PRIVATE "")
set(ARROW_S3_PC_REQUIRES "${ARROW_S3_PC_REQUIRES_PRIVATE}")
set(ARROW_S3_PC_REQUIRES_PRIVATE "")
else()
set(ARROW_S3_PC_LIBS "")
set(ARROW_S3_PC_REQUIRES "")
endif()
list(APPEND ARROW_S3_LIB_SRCS filesystem/s3fs_module.cc filesystem/s3fs.cc)
set(ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
if(AWSSDK_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS ${AWSSDK_LINK_LIBRARIES})
endif()
add_arrow_lib(arrow_s3
CMAKE_PACKAGE_NAME
ArrowS3
PKG_CONFIG_NAME
arrow-s3
SOURCES
${ARROW_S3_LIB_SRCS}
SHARED_LINK_LIBS
arrow_shared
SHARED_PRIVATE_LINK_LIBS
${AWSSDK_LINK_LIBRARIES}
SHARED_INSTALL_INTERFACE_LIBS
Arrow::arrow_shared
STATIC_LINK_LIBS
arrow_static
${AWSSDK_LINK_LIBRARIES}
STATIC_INSTALL_INTERFACE_LIBS
${ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS}
OUTPUTS
ARROW_S3_LIBRARIES)
foreach(LIB_TARGET ${ARROW_S3_LIBRARIES})
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_S3_EXPORTING)
endforeach()

if(ARROW_S3_MODULE)
if(NOT ARROW_BUILD_SHARED)
message(FATAL_ERROR "ARROW_S3_MODULE without shared libarrow (-DARROW_BUILD_SHARED=ON) is not supported"
)
endif()

add_library(arrow_s3fs MODULE filesystem/s3fs_module.cc filesystem/s3fs.cc)
target_link_libraries(arrow_s3fs PRIVATE ${AWSSDK_LINK_LIBRARIES} arrow_shared)
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
if(ARROW_BUILD_STATIC AND WIN32)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The AND WIN32 isn't useful, right?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use the same pattern on other places:

  if(ARROW_BUILD_STATIC AND WIN32)
    target_compile_definitions(arrow_compute_static PUBLIC ARROW_COMPUTE_STATIC)
  endif()

or

if(ARROW_BUILD_STATIC AND WIN32)
  target_compile_definitions(arrow_static PUBLIC ARROW_STATIC)
endif()

Taking a look at the definition on visibility.h of ARROW_S3_STATIC is already guarded for WIN32:

#if defined(_WIN32) || defined(__CYGWIN__)

So it will only be used on WIN32, it does not seem necessary on others so I would say the AND WIN32 does nothing but it's hygiene?

target_compile_definitions(arrow_s3_static PUBLIC ARROW_S3_STATIC)
endif()

set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
endif()

list(APPEND ARROW_TESTING_SHARED_LINK_LIBS ${ARROW_GTEST_GMOCK})
Expand Down
30 changes: 30 additions & 0 deletions cpp/src/arrow/arrow-s3.pc.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

prefix=@CMAKE_INSTALL_PREFIX@
includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
libdir=@ARROW_PKG_CONFIG_LIBDIR@

Name: Apache Arrow S3
Description: Apache Arrow's S3 filesystem implementation.
Version: @ARROW_VERSION@
Requires: arrow@ARROW_S3_PC_REQUIRES@
Requires.private:@ARROW_S3_PC_REQUIRES_PRIVATE@
Libs: -L${libdir} -larrow_s3@ARROW_S3_PC_LIBS@
Libs.private:@ARROW_S3_PC_LIBS_PRIVATE@
Cflags:@ARROW_S3_PC_CFLAGS@
Cflags.private:@ARROW_S3_PC_CFLAGS_PRIVATE@
Comment thread
raulcd marked this conversation as resolved.
15 changes: 6 additions & 9 deletions cpp/src/arrow/filesystem/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ if(ARROW_S3)
# static variables storage of AWS SDK for C++ in libaws*.a may be
# mixed with one in libarrow.
if(ARROW_TEST_LINKAGE STREQUAL "shared")
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_shared)
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_shared)
else()
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_static)
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_static)
endif()
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS ${AWSSDK_LINK_LIBRARIES})
add_arrow_test(s3fs_test
Expand Down Expand Up @@ -107,7 +107,7 @@ if(ARROW_S3)
if(ARROW_BUILD_TESTS)
add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS}
${GFLAGS_LIBRARIES})
${ARROW_S3_TEST_EXTRA_LINK_LIBS} ${GFLAGS_LIBRARIES})
add_dependencies(arrow-tests arrow-s3fs-narrative-test)
endif()

Expand All @@ -119,6 +119,7 @@ if(ARROW_S3)
s3fs_benchmark.cc
s3_test_util.cc
STATIC_LINK_LIBS
${ARROW_S3_TEST_EXTRA_LINK_LIBS}
${AWSSDK_LINK_LIBRARIES}
${ARROW_BENCHMARK_LINK_LIBS})
if(ARROW_TEST_LINKAGE STREQUAL "static")
Expand All @@ -128,21 +129,17 @@ if(ARROW_S3)
endif()
endif()

if(ARROW_S3_MODULE AND ARROW_BUILD_TESTS)
if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED)
add_arrow_test(s3fs_module_test
SOURCES
s3fs_module_test.cc
s3_test_util.cc
EXTRA_LABELS
filesystem
DEFINITIONS
ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>"
ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3_shared>"
EXTRA_LINK_LIBS
Boost::filesystem)
target_compile_definitions(arrow-filesystem-test
PUBLIC ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>")
target_sources(arrow-filesystem-test PUBLIC s3fs_module_test.cc s3_test_util.cc)
target_link_libraries(arrow-filesystem-test PUBLIC Boost::filesystem)
endif()
endif()

Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/filesystem/filesystem.h
Original file line number Diff line number Diff line change
Expand Up @@ -560,6 +560,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {
/// will fail with an error.
///
/// The user is responsible for synchronization of calls to this function.
ARROW_EXPORT
void EnsureFinalized();

/// \defgroup filesystem-factories Functions for creating FileSystem instances
Expand Down
49 changes: 49 additions & 0 deletions cpp/src/arrow/filesystem/s3_visibility.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#if defined(_WIN32) || defined(__CYGWIN__)
# if defined(_MSC_VER)
# pragma warning(push)
# pragma warning(disable : 4251)
# else
# pragma GCC diagnostic ignored "-Wattributes"
# endif

# ifdef ARROW_S3_STATIC
# define ARROW_S3_EXPORT
# elif defined(ARROW_S3_EXPORTING)
# define ARROW_S3_EXPORT __declspec(dllexport)
# else
# define ARROW_S3_EXPORT __declspec(dllimport)
# endif

# define ARROW_S3_NO_EXPORT

# if defined(_MSC_VER)
# pragma warning(pop)
# endif

#else // Not Windows
# ifndef ARROW_S3_EXPORT
# define ARROW_S3_EXPORT __attribute__((visibility("default")))
# endif
# ifndef ARROW_S3_NO_EXPORT
# define ARROW_S3_NO_EXPORT __attribute__((visibility("hidden")))
# endif
#endif
Loading
Loading