-
Notifications
You must be signed in to change notification settings - Fork 4.3k
GH-50194: [C++] Move S3 and AWS-SDK to its own libarrow_s3.so #50195
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5df6741
b8fe8aa
3a6e8d6
64f4fa7
566c9ce
fe4b868
7beac75
de9f092
55a438d
142e154
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| # | ||
| # This config sets the following variables in your project:: | ||
| # | ||
| # ArrowS3_FOUND - true if Arrow S3 found on the system | ||
| # | ||
| # This config sets the following targets in your project:: | ||
| # | ||
| # ArrowS3::arrow_s3_shared - for linked as shared library if shared library is built | ||
| # ArrowS3::arrow_s3_static - for linked as static library if static library is built | ||
|
|
||
| @PACKAGE_INIT@ | ||
|
|
||
| set(ARROW_S3_SYSTEM_DEPENDENCIES "@ARROW_S3_SYSTEM_DEPENDENCIES@") | ||
|
|
||
| include(CMakeFindDependencyMacro) | ||
| find_dependency(Arrow CONFIG) | ||
|
|
||
| if(ARROW_BUILD_STATIC) | ||
| arrow_find_dependencies("${ARROW_S3_SYSTEM_DEPENDENCIES}") | ||
| endif() | ||
|
|
||
| include("${CMAKE_CURRENT_LIST_DIR}/ArrowS3Targets.cmake") | ||
|
|
||
| arrow_keep_backward_compatibility(ArrowS3 arrow_s3) | ||
|
|
||
| check_required_components(ArrowS3) | ||
|
|
||
| arrow_show_details(ArrowS3 ARROW_S3) |
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this remaining ARROW_S3 section get an update now? I've tried it locally on top of PR changes, seems OK; diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
@@ -111,22 +111,6 @@ if(ARROW_USE_GLOG)
endif()
endif()
-if(ARROW_S3)
- if(AWSSDK_SOURCE STREQUAL "SYSTEM")
- list(APPEND
- ARROW_STATIC_INSTALL_INTERFACE_LIBS
- aws-cpp-sdk-identity-management
- aws-cpp-sdk-sts
- aws-cpp-sdk-cognito-identity
- aws-cpp-sdk-s3
- aws-cpp-sdk-core)
- elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
- if(UNIX)
- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
- endif()
- endif()
-endif()
if(ARROW_WITH_OPENTELEMETRY)
@@ -1070,6 +1054,10 @@ if(ARROW_FILESYSTEM)
if(AWSSDK_SOURCE STREQUAL "SYSTEM")
list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS ${AWSSDK_LINK_LIBRARIES})
+ elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
+ if(UNIX)
+ list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
+ endif()
endif()
add_arrow_lib(arrow_s3 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1018,11 +1018,6 @@ if(ARROW_FILESYSTEM) | |
| if(ARROW_HDFS) | ||
| list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc) | ||
| endif() | ||
| if(ARROW_S3) | ||
| list(APPEND ARROW_FILESYSTEM_SRCS filesystem/s3fs.cc) | ||
| set_source_files_properties(filesystem/s3fs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION | ||
| ON) | ||
| endif() | ||
|
|
||
| arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS}) | ||
| if(ARROW_AZURE) | ||
|
|
@@ -1042,21 +1037,55 @@ if(ARROW_FILESYSTEM) | |
| endforeach() | ||
| endif() | ||
| if(ARROW_S3) | ||
| foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS}) | ||
| target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE ${AWSSDK_LINK_LIBRARIES}) | ||
| # If libarrow_s3.a is only built, "pkg-config --cflags --libs | ||
| # arrow-s3" outputs build flags for static linking not shared | ||
| # linking. ARROW_S3_PC_* except ARROW_S3_PC_*_PRIVATE are for the | ||
| # static linking case. | ||
| if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC) | ||
| string(APPEND ARROW_S3_PC_CFLAGS "${ARROW_S3_PC_CFLAGS_PRIVATE}") | ||
| set(ARROW_S3_PC_CFLAGS_PRIVATE "") | ||
|
raulcd marked this conversation as resolved.
|
||
| set(ARROW_S3_PC_LIBS "${ARROW_S3_PC_LIBS_PRIVATE}") | ||
| set(ARROW_S3_PC_LIBS_PRIVATE "") | ||
| set(ARROW_S3_PC_REQUIRES "${ARROW_S3_PC_REQUIRES_PRIVATE}") | ||
| set(ARROW_S3_PC_REQUIRES_PRIVATE "") | ||
| else() | ||
| set(ARROW_S3_PC_LIBS "") | ||
| set(ARROW_S3_PC_REQUIRES "") | ||
| endif() | ||
| list(APPEND ARROW_S3_LIB_SRCS filesystem/s3fs_module.cc filesystem/s3fs.cc) | ||
| set(ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) | ||
| if(AWSSDK_SOURCE STREQUAL "SYSTEM") | ||
| list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS ${AWSSDK_LINK_LIBRARIES}) | ||
| endif() | ||
| add_arrow_lib(arrow_s3 | ||
| CMAKE_PACKAGE_NAME | ||
| ArrowS3 | ||
| PKG_CONFIG_NAME | ||
| arrow-s3 | ||
| SOURCES | ||
| ${ARROW_S3_LIB_SRCS} | ||
| SHARED_LINK_LIBS | ||
| arrow_shared | ||
| SHARED_PRIVATE_LINK_LIBS | ||
| ${AWSSDK_LINK_LIBRARIES} | ||
| SHARED_INSTALL_INTERFACE_LIBS | ||
| Arrow::arrow_shared | ||
| STATIC_LINK_LIBS | ||
| arrow_static | ||
| ${AWSSDK_LINK_LIBRARIES} | ||
| STATIC_INSTALL_INTERFACE_LIBS | ||
| ${ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS} | ||
| OUTPUTS | ||
| ARROW_S3_LIBRARIES) | ||
| foreach(LIB_TARGET ${ARROW_S3_LIBRARIES}) | ||
| target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_S3_EXPORTING) | ||
| endforeach() | ||
|
|
||
| if(ARROW_S3_MODULE) | ||
| if(NOT ARROW_BUILD_SHARED) | ||
| message(FATAL_ERROR "ARROW_S3_MODULE without shared libarrow (-DARROW_BUILD_SHARED=ON) is not supported" | ||
| ) | ||
| endif() | ||
|
|
||
| add_library(arrow_s3fs MODULE filesystem/s3fs_module.cc filesystem/s3fs.cc) | ||
| target_link_libraries(arrow_s3fs PRIVATE ${AWSSDK_LINK_LIBRARIES} arrow_shared) | ||
| set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc | ||
| PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) | ||
| if(ARROW_BUILD_STATIC AND WIN32) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We use the same pattern on other places: or Taking a look at the definition on visibility.h of So it will only be used on WIN32, it does not seem necessary on others so I would say the |
||
| target_compile_definitions(arrow_s3_static PUBLIC ARROW_S3_STATIC) | ||
| endif() | ||
|
|
||
| set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc | ||
| PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) | ||
| endif() | ||
|
|
||
| list(APPEND ARROW_TESTING_SHARED_LINK_LIBS ${ARROW_GTEST_GMOCK}) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| prefix=@CMAKE_INSTALL_PREFIX@ | ||
| includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@ | ||
| libdir=@ARROW_PKG_CONFIG_LIBDIR@ | ||
|
|
||
| Name: Apache Arrow S3 | ||
| Description: Apache Arrow's S3 filesystem implementation. | ||
| Version: @ARROW_VERSION@ | ||
| Requires: arrow@ARROW_S3_PC_REQUIRES@ | ||
| Requires.private:@ARROW_S3_PC_REQUIRES_PRIVATE@ | ||
| Libs: -L${libdir} -larrow_s3@ARROW_S3_PC_LIBS@ | ||
| Libs.private:@ARROW_S3_PC_LIBS_PRIVATE@ | ||
| Cflags:@ARROW_S3_PC_CFLAGS@ | ||
| Cflags.private:@ARROW_S3_PC_CFLAGS_PRIVATE@ | ||
|
raulcd marked this conversation as resolved.
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| // Licensed to the Apache Software Foundation (ASF) under one | ||
| // or more contributor license agreements. See the NOTICE file | ||
| // distributed with this work for additional information | ||
| // regarding copyright ownership. The ASF licenses this file | ||
| // to you under the Apache License, Version 2.0 (the | ||
| // "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #if defined(_WIN32) || defined(__CYGWIN__) | ||
| # if defined(_MSC_VER) | ||
| # pragma warning(push) | ||
| # pragma warning(disable : 4251) | ||
| # else | ||
| # pragma GCC diagnostic ignored "-Wattributes" | ||
| # endif | ||
|
|
||
| # ifdef ARROW_S3_STATIC | ||
| # define ARROW_S3_EXPORT | ||
| # elif defined(ARROW_S3_EXPORTING) | ||
| # define ARROW_S3_EXPORT __declspec(dllexport) | ||
| # else | ||
| # define ARROW_S3_EXPORT __declspec(dllimport) | ||
| # endif | ||
|
|
||
| # define ARROW_S3_NO_EXPORT | ||
|
|
||
| # if defined(_MSC_VER) | ||
| # pragma warning(pop) | ||
| # endif | ||
|
|
||
| #else // Not Windows | ||
| # ifndef ARROW_S3_EXPORT | ||
| # define ARROW_S3_EXPORT __attribute__((visibility("default"))) | ||
| # endif | ||
| # ifndef ARROW_S3_NO_EXPORT | ||
| # define ARROW_S3_NO_EXPORT __attribute__((visibility("hidden"))) | ||
| # endif | ||
| #endif |
Uh oh!
There was an error while loading. Please reload this page.