You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/04/08 20:35:17 UTC
[arrow] branch master updated: ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 5fa08449a0 ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS
5fa08449a0 is described below
commit 5fa08449a09a452f7a93cd20d0ca1d76a6637449
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Sat Apr 9 05:35:07 2022 +0900
ARROW-16102: [C++] Add support for building with system gRPC and bundled GCS
`find_package(gRPC)` may find system `absl::*` targets.
If we found system `absl::*` targets, we use them instead of building `absl::*`.
Bundle google-cloud-cpp is updated to 1.39.0 for
https://github.com/googleapis/google-cloud-cpp/issues/8536 .
Closes #12795 from kou/system-grpc-and-bundled-gcs
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
.github/workflows/cpp.yml | 9 +++++++++
.github/workflows/ruby.yml | 3 +++
ci/scripts/install_gcs_testbench.sh | 5 +++--
cpp/cmake_modules/ThirdpartyToolchain.cmake | 27 ++++++++++++++++++---------
cpp/thirdparty/versions.txt | 6 +++---
5 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 7401fc489c..00f9e335f5 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -126,6 +126,7 @@ jobs:
ARROW_DATASET: ON
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
+ ARROW_GCS: ON
ARROW_HDFS: ON
ARROW_HOME: /usr/local
ARROW_JEMALLOC: ON
@@ -141,6 +142,8 @@ jobs:
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_ZSTD: ON
+ # System Abseil installed by Homebrew uses C++ 17
+ CMAKE_CXX_STANDARD: 17
steps:
- name: Checkout Arrow
uses: actions/checkout@v2
@@ -153,6 +156,9 @@ jobs:
rm -f /usr/local/bin/2to3
brew update --preinstall
brew bundle --file=cpp/Brewfile
+ - name: Install Google Cloud Storage Testbench
+ shell: bash
+ run: ci/scripts/install_gcs_testbench.sh default
- name: Setup ccache
run: |
ci/scripts/ccache_setup.sh
@@ -268,6 +274,9 @@ jobs:
ARROW_DATASET: ON
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
+ # google-could-cpp uses _dupenv_s() but it can't be used with msvcrt.
+ # We need to use ucrt to use _dupenv_s().
+ # ARROW_GCS: ON
ARROW_HDFS: OFF
ARROW_HOME: /mingw${{ matrix.mingw-n-bits }}
ARROW_JEMALLOC: OFF
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index 0770f2f32a..54292eafc8 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -90,6 +90,7 @@ jobs:
ulimit -c unlimited
archery docker run \
-e ARROW_FLIGHT=ON \
+ -e ARROW_GCS=ON \
-e Protobuf_SOURCE=BUNDLED \
-e gRPC_SOURCE=BUNDLED \
ubuntu-ruby
@@ -110,6 +111,7 @@ jobs:
ARROW_BUILD_TESTS: OFF
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
+ ARROW_GCS: ON
ARROW_GLIB_GTK_DOC: true
ARROW_GLIB_WERROR: true
ARROW_HOME: /usr/local
@@ -188,6 +190,7 @@ jobs:
ARROW_BUILD_TYPE: release
ARROW_FLIGHT: ON
ARROW_GANDIVA: ON
+ ARROW_GCS: ON
ARROW_HDFS: OFF
ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}
ARROW_JEMALLOC: OFF
diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh
index 16e3c4042c..0282e0fda5 100755
--- a/ci/scripts/install_gcs_testbench.sh
+++ b/ci/scripts/install_gcs_testbench.sh
@@ -31,7 +31,8 @@ fi
version=$1
if [[ "${version}" -eq "default" ]]; then
- version="v0.7.0"
+ version="v0.16.0"
fi
-pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
+${PYTHON:-python3} -m pip install \
+ "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index e315346378..3a0353bc7d 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -2593,8 +2593,9 @@ endmacro()
# ----------------------------------------------------------------------
# Dependencies for Arrow Flight RPC
-macro(build_absl_once)
- if(NOT TARGET absl_ep)
+macro(resolve_dependency_absl)
+ # Choose one of built absl::* targets
+ if(NOT TARGET absl::algorithm)
message(STATUS "Building Abseil-cpp from source")
set(ABSL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/absl_ep-install")
set(ABSL_INCLUDE_DIR "${ABSL_PREFIX}/include")
@@ -3451,6 +3452,7 @@ macro(build_absl_once)
# Work around https://gitlab.kitware.com/cmake/cmake/issues/15052
file(MAKE_DIRECTORY ${ABSL_INCLUDE_DIR})
+ set(ABSL_VENDORED TRUE)
endif()
endmacro()
@@ -3464,8 +3466,8 @@ macro(build_grpc)
get_target_property(c-ares_INCLUDE_DIR c-ares::cares INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${c-ares_INCLUDE_DIR})
- # First need to build Abseil
- build_absl_once()
+ # First need Abseil
+ resolve_dependency_absl()
message(STATUS "Building gRPC from source")
@@ -3498,7 +3500,9 @@ macro(build_grpc)
add_custom_target(grpc_dependencies)
- add_dependencies(grpc_dependencies absl_ep)
+ if(ABSL_VENDORED)
+ add_dependencies(grpc_dependencies absl_ep)
+ endif()
if(CARES_VENDORED)
add_dependencies(grpc_dependencies cares_ep)
endif()
@@ -3819,7 +3823,7 @@ macro(build_google_cloud_cpp_storage)
message(STATUS "Only building the google-cloud-cpp::storage component")
# List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
- build_absl_once()
+ resolve_dependency_absl()
build_crc32c_once()
# Curl is required on all platforms, but building it internally might also trip over S3's copy.
@@ -3830,7 +3834,9 @@ macro(build_google_cloud_cpp_storage)
# Build google-cloud-cpp, with only storage_client
# Inject vendored packages via CMAKE_PREFIX_PATH
- list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
+ if(ABSL_VENDORED)
+ list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${ABSL_PREFIX})
+ endif()
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${CRC32C_PREFIX})
list(APPEND GOOGLE_CLOUD_CPP_PREFIX_PATH_LIST ${NLOHMANN_JSON_PREFIX})
@@ -3862,7 +3868,9 @@ macro(build_google_cloud_cpp_storage)
add_custom_target(google_cloud_cpp_dependencies)
- add_dependencies(google_cloud_cpp_dependencies absl_ep)
+ if(ABSL_VENDORED)
+ add_dependencies(google_cloud_cpp_dependencies absl_ep)
+ endif()
add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
add_dependencies(google_cloud_cpp_dependencies nlohmann_json::nlohmann_json)
@@ -3903,7 +3911,8 @@ macro(build_google_cloud_cpp_storage)
absl::memory
absl::optional
absl::time
- Threads::Threads)
+ Threads::Threads
+ OpenSSL::Crypto)
add_library(google-cloud-cpp::storage STATIC IMPORTED)
set_target_properties(google-cloud-cpp::storage
diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
index 2d54da8350..4e90f4c968 100644
--- a/cpp/thirdparty/versions.txt
+++ b/cpp/thirdparty/versions.txt
@@ -49,8 +49,8 @@ ARROW_GFLAGS_BUILD_VERSION=v2.2.2
ARROW_GFLAGS_BUILD_SHA256_CHECKSUM=34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
ARROW_GLOG_BUILD_VERSION=v0.5.0
ARROW_GLOG_BUILD_SHA256_CHECKSUM=eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5
-ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.35.0
-ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=e4e9eac1e7999eff195db270bc2a719004660b3730ebb5d2f444f2d2057e49b2
+ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.39.0
+ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=73e4e840018b24bec2beb49e036a3c2d8c471d4dc4a18b9026ccc4d8ab8e78cc
ARROW_GRPC_BUILD_VERSION=v1.35.0
ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958
ARROW_GTEST_BUILD_VERSION=1.11.0
@@ -119,7 +119,7 @@ DEPENDENCIES=(
"ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
"ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"
"ARROW_GLOG_URL glog-${ARROW_GLOG_BUILD_VERSION}.tar.gz https://github.com/google/glog/archive/${ARROW_GLOG_BUILD_VERSION}.tar.gz"
- "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
+ "ARROW_GOOGLE_CLOUD_CPP_URL google-cloud-cpp-${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz https://github.com/googleapis/google-cloud-cpp/archive/${ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION}.tar.gz"
"ARROW_GRPC_URL grpc-${ARROW_GRPC_BUILD_VERSION}.tar.gz https://github.com/grpc/grpc/archive/${ARROW_GRPC_BUILD_VERSION}.tar.gz"
"ARROW_GTEST_URL gtest-${ARROW_GTEST_BUILD_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${ARROW_GTEST_BUILD_VERSION}.tar.gz"
"ARROW_JEMALLOC_URL jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/${ARROW_JEMALLOC_BUILD_VERSION}/jemalloc-${ARROW_JEMALLOC_BUILD_VERSION}.tar.bz2"