You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/08/30 18:58:20 UTC
[arrow] branch master updated: ARROW-3142: [C++] Fetch all libs
from toolchain environment
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 84fcb82 ARROW-3142: [C++] Fetch all libs from toolchain environment
84fcb82 is described below
commit 84fcb8239796d54378125e96be08439512fa5984
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Thu Aug 30 14:58:13 2018 -0400
ARROW-3142: [C++] Fetch all libs from toolchain environment
When ARROW_BUILD_TOOLCHAIN is set, orc and gtest would still be downloaded from source.
Author: Antoine Pitrou <an...@python.org>
Closes #2495 from pitrou/ARROW-3142-orc-gtest-toolchain and squashes the following commits:
86d02a27 <Antoine Pitrou> ARROW-3142: Fetch most libs from toolchain environment
---
ci/cpp-python-msvc-build.bat | 13 ++-
ci/travis_install_toolchain.sh | 25 +++--
ci/travis_script_python.sh | 2 +-
cpp/cmake_modules/FindGTest.cmake | 4 +-
cpp/cmake_modules/ThirdpartyToolchain.cmake | 141 ++++++++++++++++------------
5 files changed, 107 insertions(+), 78 deletions(-)
diff --git a/ci/cpp-python-msvc-build.bat b/ci/cpp-python-msvc-build.bat
index ecc68e0..a0f5fbe 100644
--- a/ci/cpp-python-msvc-build.bat
+++ b/ci/cpp-python-msvc-build.bat
@@ -77,11 +77,18 @@ call activate arrow
if "%JOB%" == "Toolchain" (
@rem Install pre-built "toolchain" packages for faster builds
conda install -q -y -c conda-forge ^
- flatbuffers rapidjson ^
+ boost-cpp ^
+ brotli ^
cmake ^
+ flatbuffers ^
+ gflags ^
+ gtest ^
git ^
- boost-cpp ^
- snappy zlib brotli gflags lz4-c zstd
+ lz4-c ^
+ rapidjson ^
+ snappy ^
+ zlib ^
+ zstd
set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
)
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
index a153d89..924f752 100755
--- a/ci/travis_install_toolchain.sh
+++ b/ci/travis_install_toolchain.sh
@@ -23,26 +23,25 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
if [ ! -e $CPP_TOOLCHAIN ]; then
# Set up C++ toolchain from conda-forge packages for faster builds
- conda create -y -q -p $CPP_TOOLCHAIN python=3.6 \
- nomkl \
+ conda create -y -q -p $CPP_TOOLCHAIN python=3.6 nomkl \
boost-cpp \
- libprotobuf \
- rapidjson \
+ brotli \
+ ccache \
+ cmake \
+ curl \
flatbuffers \
+ lz4-c \
gflags \
gtest \
- lz4-c \
+ libprotobuf \
+ ninja \
+ rapidjson \
snappy \
- ccache \
- zstd \
- brotli \
- zlib \
- cmake \
- curl \
thrift-cpp=0.11.0 \
- ninja
+ zlib \
+ zstd
# HACK(wesm): We started experiencing OpenSSL failures when Miniconda was
# updated sometime on October 2 or October 3
- conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
+# conda update -y -q -p $CPP_TOOLCHAIN ca-certificates -c defaults
fi
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index 89dbfed..46dad80 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -51,7 +51,7 @@ conda install -y -q pip \
cython
if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ] && [ "$PYTHON_VERSION" == "3.6" ]; then
- # Build documentation depedencies
+ # Install documentation dependencies
conda install -y -q \
ipython \
numpydoc \
diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake
index 0e2af2f..8a31ae6 100644
--- a/cpp/cmake_modules/FindGTest.cmake
+++ b/cpp/cmake_modules/FindGTest.cmake
@@ -31,8 +31,8 @@
# GTEST_SHARED_LIB, path to libgtest's shared library
# GTEST_FOUND, whether gtest has been found
-if( NOT "$ENV{GTEST_HOME}" STREQUAL "")
- file( TO_CMAKE_PATH "$ENV{GTEST_HOME}" _native_path )
+if( NOT "${GTEST_HOME}" STREQUAL "")
+ file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path )
list( APPEND _gtest_roots ${_native_path} )
elseif ( GTest_HOME )
list( APPEND _gtest_roots ${GTest_HOME} )
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 124a8d3..a7c4b20 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -22,16 +22,24 @@
set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
+ set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
set(FLATBUFFERS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
- set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
- set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
set(GFLAGS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ set(GRPC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ # Using gtest from the toolchain breaks AppVeyor builds
+ if (NOT MSVC)
+ set(GTEST_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ endif()
+ set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ # orc disabled as it's not in conda-forge (but in Anaconda with an incompatible ABI)
+# set(ORC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ set(PROTOBUF_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
set(SNAPPY_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
+ set(THRIFT_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
- set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
- set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
set(ZSTD_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
- set(THRIFT_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}")
if (NOT DEFINED ENV{BOOST_ROOT})
# Since we have to set this in the environment, we check whether
@@ -40,50 +48,65 @@ if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "")
endif()
endif()
-if (DEFINED ENV{FLATBUFFERS_HOME})
- set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}")
+# Home path for each third-party lib can be overriden with env vars
+
+if (DEFINED ENV{BROTLI_HOME})
+ set(BROTLI_HOME "$ENV{BROTLI_HOME}")
endif()
-if (DEFINED ENV{RAPIDJSON_HOME})
- set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
+if (DEFINED ENV{FLATBUFFERS_HOME})
+ set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}")
endif()
if (DEFINED ENV{GFLAGS_HOME})
set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
endif()
-if (DEFINED ENV{SNAPPY_HOME})
- set(SNAPPY_HOME "$ENV{SNAPPY_HOME}")
+if (DEFINED ENV{GRPC_HOME})
+ set(GRPC_HOME "$ENV{GRPC_HOME}")
endif()
-if (DEFINED ENV{ZLIB_HOME})
- set(ZLIB_HOME "$ENV{ZLIB_HOME}")
+if (DEFINED ENV{GTEST_HOME})
+ set(GTEST_HOME "$ENV{GTEST_HOME}")
endif()
-if (DEFINED ENV{BROTLI_HOME})
- set(BROTLI_HOME "$ENV{BROTLI_HOME}")
+if (DEFINED ENV{JEMALLOC_HOME})
+ set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}")
endif()
if (DEFINED ENV{LZ4_HOME})
set(LZ4_HOME "$ENV{LZ4_HOME}")
endif()
-if (DEFINED ENV{ZSTD_HOME})
- set(ZSTD_HOME "$ENV{ZSTD_HOME}")
-endif()
-
-if (DEFINED ENV{GRPC_HOME})
- set(GRPC_HOME "$ENV{GRPC_HOME}")
+if (DEFINED ENV{ORC_HOME})
+ set(ORC_HOME "$ENV{ORC_HOME}")
endif()
if (DEFINED ENV{PROTOBUF_HOME})
set(PROTOBUF_HOME "$ENV{PROTOBUF_HOME}")
endif()
+if (DEFINED ENV{RAPIDJSON_HOME})
+ set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}")
+endif()
+
+if (DEFINED ENV{SNAPPY_HOME})
+ set(SNAPPY_HOME "$ENV{SNAPPY_HOME}")
+endif()
+
if (DEFINED ENV{THRIFT_HOME})
set(THRIFT_HOME "$ENV{THRIFT_HOME}")
endif()
+if (DEFINED ENV{ZLIB_HOME})
+ set(ZLIB_HOME "$ENV{ZLIB_HOME}")
+endif()
+
+if (DEFINED ENV{ZSTD_HOME})
+ set(ZSTD_HOME "$ENV{ZSTD_HOME}")
+endif()
+
+
# ----------------------------------------------------------------------
# Versions and URLs for toolchain builds, which also can be used to configure
# offline builds
@@ -118,16 +141,16 @@ else()
"https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_GTEST_URL})
- set(GTEST_SOURCE_URL "$ENV{ARROW_GTEST_URL}")
+if (DEFINED ENV{ARROW_BROTLI_URL})
+ set(BROTLI_SOURCE_URL "$ENV{ARROW_BROTLI_URL}")
else()
- set(GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
+ set(BROTLI_SOURCE_URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_GFLAGS_URL})
- set(GFLAGS_SOURCE_URL "$ENV{ARROW_GFLAGS_URL}")
+if (DEFINED ENV{ARROW_FLATBUFFERS_URL})
+ set(FLATBUFFERS_SOURCE_URL "$ENV{ARROW_FLATBUFFERS_URL}")
else()
- set(GFLAGS_SOURCE_URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz")
+ set(FLATBUFFERS_SOURCE_URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz")
endif()
if (DEFINED ENV{ARROW_GBENCHMARK_URL})
@@ -136,29 +159,22 @@ else()
set(GBENCHMARK_SOURCE_URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz")
endif()
-set(RAPIDJSON_SOURCE_MD5 "badd12c511e081fec6c89c43a7027bce")
-if (DEFINED ENV{ARROW_RAPIDJSON_URL})
- set(RAPIDJSON_SOURCE_URL "$ENV{ARROW_RAPIDJSON_URL}")
-else()
- set(RAPIDJSON_SOURCE_URL "https://github.com/miloyip/rapidjson/archive/v${RAPIDJSON_VERSION}.tar.gz")
-endif()
-
-if (DEFINED ENV{ARROW_FLATBUFFERS_URL})
- set(FLATBUFFERS_SOURCE_URL "$ENV{ARROW_FLATBUFFERS_URL}")
+if (DEFINED ENV{ARROW_GFLAGS_URL})
+ set(GFLAGS_SOURCE_URL "$ENV{ARROW_GFLAGS_URL}")
else()
- set(FLATBUFFERS_SOURCE_URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz")
+ set(GFLAGS_SOURCE_URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_SNAPPY_URL})
- set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}")
+if (DEFINED ENV{ARROW_GRPC_URL})
+ set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
else()
- set(SNAPPY_SOURCE_URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz")
+ set(GRPC_SOURCE_URL "https://github.com/grpc/grpc/archive/v${GRPC_VERSION}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_BROTLI_URL})
- set(BROTLI_SOURCE_URL "$ENV{ARROW_BROTLI_URL}")
+if (DEFINED ENV{ARROW_GTEST_URL})
+ set(GTEST_SOURCE_URL "$ENV{ARROW_GTEST_URL}")
else()
- set(BROTLI_SOURCE_URL "https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz")
+ set(GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
endif()
if (DEFINED ENV{ARROW_LZ4_URL})
@@ -167,16 +183,10 @@ else()
set(LZ4_SOURCE_URL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_ZLIB_URL})
- set(ZLIB_SOURCE_URL "$ENV{ARROW_ZLIB_URL}")
-else()
- set(ZLIB_SOURCE_URL "http://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz")
-endif()
-
-if (DEFINED ENV{ARROW_ZSTD_URL})
- set(ZSTD_SOURCE_URL "$ENV{ARROW_ZSTD_URL}")
+if (DEFINED ENV{ARROW_ORC_URL})
+ set(ORC_SOURCE_URL "$ENV{ARROW_ORC_URL}")
else()
- set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz")
+ set(ORC_SOURCE_URL "https://github.com/apache/orc/archive/rel/release-${ORC_VERSION}.tar.gz")
endif()
if (DEFINED ENV{ARROW_PROTOBUF_URL})
@@ -185,16 +195,17 @@ else()
set(PROTOBUF_SOURCE_URL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_GRPC_URL})
- set(GRPC_SOURCE_URL "$ENV{ARROW_GRPC_URL}")
+set(RAPIDJSON_SOURCE_MD5 "badd12c511e081fec6c89c43a7027bce")
+if (DEFINED ENV{ARROW_RAPIDJSON_URL})
+ set(RAPIDJSON_SOURCE_URL "$ENV{ARROW_RAPIDJSON_URL}")
else()
- set(GRPC_SOURCE_URL "https://github.com/grpc/grpc/archive/v${GRPC_VERSION}.tar.gz")
+ set(RAPIDJSON_SOURCE_URL "https://github.com/miloyip/rapidjson/archive/v${RAPIDJSON_VERSION}.tar.gz")
endif()
-if (DEFINED ENV{ARROW_ORC_URL})
- set(ORC_SOURCE_URL "$ENV{ARROW_ORC_URL}")
+if (DEFINED ENV{ARROW_SNAPPY_URL})
+ set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}")
else()
- set(ORC_SOURCE_URL "https://github.com/apache/orc/archive/rel/release-${ORC_VERSION}.tar.gz")
+ set(SNAPPY_SOURCE_URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz")
endif()
if (DEFINED ENV{ARROW_THRIFT_URL})
@@ -203,6 +214,18 @@ else()
set(THRIFT_SOURCE_URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz")
endif()
+if (DEFINED ENV{ARROW_ZLIB_URL})
+ set(ZLIB_SOURCE_URL "$ENV{ARROW_ZLIB_URL}")
+else()
+ set(ZLIB_SOURCE_URL "http://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz")
+endif()
+
+if (DEFINED ENV{ARROW_ZSTD_URL})
+ set(ZSTD_SOURCE_URL "$ENV{ARROW_ZSTD_URL}")
+else()
+ set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz")
+endif()
+
# ----------------------------------------------------------------------
# ExternalProject options
@@ -377,7 +400,7 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIR})
if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
add_custom_target(unittest ctest -L unittest)
- if("$ENV{GTEST_HOME}" STREQUAL "")
+ if("${GTEST_HOME}" STREQUAL "")
if(APPLE)
set(GTEST_CMAKE_CXX_FLAGS "-fPIC -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes")
elseif(NOT MSVC)