You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by uw...@apache.org on 2017/07/10 17:40:14 UTC
parquet-cpp git commit: PARQUET-1048: Apache Arrow static transitive
dependencies
Repository: parquet-cpp
Updated Branches:
refs/heads/master 5c02c7583 -> 562651e11
PARQUET-1048: Apache Arrow static transitive dependencies
Author: Deepak Majeti <de...@hpe.com>
Closes #367 from majetideepak/PARQUET-1048 and squashes the following commits:
42635f2 [Deepak Majeti] try another script
bb96b28 [Deepak Majeti] disable boost shared
b3a0e75 [Deepak Majeti] enable mem check
db50b65 [Deepak Majeti] build arrow shared only if parquet shared is specified
5fe857e [Deepak Majeti] Remove mem check
68f8c59 [Deepak Majeti] Fix Arrow visibility warnings
0bbd073 [Deepak Majeti] add env vars
e4406ab [Deepak Majeti] Add static linking for travis ci
032c8c7 [Deepak Majeti] Fix linking of benchmarks and examples
bdb5b2f [Deepak Majeti] Add Arrow lib transitive dependencies
Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/562651e1
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/562651e1
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/562651e1
Branch: refs/heads/master
Commit: 562651e11a3a148ead4a27c5ab5ede903f1fd5bb
Parents: 5c02c75
Author: Deepak Majeti <de...@hpe.com>
Authored: Mon Jul 10 19:40:10 2017 +0200
Committer: Uwe L. Korn <uw...@apache.org>
Committed: Mon Jul 10 19:40:10 2017 +0200
----------------------------------------------------------------------
.travis.yml | 6 +++
CMakeLists.txt | 39 +++++++++++++-
benchmarks/CMakeLists.txt | 5 +-
ci/travis_script_static.sh | 76 ++++++++++++++++++++++++++++
cmake_modules/ThirdpartyToolchain.cmake | 1 +
examples/CMakeLists.txt | 2 +-
src/parquet/arrow/reader.cc | 4 +-
7 files changed, 127 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b332cf1..70da9ed 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -45,6 +45,12 @@ matrix:
env: PARQUET_BUILD_GROUP=toolchain
before_script:
script:
+ - $TRAVIS_BUILD_DIR/ci/travis_script_static.sh
+ - compiler: gcc
+ os: linux
+ env: PARQUET_BUILD_GROUP=toolchain
+ before_script:
+ script:
- $TRAVIS_BUILD_DIR/ci/travis_script_toolchain.sh
- compiler: clang
os: linux
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5db2a54..b7a41d8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -509,8 +509,45 @@ if ("${PARQUET_ARROW_LINKAGE}" STREQUAL "shared")
set(ARROW_LINK_LIBS
arrow)
else()
+ #############################################################
+ # Transitive Library Linkage
+
+ if (NOT DEFINED ENV{BROTLI_STATIC_LIB_ENC} OR
+ NOT DEFINED ENV{BROTLI_STATIC_LIB_DEC} OR
+ NOT DEFINED ENV{BROTLI_STATIC_LIB_COMMON} OR
+ NOT DEFINED ENV{SNAPPY_STATIC_LIB} OR
+ NOT DEFINED ENV{ZLIB_STATIC_LIB})
+ message(FATAL_ERROR "Missing transitive dependencies for Arrow static linking")
+ endif()
+
+ set(BROTLI_STATIC_LIB_ENC "$ENV{BROTLI_STATIC_LIB_ENC}")
+ set(BROTLI_STATIC_LIB_DEC "$ENV{BROTLI_STATIC_LIB_DEC}")
+ set(BROTLI_STATIC_LIB_COMMON "$ENV{BROTLI_STATIC_LIB_COMMON}")
+ set(SNAPPY_STATIC_LIB "$ENV{SNAPPY_STATIC_LIB}")
+ set(ZLIB_STATIC_LIB "$ENV{ZLIB_STATIC_LIB}")
+
+ add_library(brotli_enc STATIC IMPORTED)
+ set_target_properties(brotli_enc PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_ENC})
+ add_library(brotli_dec STATIC IMPORTED)
+ set_target_properties(brotli_dec PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_DEC})
+ add_library(brotli_common STATIC IMPORTED)
+ set_target_properties(brotli_common PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_COMMON})
+ add_library(snappy STATIC IMPORTED)
+ set_target_properties(snappy PROPERTIES IMPORTED_LOCATION ${SNAPPY_STATIC_LIB})
+ add_library(zlib STATIC IMPORTED)
+ set_target_properties(zlib PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB})
+
+ set(TRANSITIVE_LINK_LIBS
+ snappy
+ zlib
+ brotli_enc
+ brotli_dec
+ brotli_common
+ )
+
set(ARROW_LINK_LIBS
- arrow_static)
+ arrow_static
+ ${TRANSITIVE_LINK_LIBS})
endif()
#############################################################
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/benchmarks/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
index 61c1491..43af746 100644
--- a/benchmarks/CMakeLists.txt
+++ b/benchmarks/CMakeLists.txt
@@ -16,13 +16,14 @@
# under the License.
SET(LINK_LIBS
+ ${ARROW_LINK_LIBS}
thriftstatic)
if (PARQUET_BUILD_BENCHMARKS)
add_executable(decode_benchmark decode_benchmark.cc)
# This uses private APIs
- target_link_libraries(decode_benchmark ${LINK_LIBS}
- arrow
+ target_link_libraries(decode_benchmark
+ ${LINK_LIBS}
parquet_static)
endif()
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/ci/travis_script_static.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_static.sh b/ci/travis_script_static.sh
new file mode 100755
index 0000000..de57acc
--- /dev/null
+++ b/ci/travis_script_static.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+set -xe
+
+# Use a C++11 compiler on Linux
+export CC="gcc-4.9"
+export CXX="g++-4.9"
+
+# ----------------------------------------------------------------------
+# Set up external toolchain
+
+MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+export MINICONDA=$HOME/miniconda
+wget -O miniconda.sh $MINICONDA_URL
+bash miniconda.sh -b -p $MINICONDA
+export PATH="$MINICONDA/bin:$PATH"
+export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+
+conda update -y -q conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+# Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda info -a
+
+conda create -y -q -p $CPP_TOOLCHAIN \
+ boost-cpp thrift-cpp cmake git \
+ -c conda-forge
+
+# ----------------------------------------------------------------------
+
+: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/parquet-build}
+export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
+export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+export BOOST_ROOT=$CPP_TOOLCHAIN
+export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
+export SNAPPY_STATIC_LIB=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/snappy_ep/src/snappy_ep-install/lib/libsnappy.a
+export BROTLI_STATIC_LIB_ENC=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlienc.a
+export BROTLI_STATIC_LIB_DEC=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlidec.a
+export BROTLI_STATIC_LIB_COMMON=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlicommon.a
+export ZLIB_STATIC_LIB=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/zlib_ep/src/zlib_ep-install/lib/libz.a
+
+
+cmake -DPARQUET_CXXFLAGS=-Werror \
+ -DPARQUET_TEST_MEMCHECK=ON \
+ -DPARQUET_ARROW_LINKAGE="static" \
+ -DPARQUET_BUILD_SHARED=OFF \
+ -DPARQUET_BOOST_USE_SHARED=OFF \
+ -DPARQUET_BUILD_BENCHMARKS=ON \
+ -DPARQUET_BUILD_EXAMPLES=ON \
+ -DPARQUET_GENERATE_COVERAGE=1 \
+ $TRAVIS_BUILD_DIR
+
+pushd $CPP_BUILD_DIR
+
+make -j4 || exit 1
+ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; }
+
+popd
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 716debc..85630a4 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -332,6 +332,7 @@ if (NOT ARROW_FOUND)
-DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR}
-DARROW_JEMALLOC=OFF
-DARROW_IPC=OFF
+ -DARROW_BUILD_SHARED=${PARQUET_BUILD_SHARED}
-DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED}
-DARROW_BUILD_TESTS=OFF)
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/examples/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 594295d..721fa9a 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -17,5 +17,5 @@
if (PARQUET_BUILD_EXECUTABLES)
add_executable(reader-writer reader-writer.cc)
- target_link_libraries(reader-writer parquet_shared arrow)
+ target_link_libraries(reader-writer parquet_static)
endif()
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/src/parquet/arrow/reader.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc
index dd1c9d2..addfb85 100644
--- a/src/parquet/arrow/reader.cc
+++ b/src/parquet/arrow/reader.cc
@@ -226,7 +226,7 @@ class ColumnReader::Impl {
};
// Reader implementation for primitive arrays
-class PrimitiveImpl : public ColumnReader::Impl {
+class PARQUET_NO_EXPORT PrimitiveImpl : public ColumnReader::Impl {
public:
PrimitiveImpl(MemoryPool* pool, std::unique_ptr<FileColumnIterator> input)
: pool_(pool),
@@ -300,7 +300,7 @@ class PrimitiveImpl : public ColumnReader::Impl {
};
// Reader implementation for struct array
-class StructImpl : public ColumnReader::Impl {
+class PARQUET_NO_EXPORT StructImpl : public ColumnReader::Impl {
public:
explicit StructImpl(const std::vector<std::shared_ptr<Impl>>& children,
int16_t struct_def_level, MemoryPool* pool, const NodePtr& node)