You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/07/25 17:14:01 UTC
arrow git commit: ARROW-1253: [C++/Python] Speed up C++ / Python
builds by using conda-forge toolchain for thirdparty libraries
Repository: arrow
Updated Branches:
refs/heads/master 08cec9097 -> ed54dce64
ARROW-1253: [C++/Python] Speed up C++ / Python builds by using conda-forge toolchain for thirdparty libraries
@pcmoritz I removed the travis_script_plasma.sh and combined the common bits with travis_script_python.sh; it seemed like there was some redundant work and would be easier to test everything in a single build. Let me know if I messed something up
Author: Wes McKinney <we...@twosigma.com>
Closes #884 from wesm/ARROW-1253 and squashes the following commits:
30a13ee3 [Wes McKinney] Formatting
28bc6be8 [Wes McKinney] Run unit tests from installed library
fcf7940d [Wes McKinney] Prevent existing LD_LIBRARY_PATH from being blown away
c67bb68b [Wes McKinney] Speed up C++ / Python builds by using conda-forge toolchain to avoid rebuilding EPs. Use ninja for builds when using toolchain
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/ed54dce6
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/ed54dce6
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/ed54dce6
Branch: refs/heads/master
Commit: ed54dce64a4651a1d26aed373b09b9af9ee2123e
Parents: 08cec90
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue Jul 25 13:13:56 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue Jul 25 13:13:56 2017 -0400
----------------------------------------------------------------------
.travis.yml | 27 ++---------
ci/travis_before_script_cpp.sh | 50 +++++++++++++------
ci/travis_env_common.sh | 13 +++++
ci/travis_script_cpp.sh | 4 +-
ci/travis_script_plasma.sh | 97 -------------------------------------
ci/travis_script_python.sh | 72 +++++++++++++--------------
cpp/.gitignore | 17 +++++++
cpp/CMakeLists.txt | 28 +++++------
cpp/src/plasma/.gitignore | 18 +++++++
python/doc/source/api.rst | 1 -
python/setup.py | 6 +--
11 files changed, 141 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 9cc2b86..b93f1c2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,6 @@ cache:
ccache: true
directories:
- $HOME/.conda_packages
- - $HOME/.ccache
matrix:
fast_finish: true
@@ -56,6 +55,9 @@ matrix:
before_script:
- export CC="gcc-4.9"
- export CXX="g++-4.9"
+ - export ARROW_TRAVIS_USE_TOOLCHAIN=1
+ - export ARROW_TRAVIS_VALGRIND=1
+ - export ARROW_TRAVIS_PLASMA=1
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -66,6 +68,8 @@ matrix:
cache:
addons:
before_script:
+ - export ARROW_TRAVIS_USE_TOOLCHAIN=1
+ - export ARROW_TRAVIS_PLASMA=1
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
@@ -120,27 +124,6 @@ matrix:
- $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
- - compiler: gcc
- language: cpp
- os: linux
- group: deprecated
- before_script:
- - export CC="gcc-4.9"
- - export CXX="g++-4.9"
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
- - compiler: clang
- osx_image: xcode6.4
- os: osx
- cache:
- addons:
- before_script:
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_plasma.sh
before_install:
- ulimit -c unlimited -S
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/ci/travis_before_script_cpp.sh
----------------------------------------------------------------------
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index e250e70..a1697da 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,22 +15,33 @@
set -ex
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
if [ "$1" == "--only-library" ]; then
only_library_mode=yes
else
only_library_mode=no
+ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
fi
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-
-if [ $only_library_mode == "no" ]; then
- # C++ toolchain
- export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
- export RAPIDJSON_HOME=$CPP_TOOLCHAIN
-
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
# Set up C++ toolchain from conda-forge packages for faster builds
- source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
- conda create -y -q -p $CPP_TOOLCHAIN python=2.7 rapidjson
+ conda create -y -q -p $CPP_TOOLCHAIN python=2.7 \
+ jemalloc=4.4.0 \
+ nomkl \
+ boost-cpp \
+ rapidjson \
+ flatbuffers \
+ gflags \
+ lz4-c \
+ snappy \
+ zstd \
+ brotli \
+ zlib \
+ cmake \
+ curl \
+ thrift-cpp \
+ ninja
fi
if [ $TRAVIS_OS_NAME == "osx" ]; then
@@ -45,7 +56,6 @@ pushd $ARROW_CPP_BUILD_DIR
CMAKE_COMMON_FLAGS="\
-DARROW_BUILD_BENCHMARKS=ON \
-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL \
--DARROW_PLASMA=ON \
-DARROW_NO_DEPRECATED_API=ON"
CMAKE_LINUX_FLAGS=""
CMAKE_OSX_FLAGS=""
@@ -60,8 +70,20 @@ else
# also in the manylinux1 image.
CMAKE_LINUX_FLAGS="\
$CMAKE_LINUX_FLAGS \
--DARROW_JEMALLOC=ON \
--DARROW_TEST_MEMCHECK=ON"
+-DARROW_JEMALLOC=ON"
+fi
+
+# Use Ninja for faster builds when using toolchain
+if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then
+ CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja"
+fi
+
+if [ $ARROW_TRAVIS_PLASMA == "1" ]; then
+ CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_PLASMA=ON"
+fi
+
+if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
+ CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_TEST_MEMCHECK=ON"
fi
if [ $TRAVIS_OS_NAME == "linux" ]; then
@@ -76,7 +98,7 @@ else
$ARROW_CPP_DIR
fi
-make VERBOSE=1 -j4
-make install
+$TRAVIS_MAKE -j4
+$TRAVIS_MAKE install
popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/ci/travis_env_common.sh
----------------------------------------------------------------------
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index a2e5910..4122eac 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -29,6 +29,19 @@ export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
+if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then
+ # C++ toolchain
+ export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+ export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+ export BOOST_ROOT=$CPP_TOOLCHAIN
+
+ export PATH=$CPP_TOOLCHAIN/bin:$PATH
+ export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+ export TRAVIS_MAKE=ninja
+else
+ export TRAVIS_MAKE=make
+fi
+
if [ $TRAVIS_OS_NAME == "osx" ]; then
export GOPATH=$TRAVIS_BUILD_DIR/gopath
fi
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/ci/travis_script_cpp.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh
index c368a1d..a4f6b05 100755
--- a/ci/travis_script_cpp.sh
+++ b/ci/travis_script_cpp.sh
@@ -14,7 +14,7 @@
set -e
-: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
# Check licenses according to Apache policy
git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
@@ -22,7 +22,7 @@ git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz
pushd $CPP_BUILD_DIR
-make lint
+$TRAVIS_MAKE lint
# ARROW-209: checks depending on the LLVM toolchain are disabled temporarily
# until we are able to install the full LLVM toolchain in Travis CI again
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/ci/travis_script_plasma.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_plasma.sh b/ci/travis_script_plasma.sh
deleted file mode 100755
index fa384ad..0000000
--- a/ci/travis_script_plasma.sh
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-set -e
-
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-
-export ARROW_HOME=$ARROW_CPP_INSTALL
-export PYARROW_WITH_PLASMA=1
-
-pushd $ARROW_PYTHON_DIR
-
-function build_arrow_libraries() {
- CPP_BUILD_DIR=$1
- CPP_DIR=$TRAVIS_BUILD_DIR/cpp
-
- mkdir $CPP_BUILD_DIR
- pushd $CPP_BUILD_DIR
-
- cmake -DARROW_BUILD_TESTS=off \
- -DARROW_PYTHON=on \
- -DARROW_PLASMA=on \
- -DCMAKE_INSTALL_PREFIX=$2 \
- $CPP_DIR
-
- make -j4
- make install
-
- popd
-}
-
-python_version_tests() {
- PYTHON_VERSION=$1
- CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
-
- export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
- export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
-
- conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
- source activate $CONDA_ENV_DIR
-
- python --version
- which python
-
- # faster builds, please
- conda install -y -q nomkl
-
- # Expensive dependencies install from Continuum package repo
- conda install -y -q pip numpy pandas cython
-
- # Build C++ libraries
- build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
-
- # Other stuff pip install
- pip install -r requirements.txt
-
- python setup.py build_ext --inplace
-
- python -m pytest -vv -r sxX pyarrow
-
- # Build documentation once
- if [[ "$PYTHON_VERSION" == "3.6" ]]
- then
- conda install -y -q --file=doc/requirements.txt
- python setup.py build_sphinx -s doc/source
- fi
-
- # Build and install pyarrow
- pushd $TRAVIS_BUILD_DIR/python
- python setup.py install
- popd
-
- # Run Plasma tests
- pushd $TRAVIS_BUILD_DIR/python
- python -m pytest pyarrow/tests/test_plasma.py
- if [ $TRAVIS_OS_NAME == "linux" ]; then
- PLASMA_VALGRIND=1 python -m pytest pyarrow/tests/test_plasma.py
- fi
- popd
-}
-
-# run tests for python 2.7 and 3.6
-python_version_tests 2.7
-python_version_tests 3.6
-
-popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/ci/travis_script_python.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh
index fdb5ad6..dcba782 100755
--- a/ci/travis_script_python.sh
+++ b/ci/travis_script_python.sh
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -17,26 +16,14 @@ set -e
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
export ARROW_HOME=$ARROW_CPP_INSTALL
-export PYARROW_WITH_PLASMA=1
-
-pushd $ARROW_PYTHON_DIR
export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env
+export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH
build_parquet_cpp() {
export PARQUET_ARROW_VERSION=$(git rev-parse HEAD)
- conda create -y -q -p $PARQUET_HOME python=3.6 cmake curl
- source activate $PARQUET_HOME
-
- # In case some package wants to download the MKL
- conda install -y -q nomkl
-
- conda install -y -q thrift-cpp snappy zlib brotli boost
- export BOOST_ROOT=$PARQUET_HOME
- export SNAPPY_HOME=$PARQUET_HOME
- export THRIFT_HOME=$PARQUET_HOME
- export ZLIB_HOME=$PARQUET_HOME
- export BROTLI_HOME=$PARQUET_HOME
+ # $CPP_TOOLCHAIN set up in before_script_cpp
+ export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
PARQUET_DIR=$TRAVIS_BUILD_DIR/parquet
mkdir -p $PARQUET_DIR
@@ -48,37 +35,39 @@ build_parquet_cpp() {
cd build-dir
cmake \
+ -GNinja \
-DCMAKE_BUILD_TYPE=debug \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+ -DPARQUET_BOOST_USE_SHARED=off \
-DPARQUET_BUILD_BENCHMARKS=off \
-DPARQUET_BUILD_EXECUTABLES=off \
- -DPARQUET_ZLIB_VENDORED=off \
- -DPARQUET_BUILD_TESTS=on \
+ -DPARQUET_BUILD_TESTS=off \
..
- make -j${CPU_COUNT}
- make install
+ ninja
+ ninja install
popd
}
build_parquet_cpp
-function build_arrow_libraries() {
- CPP_BUILD_DIR=$1
- CPP_DIR=$TRAVIS_BUILD_DIR/cpp
+function rebuild_arrow_libraries() {
+ pushd $ARROW_CPP_BUILD_DIR
- mkdir $CPP_BUILD_DIR
- pushd $CPP_BUILD_DIR
+ # Clear out prior build files
+ rm -rf *
- cmake -DARROW_BUILD_TESTS=off \
+ cmake -GNinja \
+ -DARROW_BUILD_TESTS=off \
+ -DARROW_BUILD_UTILITIES=off \
-DARROW_PLASMA=on \
-DARROW_PYTHON=on \
- -DCMAKE_INSTALL_PREFIX=$2 \
- $CPP_DIR
+ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+ $ARROW_CPP_DIR
- make -j4
- make install
+ ninja
+ ninja install
popd
}
@@ -87,9 +76,6 @@ python_version_tests() {
PYTHON_VERSION=$1
CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
- export ARROW_HOME=$TRAVIS_BUILD_DIR/arrow-install-$PYTHON_VERSION
- export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib
-
conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
source activate $CONDA_ENV_DIR
@@ -103,27 +89,35 @@ python_version_tests() {
conda install -y -q pip numpy pandas cython
# Build C++ libraries
- build_arrow_libraries arrow-build-$PYTHON_VERSION $ARROW_HOME
+ rebuild_arrow_libraries
# Other stuff pip install
+ pushd $ARROW_PYTHON_DIR
pip install -r requirements.txt
-
- python setup.py build_ext --inplace --with-parquet
+ python setup.py build_ext --with-parquet --with-plasma \
+ install --single-version-externally-managed --record=record.text
+ popd
python -c "import pyarrow.parquet"
+ python -c "import pyarrow.plasma"
+
+ if [ $TRAVIS_OS_NAME == "linux" ]; then
+ export PLASMA_VALGRIND=1
+ fi
- python -m pytest -vv -r sxX pyarrow --parquet
+ PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
+ python -m pytest -vv -r sxX -s $PYARROW_PATH --parquet
+ pushd $ARROW_PYTHON_DIR
# Build documentation once
if [[ "$PYTHON_VERSION" == "3.6" ]]
then
conda install -y -q --file=doc/requirements.txt
python setup.py build_sphinx -s doc/source
fi
+ popd
}
# run tests for python 2.7 and 3.6
python_version_tests 2.7
python_version_tests 3.6
-
-popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/cpp/.gitignore
----------------------------------------------------------------------
diff --git a/cpp/.gitignore b/cpp/.gitignore
index 4910544..ec846b3 100644
--- a/cpp/.gitignore
+++ b/cpp/.gitignore
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
thirdparty/
CMakeFiles/
CMakeCache.txt
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 2891a5d..41af1e1 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -582,20 +582,6 @@ if (ARROW_STATIC_LINK_LIBS)
add_dependencies(arrow_dependencies ${ARROW_STATIC_LINK_LIBS})
endif()
-set(ARROW_MIN_TEST_LIBS
- arrow_static
- ${ARROW_STATIC_LINK_LIBS}
- gtest
- gtest_main)
-
-if(NOT MSVC)
- set(ARROW_MIN_TEST_LIBS
- ${ARROW_MIN_TEST_LIBS}
- ${CMAKE_DL_LIBS})
-endif()
-
-set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
-
set(ARROW_BENCHMARK_LINK_LIBS
arrow_static
arrow_benchmark_main
@@ -618,6 +604,20 @@ if (NOT MSVC)
${CMAKE_DL_LIBS})
endif()
+set(ARROW_MIN_TEST_LIBS
+ arrow_static
+ ${ARROW_STATIC_LINK_LIBS}
+ gtest
+ gtest_main)
+
+if(NOT MSVC)
+ set(ARROW_MIN_TEST_LIBS
+ ${ARROW_MIN_TEST_LIBS}
+ ${CMAKE_DL_LIBS})
+endif()
+
+set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
+
if (ARROW_JEMALLOC)
add_definitions(-DARROW_JEMALLOC)
# In the case that jemalloc is only available as a shared library also use it to
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/cpp/src/plasma/.gitignore
----------------------------------------------------------------------
diff --git a/cpp/src/plasma/.gitignore b/cpp/src/plasma/.gitignore
new file mode 100644
index 0000000..163b5c5
--- /dev/null
+++ b/cpp/src/plasma/.gitignore
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+*_generated.h
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/python/doc/source/api.rst
----------------------------------------------------------------------
diff --git a/python/doc/source/api.rst b/python/doc/source/api.rst
index 780aa48..6554465 100644
--- a/python/doc/source/api.rst
+++ b/python/doc/source/api.rst
@@ -225,7 +225,6 @@ In-Memory Object Store
ObjectID
PlasmaClient
PlasmaBuffer
- MutablePlasmaBuffer
.. currentmodule:: pyarrow.parquet
http://git-wip-us.apache.org/repos/asf/arrow/blob/ed54dce6/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 7425b71..be0e26b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -82,6 +82,7 @@ class build_ext(_build_ext):
user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
('build-type=', None, 'build type (debug or release)'),
('with-parquet', None, 'build the Parquet extension'),
+ ('with-plasma', None, 'build the Plasma extension'),
('bundle-arrow-cpp', None,
'bundle the Arrow C++ libraries')] +
_build_ext.user_options)
@@ -101,14 +102,13 @@ class build_ext(_build_ext):
os.environ.get('PYARROW_WITH_PARQUET', '0'))
self.with_plasma = strtobool(
os.environ.get('PYARROW_WITH_PLASMA', '0'))
- if self.with_plasma and "plasma" not in self.CYTHON_MODULE_NAMES:
- self.CYTHON_MODULE_NAMES.append("plasma")
self.bundle_arrow_cpp = strtobool(
os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
CYTHON_MODULE_NAMES = [
'lib',
- '_parquet']
+ '_parquet',
+ 'plasma']
def _run_cmake(self):
# The directory containing this setup.py